diff --git a/server/tests/postgres/sqlite-move.test.ts b/server/tests/postgres/sqlite-move.test.ts new file mode 100644 index 000000000..f0dee47ae --- /dev/null +++ b/server/tests/postgres/sqlite-move.test.ts @@ -0,0 +1,289 @@ +/** + * PGMOVE-001..006: moving a SQLite install into Postgres + * (`src/db/engine/postgres/sqlite-move/`), end to end on the lane's server. + * + * A SQLite file is migrated by the SQLite engine's own bootstrap and seeded + * through the test factories with rows across the main tables, including a + * reply to a chat message (a table that points at itself) and a plugin's own + * database. The move copies it into this file's database, emptied first, and + * the app then boots on the result: the rows read back through the entities + * and the repositories exactly as they read from the SQLite file, the plugin + * finds its rows in its schema, new ids continue after the copied ones, and + * a second move into the now filled database is refused. + */ +import { buildApp } from '../../src/bootstrap'; +import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; +import { readPostgresBaseline } from '../../src/db/engine/postgres/baseline'; +import { PluginDataSlots, PostgresPluginDataStore } from '../../src/db/engine/postgres/plugin-data-store'; +import { MoveRefusal, moveSqliteToPostgres, type MoveSummary } from '../../src/db/engine/postgres/sqlite-move/move'; +import { SqliteEngine } from '../../src/db/engine/sqlite/sqlite-engine'; +import { AppSettings } from '../../src/db/entities/AppSettings.entity'; +import { BudgetItems } from '../../src/db/entities/BudgetItems.entity'; +import { CollabMessages } from '../../src/db/entities/CollabMessages.entity'; +import { DayAssignments } from '../../src/db/entities/DayAssignments.entity'; +import { Days } from '../../src/db/entities/Days.entity'; +import { Places } from '../../src/db/entities/Places.entity'; +import { ReservationEndpoints } from '../../src/db/entities/ReservationEndpoints.entity'; +import { Reservations } from '../../src/db/entities/Reservations.entity'; +import { Settings } from '../../src/db/entities/Settings.entity'; +import { TripFiles } from '../../src/db/entities/TripFiles.entity'; +import { Trips } from '../../src/db/entities/Trips.entity'; +import { Users } from '../../src/db/entities/Users.entity'; +import { knownMigrationNames } from '../../src/db/known-migrations'; +import { runSchemaBootstrap, type AnyOrm } from '../../src/db/orm'; +import { createBoundSqliteDriver } from '../../src/db/orm-driver'; +import { makeBudgetItem } from '../helpers/factories/budget'; +import { makeCollabMessage } from '../helpers/factories/collab'; +import { inContext, type FactoryOrm } from '../helpers/factories/context'; +import { makeTripFile } from '../helpers/factories/files'; +import { makeDayAssignment } from '../helpers/factories/itinerary'; +import { makePlace } from '../helpers/factories/places'; +import { makeReservation, makeReservationEndpoint } from '../helpers/factories/reservations'; +import { findRows } from '../helpers/factories/rows'; +import { setAppSetting, setUserSetting } from '../helpers/factories/settings'; +import { makeTrip, readTripDays } from '../helpers/factories/trips'; +import { makeUser } from '../helpers/factories/users'; +import { databaseUrl, withAdmin } from './databases'; +import { MikroORM, type EntityClass } from '@mikro-orm/core'; +import type { AbstractSqlConnection } from '@mikro-orm/sql'; +import type { INestApplication } from '@nestjs/common'; + +import Database from 'better-sqlite3'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const PLUGIN_ID = 'move-notes'; + +let dir: string; +let handle: Database.Database; +let sqliteOrm: AnyOrm; +let app: INestApplication; +let pgOrm: MikroORM; +let summary: MoveSummary; +let tripId: number; + +/** An ORM on its own SQLite file, migrated and seeded the way the SQLite engine brings a file to head. */ +async function migratedSqlite(file: string): Promise { + handle = new Database(file); + handle.pragma('foreign_keys = ON'); + const config = buildOrmConfig(new SqliteEngine()); + const orm = await MikroORM.init({ + ...config, + driver: createBoundSqliteDriver(() => handle), + dbName: file, + migrations: { ...config.migrations, silent: true }, + }); + await runSchemaBootstrap(orm, { snapshot: false }); + return orm; +} + +/** Rows across the main tables, through the factories. */ +async function seed(orm: FactoryOrm): Promise { + const { user } = await makeUser(orm, { display_name: 'Ada Lovelace' }); + const { user: friend } = await makeUser(orm); + const trip = await makeTrip(orm, user.id, { + title: 'Lisbon in spring', + start_date: '2027-04-01', + end_date: '2027-04-03', + currency: 'EUR', + }); + tripId = trip.id; + const days = await readTripDays(orm, trip.id); + const tower = await makePlace(orm, trip.id, { name: 'Torre de Belém', lat: 38.6916, lng: -9.216, notes: 'Go early' }); + const tram = await makePlace(orm, trip.id, { name: 'Tram 28', lat: 38.7139, lng: -9.1334, category: null }); + await makeDayAssignment(orm, days[0].id, tower.id, { notes: 'Morning' }); + await makeDayAssignment(orm, days[1].id, tram.id); + const flight = await makeReservation(orm, trip.id, { title: 'TP 1234', type: 'flight', day: days[0].id }); + await makeReservationEndpoint(orm, flight.id, { role: 'from', sequence: 0 }); + await makeReservationEndpoint(orm, flight.id, { + role: 'to', + sequence: 1, + name: 'Lisbon Humberto Delgado', + code: 'LIS', + lat: 38.7742, + lng: -9.1342, + }); + await makeBudgetItem(orm, trip.id, { name: 'Flights', total_price: 412.37, category: 'Transport' }); + await makeBudgetItem(orm, trip.id, { name: 'Pastéis', total_price: 7.5, category: 'Food' }); + await makeTripFile(orm, trip.id, { original_name: 'Boarding pass.pdf', file_size: 52_431 }); + await setUserSetting(orm, user.id, 'temperature_unit', 'celsius'); + await setAppSetting(orm, 'move_test_marker', 'ünïcode ✓'); + const question = await makeCollabMessage(orm, trip.id, user.id, { text: 'Who books the tram?' }); + await makeCollabMessage(orm, trip.id, friend.id, { text: 'I do', replyToRef: question.id }); +} + +/** A plugin's own database, as the SQLite store keeps it. */ +function seedPluginData(root: string): void { + fs.mkdirSync(path.join(root, PLUGIN_ID), { recursive: true }); + const plugin = new Database(path.join(root, PLUGIN_ID, 'plugin.db')); + plugin.exec( + 'create table _plugin_migrations (id text primary key, applied_at integer);' + + "insert into _plugin_migrations values ('001-notes', 1700000000000);" + + "create table notes (id integer primary key autoincrement, body text not null default '', score real, raw blob);" + + 'create unique index notes_body on notes (body);' + + "insert into notes (body, score, raw) values ('first', 1.5, x'00ff'), ('second', null, null), ('gone', 2, null);" + + "delete from notes where body = 'gone';", + ); + plugin.close(); +} + +/** The worker's database, emptied: the move fills an empty database only. */ +async function emptyWorkerDatabase(): Promise { + const adminUrl = process.env.TREK_TEST_DB_URL as string; + const name = new URL(process.env.TREK_DB_URL as string).pathname.slice(1); + expect(process.env.TREK_DB_URL).toBe(databaseUrl(adminUrl, name)); + await withAdmin(adminUrl, async (run) => { + await run(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); + await run(`CREATE DATABASE "${name}"`); + }); +} + +/** Runs the move into the worker's database through an ORM of the Postgres engine's own config. */ +async function move(): Promise { + const config = buildOrmConfig(resolveEngine()); + const orm = await MikroORM.init({ + ...config, + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + migrations: { ...config.migrations, silent: true }, + }); + try { + return await moveSqliteToPostgres({ + sqliteFile: path.join(dir, 'travel.db'), + pluginsDataDir: path.join(dir, 'plugins-data'), + orm, + known: knownMigrationNames(), + baseline: readPostgresBaseline, + log: () => undefined, + }); + } finally { + await orm.close(true); + } +} + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-pg-move-')); + sqliteOrm = await migratedSqlite(path.join(dir, 'travel.db')); + await seed(sqliteOrm); + seedPluginData(path.join(dir, 'plugins-data')); + await emptyWorkerDatabase(); + summary = await move(); + app = await buildApp(); + pgOrm = app.get(MikroORM); +}, 180_000); + +afterAll(async () => { + await app?.close(); + await sqliteOrm?.close(true); + if (handle?.open) handle.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +/** Rows in a stable order: Postgres and SQLite need not sort text alike. */ +function sorted(rows: T[]): T[] { + return [...rows].sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); +} + +/** Runs `read` on both databases in a request context of its own. */ +async function bothSides(read: (orm: FactoryOrm) => Promise): Promise<{ sqlite: T; postgres: T }> { + return { sqlite: await read(sqliteOrm), postgres: await read(pgOrm) }; +} + +describe('moving a SQLite install to Postgres', () => { + it('PGMOVE-001: copies every table and reports what it moved', async () => { + const byLabel = new Map(summary.tables.map((table) => [table.label, table.rows])); + expect(byLabel.get('users')).toBe((await findRows(sqliteOrm, Users)).length); + expect(byLabel.get('trips')).toBe(1); + expect(byLabel.get('days')).toBe(3); + expect(byLabel.get('reservation_endpoints')).toBe(2); + expect(byLabel.has('mikro_orm_migrations')).toBe(false); + expect(summary.sequences).toBeGreaterThan(100); + expect(summary.plugins).toEqual([ + expect.objectContaining({ + pluginId: PLUGIN_ID, + schema: 'trek_plugin_move_notes', + tables: [ + expect.objectContaining({ label: '_plugin_migrations', rows: 1 }), + expect.objectContaining({ label: 'notes', rows: 2 }), + ], + }), + ]); + }); + + const entities: Array<[string, EntityClass]> = [ + ['users', Users], + ['trips', Trips], + ['days', Days], + ['places', Places], + ['day_assignments', DayAssignments], + ['reservations', Reservations], + ['reservation_endpoints', ReservationEndpoints], + ['budget_items', BudgetItems], + ['trip_files', TripFiles], + ['settings', Settings], + ['collab_messages', CollabMessages], + ]; + it.each(entities)('PGMOVE-002: %s read through the entities exactly as on SQLite', async (_name, entity) => { + const { sqlite, postgres } = await bothSides((orm) => findRows(orm, entity)); + expect(sqlite.length).toBeGreaterThan(0); + expect(sorted(postgres)).toEqual(sorted(sqlite)); + }); + + it('PGMOVE-002b: app_settings keeps every row of the file (the boot adds its own keys beside them)', async () => { + const { sqlite, postgres } = await bothSides((orm) => findRows(orm, AppSettings)); + expect(sqlite.some((row) => row.key === 'move_test_marker')).toBe(true); + expect(postgres).toEqual(expect.arrayContaining(sqlite)); + }); + + it('PGMOVE-003: the repositories answer the same on both engines', async () => { + const { sqlite, postgres } = await bothSides((orm) => + inContext(orm, async (em) => ({ + days: await em.getRepository(Days).listByTrip(tripId), + places: await em.getRepository(Places).listAllForTrip(tripId), + assignments: await em.getRepository(DayAssignments).listAllForTrip(tripId), + reservations: await em.getRepository(Reservations).listForTrip(tripId), + endpoints: await em.getRepository(Reservations).listEndpointsForShare(tripId), + budget: await em.getRepository(BudgetItems).listWithCategoryOrder(tripId), + })), + ); + for (const key of Object.keys(sqlite) as Array) { + expect(sorted(postgres[key] as unknown[]), key).toEqual(sorted(sqlite[key] as unknown[])); + } + expect(postgres.endpoints).toHaveLength(2); + }); + + it('PGMOVE-004: a reply keeps the message it answers (a table pointing at itself)', async () => { + const messages = await findRows(pgOrm, CollabMessages); + const reply = messages.find((message) => message.text === 'I do'); + const question = messages.find((message) => message.text === 'Who books the tram?'); + expect(reply?.reply_to).toBe(question?.id); + }); + + it('PGMOVE-005: new rows continue after the copied ids; the plugin finds its rows and its migration record', async () => { + const { user } = await makeUser(pgOrm); + const highest = Math.max(...(await findRows(sqliteOrm, Users)).map((row) => row.id)); + expect(user.id).toBeGreaterThan(highest); + + const client = (pgOrm.em.getConnection() as AbstractSqlConnection).getClient(); + const store = new PostgresPluginDataStore(PLUGIN_ID, client, new PluginDataSlots(1)); + expect(await store.query('select id, body, score from notes order by id')).toEqual([ + { id: 1, body: 'first', score: 1.5 }, + { id: 2, body: 'second', score: null }, + ]); + // The migration the plugin ran on SQLite is not run again, and the next id skips the one SQLite handed out. + expect(await store.migrate('001-notes', 'create table notes (id bigint)')).toEqual({ applied: false }); + await store.exec("insert into notes (body) values ('third')"); + expect(await store.query("select id from notes where body = 'third'")).toEqual([{ id: 4 }]); + store.close(); + }); + + it('PGMOVE-006: a second move into the filled database is refused and changes nothing', async () => { + const before = await findRows(pgOrm, Trips); + await expect(move()).rejects.toBeInstanceOf(MoveRefusal); + await expect(move()).rejects.toThrow(/The Postgres database is not empty/); + expect(await findRows(pgOrm, Trips)).toEqual(before); + }); +});