diff --git a/server/src/db/engine/postgres/pg-tools.ts b/server/src/db/engine/postgres/pg-tools.ts index 1091768aa..37659ba77 100644 --- a/server/src/db/engine/postgres/pg-tools.ts +++ b/server/src/db/engine/postgres/pg-tools.ts @@ -243,7 +243,7 @@ async function dumpPluginSchemas(source: string, toc: string, binDir: string | u } /** The tables in the current schema of the database at `url`, read before TREK opens it. */ -export async function pgTableNames(url: string, binDir: string | undefined): Promise { +async function pgTableNames(url: string, binDir: string | undefined): Promise { const out = await run( 'psql', binDir, @@ -253,6 +253,30 @@ export async function pgTableNames(url: string, binDir: string | undefined): Pro return out.split('\n').filter((line) => line !== ''); } +/** The migration history's table, the first one a boot creates. */ +const HISTORY_TABLE = 'mikro_orm_migrations'; + +/** + * The tables that make the current schema of the database at `url` a + * database in use, read before TREK opens it: every table, except a + * migration history that is the only table and records nothing. A first boot + * that failed while it created the schema left such a table behind before + * the bootstrap created it in the schema's own transaction, and it holds + * nothing a restore, which clears the schema first, could lose. Any answer + * but a count of 0 keeps the table, so the check fails closed. + */ +export async function pgTablesInUse(url: string, binDir: string | undefined): Promise { + const tables = await pgTableNames(url, binDir); + if (tables.length !== 1 || tables[0] !== HISTORY_TABLE) return tables; + const out = await run( + 'psql', + binDir, + ['--no-psqlrc', '--no-align', '--tuples-only', `--command=select count(*) as history_rows from ${HISTORY_TABLE}`], + pgConnectionEnv(url), + ); + return out.trim() === '0' ? [] : tables; +} + /** * The migration names in a restore script's COPY block for * `mikro_orm_migrations`, read from its `name` column; none when the script diff --git a/server/src/db/engine/postgres/schema-bootstrap.ts b/server/src/db/engine/postgres/schema-bootstrap.ts index 38597dfaf..0b6871db2 100644 --- a/server/src/db/engine/postgres/schema-bootstrap.ts +++ b/server/src/db/engine/postgres/schema-bootstrap.ts @@ -84,9 +84,10 @@ async function currentTables(connection: Connection): Promise { } /** - * Creates the schema in an empty Postgres database: the baseline's statements - * and the record of every migration it stands for, in one transaction (DDL is - * transactional on Postgres, so the database ends with all of it or none). A + * Creates the schema in an empty Postgres database: the migration history's + * table, the baseline's statements and the record of every migration it + * stands for, in one transaction (DDL is transactional on Postgres, so the + * database ends with all of it or none, not even an empty history table). A * database holding tables beside no TREK history is refused before anything is * written. */ @@ -104,12 +105,21 @@ async function createFromBaseline( 'migration history. Point TREK_DB_URL at an empty database.', ); } - await storage.ensureTable(); const schema = baseline(); logger.log(`Creating the schema in an empty Postgres database (through ${migrationLabel(schema.through)})`); await connection.transactional(async (trx) => { - await connection.execute(schema.statements.join(';\n'), [], 'run', trx); - await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx))); + // The history table is created inside the transaction too: MikroORM + // creates it on the storage's master transaction, and without one it would + // commit on its own, so a baseline that fails would leave it behind and a + // later RESTORE_FROM_BACKUP would take the database for one in use. + storage.setMasterMigration(trx); + try { + await storage.ensureTable(); + await connection.execute(schema.statements.join(';\n'), [], 'run', trx); + await Promise.all(schema.migrations.map((name) => storage.logMigration({ name }, trx))); + } finally { + storage.unsetMasterMigration(); + } }); } diff --git a/server/src/nest/backup/boot-restore.ts b/server/src/nest/backup/boot-restore.ts index 626c67b9d..388deb593 100644 --- a/server/src/nest/backup/boot-restore.ts +++ b/server/src/nest/backup/boot-restore.ts @@ -1,4 +1,4 @@ -import { pgRestore, pgTableNames } from '../../db/engine/postgres/pg-tools'; +import { pgRestore, pgTablesInUse } from '../../db/engine/postgres/pg-tools'; import { stageExtractedPluginTrees } from '../plugins/plugin-backup'; import { checkBackupDatabase, @@ -68,10 +68,14 @@ export type BootRestoreOutcome = export class BootRestoreError extends Error {} -/** A database is there already: on a file, the file; on Postgres, any table in its schema. */ +/** + * A database is there already: on a file, the file; on Postgres, any table in + * its schema but a lone migration history that records nothing, which a first + * boot cut short leaves (pgTablesInUse). + */ async function existingDatabase(plan: BootRestorePlan): Promise { if (!plan.postgres) return fs.existsSync(plan.dbFile) ? plan.dbFile : null; - const tables = await pgTableNames(plan.postgres.url, plan.postgres.binDir); + const tables = await pgTablesInUse(plan.postgres.url, plan.postgres.binDir); return tables.length > 0 ? `the Postgres database (${tables.length} table(s))` : null; } diff --git a/server/tests/helpers/fake-pg-tools.ts b/server/tests/helpers/fake-pg-tools.ts index 77db38bc4..da79e64fb 100644 --- a/server/tests/helpers/fake-pg-tools.ts +++ b/server/tests/helpers/fake-pg-tools.ts @@ -59,11 +59,16 @@ export function answersSchema(name = 'public', quoted = name, server: FakeServer return printsWhen('quote_ident', `${name}\t${quoted}\t${versionNum}\t${String(privileged)}\t${plugins}\n`); } -/** psql's table listing, one name per line, as pgTableNames reads it. */ +/** psql's table listing, one name per line, as pgTablesInUse reads it. */ export function answersTables(names: readonly string[]): string { return printsWhen('table_name', names.map((name) => `${name}\n`).join('')); } +/** psql's row count of the migration history, as pgTablesInUse reads it when that table stands alone. */ +export function answersHistoryRows(count: number): string { + return printsWhen('history_rows', `${count}\n`); +} + /** Writes `content` to the output file the call names with `--file=`. */ export function writesFile(content: string): string { return `printf '%s' ${shellWord(content)} | fake_write "$@"`; diff --git a/server/tests/postgres/schema-bootstrap.test.ts b/server/tests/postgres/schema-bootstrap.test.ts new file mode 100644 index 000000000..b6a52ec9a --- /dev/null +++ b/server/tests/postgres/schema-bootstrap.test.ts @@ -0,0 +1,71 @@ +/** + * PGBOOTLANE-001: a first boot on an empty Postgres database that fails while + * it creates the schema (here a baseline statement the server refuses, as one + * without ICU refuses the `nocase` collation) leaves the database as empty as + * it found it, the migration history's table included. A table left behind + * would make RESTORE_FROM_BACKUP take the database for one in use on the next + * start and bring up an empty instance instead of the backup. + * + * The suite works on a scratch database of its own beside the lane's, created + * and dropped here, since the lane's copy of the template already holds the + * schema. + */ +import { CURRENT_SCHEMA_TABLES_SQL } from '../../src/db/dialect/catalog'; +import { buildOrmConfig, resolveEngine } from '../../src/db/engine'; +import { bootstrapPostgresSchema } from '../../src/db/engine/postgres/schema-bootstrap'; +import { databaseUrl, withAdmin } from './databases'; +import { MikroORM } from '@mikro-orm/core'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const adminUrl = process.env.TREK_TEST_DB_URL as string; +const SCRATCH = `trek_test_boot_${process.pid}`; + +let orm: MikroORM; + +beforeAll(async () => { + await withAdmin(adminUrl, async (run) => { + await run(`DROP DATABASE IF EXISTS "${SCRATCH}" WITH (FORCE)`); + await run(`CREATE DATABASE "${SCRATCH}"`); + }); + const config = buildOrmConfig(resolveEngine()); + orm = await MikroORM.init({ + ...config, + clientUrl: databaseUrl(adminUrl, SCRATCH), + entities: [], + entitiesTs: [], + discovery: { warnWhenNoEntities: false }, + migrations: { ...config.migrations, silent: true }, + pool: { min: 0, max: 2 }, + }); +}, 60_000); + +afterAll(async () => { + await orm?.close(true); + await withAdmin(adminUrl, (run) => run(`DROP DATABASE IF EXISTS "${SCRATCH}" WITH (FORCE)`)); + // Dropping a database waits for a checkpoint, which takes seconds on a slow disk. +}, 60_000); + +describe('a first boot that fails on Postgres', () => { + it('PGBOOTLANE-001: leaves no table behind, not even the migration history', async () => { + const first = 'Migration20200101000000_baseline_schema'; + await expect( + bootstrapPostgresSchema(orm, { + baseline: () => ({ + through: first, + migrations: [first], + statements: [ + 'create table "boot_probe" ("id" bigserial primary key)', + 'create table "broken" ("a" no_such_type)', + ], + }), + known: new Set([first]), + safetyNet: { beforeMigrate: async () => undefined, restoreHint: async () => null }, + sqliteLeftBehind: () => null, + }), + ).rejects.toThrow(/no_such_type/); + + const tables = await orm.em.getConnection().execute>(CURRENT_SCHEMA_TABLES_SQL); + expect(tables.map((row) => row.table_name)).toEqual([]); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/schema-bootstrap.test.ts b/server/tests/unit/db/engine/postgres/schema-bootstrap.test.ts index 816444459..31b8e063f 100644 --- a/server/tests/unit/db/engine/postgres/schema-bootstrap.test.ts +++ b/server/tests/unit/db/engine/postgres/schema-bootstrap.test.ts @@ -41,10 +41,15 @@ interface FakeDb { migrations: { tableName?: string }; /** Whether the users table holds a row: a database in use does. */ accounts: boolean; + /** The transaction the history table was created in, when it was created in one. */ + historyCreatedIn: unknown; + /** A baseline statement that fails, as on a server without ICU. */ + failingBaseline?: boolean; } type FakeState = Pick & { history?: boolean; + failingBaseline?: boolean; server?: Partial; accounts?: boolean; }; @@ -59,15 +64,20 @@ function fakeOrm(state: FakeState): { migrations: { tableName: 'mikro_orm_migrations' }, history: state.history ?? state.executed.length > 0, accounts: state.accounts ?? true, + // Like MikroORM's: created on the storage's master transaction, and + // committed on its own when there is none. ensureTable: vi.fn(async () => { db.history = true; + db.historyCreatedIn = master; }), + historyCreatedIn: undefined, sql: [], logged: [], inTransaction: [], up: vi.fn(async () => undefined), }; const trx = { trx: true }; + let master: unknown; const connection = { execute: vi.fn(async (sql: string, _params?: unknown[], _method?: string, ctx?: unknown) => { if (sql.includes('server_version_num')) return [db.server]; @@ -77,12 +87,27 @@ function fakeOrm(state: FakeState): { } db.sql.push(sql); db.inTransaction.push(ctx === trx); + if (db.failingBaseline) throw new Error('collation provider "icu" is not supported on this platform'); return []; }), - transactional: vi.fn(async (work: (ctx: unknown) => Promise) => work(trx)), + // A failed transaction takes back what was created in it. + transactional: vi.fn(async (work: (ctx: unknown) => Promise) => { + try { + await work(trx); + } catch (err) { + if (db.historyCreatedIn === trx) db.history = false; + throw err; + } + }), }; const storage = { ensureTable: db.ensureTable, + setMasterMigration: (ctx: unknown) => { + master = ctx; + }, + unsetMasterMigration: () => { + master = undefined; + }, logMigration: vi.fn(async ({ name }: { name: string }, ctx?: unknown) => { db.logged.push(name); db.inTransaction.push(ctx === trx); @@ -157,6 +182,19 @@ describe('bootstrapPostgresSchema', () => { expect(db.ensureTable).not.toHaveBeenCalled(); }); + it('PGBOOT-009: the history table is created inside the baseline transaction, so a failed first boot leaves no table', async () => { + const { orm, db } = fakeOrm({ executed: [], pending: [M1, M2], tables: [], failingBaseline: true }); + await expect(bootstrapPostgresSchema(orm, deps())).rejects.toThrow('collation provider "icu"'); + expect(db.ensureTable).toHaveBeenCalledTimes(1); + expect(db.historyCreatedIn).toEqual({ trx: true }); + // Nothing is left that would make RESTORE_FROM_BACKUP take it for a database in use. + expect(db.history).toBe(false); + + const fresh = fakeOrm({ executed: [], pending: [M1, M2], tables: [] }); + await bootstrapPostgresSchema(fresh.orm, deps()); + expect(fresh.db.historyCreatedIn).toEqual({ trx: true }); + }); + it('PGBOOT-008: an empty history table beside no other table is a fresh database (a first boot cut short)', async () => { const { orm, db } = fakeOrm({ executed: [], pending: [M1, M2], tables: [], history: true }); await bootstrapPostgresSchema(orm, deps()); diff --git a/server/tests/unit/nest/boot-restore.test.ts b/server/tests/unit/nest/boot-restore.test.ts index f4232e665..484a4f001 100644 --- a/server/tests/unit/nest/boot-restore.test.ts +++ b/server/tests/unit/nest/boot-restore.test.ts @@ -10,6 +10,7 @@ import { knownMigrationNames } from '../../../src/db/known-migrations'; import { BootRestoreError, restoreOnFirstBoot } from '../../../src/nest/backup/boot-restore'; import { answersSchema, + answersHistoryRows, answersTables, installFakeTool, listsMigrations, @@ -141,11 +142,16 @@ describe('restoreOnFirstBoot', () => { * known migration as the dump's history, or writes a restricted script. * Their arguments are pinned in pg-tools.test.ts. */ - const postgresPlan = (archivePath: string, tables: string[] = []) => { + const postgresPlan = (archivePath: string, tables: string[] = [], historyRows = 0) => { const bin = path.join(root, 'bin'); fs.mkdirSync(bin, { recursive: true }); installFakeTool(bin, 'psql', { - snippets: [answersSchema(), answersTables(tables), touches(path.join(root, 'loaded'))], + snippets: [ + answersSchema(), + answersTables(tables), + answersHistoryRows(historyRows), + touches(path.join(root, 'loaded')), + ], }); const migration = [...knownMigrationNames()][0]; installFakeTool(bin, 'pg_restore', { snippets: [listsMigrations([migration]), writesRestrictedScript()] }); @@ -179,6 +185,21 @@ describe('restoreOnFirstBoot', () => { expect(fs.existsSync(dbFile)).toBe(false); }); + it('BOOT-RESTORE-004e: a lone migration history that records nothing (a first boot cut short) is loaded over', async () => { + const plan = postgresPlan(await archive({ 'travel.pgdump': 'dump' }), ['mikro_orm_migrations']); + const out = await restoreOnFirstBoot(plan); + expect(out.restored).toBe(true); + expect(loaded()).toBe(true); + }); + + it('BOOT-RESTORE-004f: a migration history with rows is a database in use and left alone', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const recorded = postgresPlan(await archive({ 'travel.pgdump': 'dump' }), ['mikro_orm_migrations'], 3); + expect(await restoreOnFirstBoot(recorded)).toEqual({ restored: false, reason: 'database-exists' }); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('the Postgres database (1 table(s))')); + expect(loaded()).toBe(false); + }); + it('BOOT-RESTORE-004d: a SQLite backup is refused before anything is loaded, leaving no scratch', async () => { const plan = postgresPlan(await goodArchive()); await expect(restoreOnFirstBoot(plan)).rejects.toThrow( diff --git a/wiki/Database-Postgres.md b/wiki/Database-Postgres.md index c60ae2c6c..551b48e4f 100644 --- a/wiki/Database-Postgres.md +++ b/wiki/Database-Postgres.md @@ -96,7 +96,7 @@ The database already exists, so the new init script does not run again, and TREK On the first start against an empty database, TREK: -1. Creates the schema from its baseline and records every migration the baseline covers as applied, in one transaction. +1. Creates the schema from its baseline and records every migration the baseline covers as applied, in one transaction, the table of the migration history included. A first start that fails there (a server without ICU, a dropped connection) leaves the database as empty as it was. 2. Applies any migrations after the baseline. 3. Seeds its default data and creates the admin account, from `ADMIN_EMAIL` and `ADMIN_PASSWORD` when set. Otherwise it prints a random password to the log. @@ -180,7 +180,7 @@ Restoring a backup on Postgres follows these rules: - **The role must not be a superuser.** The restore refuses a role that is a superuser or a member of one, holds `pg_execute_server_program`, `pg_read_server_files` or `pg_write_server_files`, or has CREATEROLE on PostgreSQL 14 or 15. See [A role that is not a superuser](#a-role-that-is-not-a-superuser). - **The restore needs free space in the data directory.** `pg_restore` writes the dump out as a SQL script next to the unpacked archive before `psql` runs it, and the script is several times larger than the dump. It may grow to `BACKUP_MAX_DECOMPRESSED_MB` (5 GB by default); a larger one is stopped, nothing is replaced, and the work files are removed. Raise the limit for a genuine backup that needs more. -`RESTORE_FROM_BACKUP` works on Postgres too. On the first start, TREK loads the archive's dump into the database when that database holds no tables. When the database already has tables, the variable is ignored and TREK logs a warning. See [Backups: Restoring on a new install, before setup](Backups#restoring-on-a-new-install-before-setup). +`RESTORE_FROM_BACKUP` works on Postgres too. On the first start, TREK loads the archive's dump into the database when that database holds no tables. An empty migration history with no other table beside it counts as no tables. When the database already has tables, the variable is ignored and TREK logs a warning. See [Backups: Restoring on a new install, before setup](Backups#restoring-on-a-new-install-before-setup). ## Plugin data