feat(db): run TREK on PostgreSQL

TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before
otherwise: an engine port under src/db/engine, the dialect helpers under
src/db/dialect, a Postgres baseline for the migrations up to the cut,
pg_dump/pg_restore backups, a probe of every repository method against a
real server and a Postgres test lane in CI.
This commit is contained in:
Isaias Tavares
2026-11-05 01:15:21 -03:00
committed by Maurice
parent 17b47b38b9
commit 2e3762afe1
176 changed files with 6728 additions and 777 deletions
+36
View File
@@ -0,0 +1,36 @@
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
import { MikroORM } from '@mikro-orm/core';
/**
* Brings the template database TREK_DB_URL names to the current schema, the
* way a first boot on an empty Postgres database does: the engine's own
* bootstrap (baseline, the migrations after its cut, seeders), with no
* pre-migrate copy, since the database is new.
*/
async function main(): Promise<void> {
const engine = resolveEngine();
if (engine.id !== 'postgres') throw new Error('build-template.ts runs on the Postgres engine only (TREK_DB_URL).');
const config = buildOrmConfig(engine);
const orm = await MikroORM.init({
...config,
// As src/db/standalone-orm.ts: the bootstrap runs SQL and seeders, which
// need no entity metadata.
entities: [],
entitiesTs: [],
discovery: { warnWhenNoEntities: false },
migrations: { ...config.migrations, silent: true },
});
try {
await engine.schemaBootstrap(orm, { snapshot: false });
process.stdout.write(`[test-db] Postgres template ready: ${engine.describe()}\n`);
} finally {
await orm.close(true);
}
}
main().catch((err: unknown) => {
process.stderr.write(
`[test-db] failed to build the Postgres template: ${err instanceof Error ? err.stack : String(err)}\n`,
);
process.exit(1);
});
+37
View File
@@ -0,0 +1,37 @@
import { MikroORM } from '@mikro-orm/postgresql';
/**
* The databases the Postgres lane creates on the server TREK_TEST_DB_URL names:
* one migrated template per run, and one copy of it per test file.
*/
export const TEMPLATE_DATABASE = 'trek_test_template';
export const WORKER_DATABASE_PREFIX = 'trek_test_w';
/** `adminUrl` pointed at `database`, everything else (credentials, host, sslmode) kept. */
export function databaseUrl(adminUrl: string, database: string): string {
const url = new URL(adminUrl);
url.pathname = `/${database}`;
return url.toString();
}
type Run = <T = unknown>(sql: string, params?: unknown[]) => Promise<T[]>;
/**
* Runs `fn` on a connection to the admin database, outside any transaction
* (CREATE and DROP DATABASE refuse to run in one), through the same Postgres
* driver the app uses.
*/
export async function withAdmin<T>(adminUrl: string, fn: (run: Run) => Promise<T>): Promise<T> {
const orm = await MikroORM.init({
clientUrl: adminUrl,
entities: [],
discovery: { warnWhenNoEntities: false },
pool: { min: 0, max: 1 },
});
try {
const connection = orm.em.getConnection();
return await fn((sql, params = []) => connection.execute(sql, params, 'all'));
} finally {
await orm.close(true);
}
}
+60
View File
@@ -0,0 +1,60 @@
import { databaseUrl, TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, withAdmin } from './databases';
import { execFileSync } from 'node:child_process';
import path from 'node:path';
/**
* Runs once per `npm run test:pg`, before any worker starts: the Postgres
* counterpart of tests/global-setup.ts's schema snapshot. One template
* database is created and brought to the current schema the way a first boot
* does (baseline, migrations after its cut, seeders); every test file then
* gets a copy of it (tests/postgres/setup.ts), which Postgres makes at the
* file level, far faster than migrating per file.
*
* TREK_TEST_DB_URL is required: a lane that skipped without it would pass
* having tested nothing.
*/
export async function setup(): Promise<void> {
const adminUrl = process.env.TREK_TEST_DB_URL;
if (!adminUrl) {
throw new Error(
'The Postgres lane needs TREK_TEST_DB_URL: a database it may create and drop databases from, ' +
'e.g. postgres://postgres:trek@127.0.0.1:55432/postgres.',
);
}
// Left behind by an earlier run that was killed; nothing else uses these names.
await dropLaneDatabases(adminUrl);
await withAdmin(adminUrl, (run) => run(`CREATE DATABASE "${TEMPLATE_DATABASE}"`));
// A child process under tsx, as tests/global-setup.ts does, for the same
// reason: globalSetup runs outside vitest's transform pipeline.
execFileSync(process.execPath, ['--import', 'tsx', path.join(__dirname, 'build-template.ts')], {
cwd: path.join(__dirname, '..', '..'),
stdio: ['ignore', 'inherit', 'inherit'],
env: {
...process.env,
NODE_ENV: 'test',
ENCRYPTION_KEY: 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2',
LOG_LEVEL: 'error',
TREK_DB_URL: databaseUrl(adminUrl, TEMPLATE_DATABASE),
},
});
}
/** The template and every worker's copy: names the lane alone uses (a prefix compared as text, not a LIKE pattern). */
async function dropLaneDatabases(adminUrl: string): Promise<void> {
await withAdmin(adminUrl, async (run) => {
const lane = await run<{ datname: string }>(
'select datname from pg_database where datname = ? or left(datname, length(?)) = ?',
[TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, WORKER_DATABASE_PREFIX],
);
await Promise.all(lane.map((row) => run(`DROP DATABASE IF EXISTS "${row.datname}" WITH (FORCE)`)));
});
}
/** After the run: the lane leaves no database behind. */
export async function teardown(): Promise<void> {
const adminUrl = process.env.TREK_TEST_DB_URL;
if (adminUrl) await dropLaneDatabases(adminUrl);
}
+133
View File
@@ -0,0 +1,133 @@
/**
* PARITY-PG — the schema a Postgres database ends with after its bootstrap
* (the baseline, then every migration after its cut) is the schema the
* entities describe, as scripts/generate-pg-baseline.ts writes it for
* Postgres (src/db/engine/postgres/schema-adjust.ts). MikroORM's comparator
* reads the live database and must find nothing to change: a migration after
* the cut that left Postgres behind SQLite, or a baseline that drifted from
* the entities, fails here. It runs on the lane's database, never skipped.
*
* The comparator also reports statements that change nothing, and each kind
* is proven so rather than recognised by its text:
* - `drop not null` on a primary key column: SQLite lets a key column be null
* and the entities say so; Postgres keeps every primary key NOT NULL (and
* refuses the statement), checked against the live keys.
* - a column default or a check constraint Postgres stores in its own spelling
* (casts added, EXTRACT rewritten, BETWEEN as two comparisons): such
* statements are applied in a transaction that is rolled back, and the
* catalogue's own rendering of every default and check must read the same
* after as before. A default or a bound that really differs changes it.
* - dropping a trigger the entities cannot carry: only the triggers ported by
* hand (POSTGRES_TRIGGERS), by exact name, and each of them must exist.
*/
import { POSTGRES_TRIGGERS } from '../../src/db/engine/postgres/postgres-triggers';
import { applyPostgresColumnAdjustments } from '../../src/db/engine/postgres/schema-adjust';
import { ALL_ENTITIES } from '../../src/db/entities';
import type { Transaction } from '@mikro-orm/core';
import { MikroORM } from '@mikro-orm/postgresql';
import { describe, expect, it } from 'vitest';
const PORTED = Object.keys(POSTGRES_TRIGGERS);
const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/;
const SPELLING =
/^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s;
/** MikroORM drops a trigger by its name, and the function it would have made for it as `<table>_<trigger>_fn`. */
const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/;
const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/;
/** Every column default and check constraint of the schema, as the catalogue renders them. */
const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from (
select 'default ' || c.relname || '.' || a.attname || ' = ' || pg_get_expr(d.adbin, d.adrelid) as item
from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum
where c.relnamespace = current_schema()::regnamespace
union all
select 'check ' || conrelid::regclass || '.' || conname || ' = ' || pg_get_constraintdef(oid)
from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace
) items`;
const ROLLBACK = new Error('roll back the spelling check');
async function liveSets(orm: MikroORM): Promise<{ primaryKeys: Set<string>; triggers: Set<string> }> {
const connection = orm.em.getConnection();
const names = (sql: string) =>
connection.execute<Array<{ name: string }>>(sql).then((rows) => new Set(rows.map((row) => row.name)));
const [primaryKeys, triggers] = await Promise.all([
names(
`select c.relname || '.' || a.attname as name from pg_index i
join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey)
where i.indisprimary and c.relnamespace = current_schema()::regnamespace`,
),
names(
`select c.relname || '.' || t.tgname as name from pg_trigger t join pg_class c on c.oid = t.tgrelid
where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`,
),
]);
return { primaryKeys, triggers };
}
/** Whether applying `statements` leaves every default and check constraint reading as it did; nothing is kept. */
async function changesNothing(
orm: MikroORM,
statements: readonly string[],
): Promise<{ before: string; after: string }> {
const connection = orm.em.getConnection();
const fingerprint = async (trx?: Transaction) =>
(await connection.execute<Array<{ fingerprint: string }>>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint;
const before = await fingerprint();
let after = '';
await connection
.transactional(async (trx) => {
// One after the other: a drop and the add that follows name the same constraint.
await statements.reduce<Promise<unknown>>(
(previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)),
Promise.resolve(),
);
after = await fingerprint(trx);
throw ROLLBACK;
})
.catch((err: unknown) => {
if (err !== ROLLBACK) throw err;
});
return { before, after };
}
describe('PARITY-PG', () => {
it('PARITY-PG-001: the bootstrapped database holds exactly the schema the entities generate for Postgres', async () => {
const orm = await MikroORM.init({
clientUrl: process.env.TREK_DB_URL,
entities: [...ALL_ENTITIES],
discovery: { warnWhenNoEntities: false },
});
try {
for (const meta of orm.getMetadata().getAll().values()) {
applyPostgresColumnAdjustments(meta.props);
}
const live = await liveSets(orm);
const statements = (await orm.schema.getUpdateSchemaSQL({ wrap: false }))
.split(';\n')
.map((statement) => statement.trim().replace(/;$/, ''))
.filter((statement) => statement !== '');
const isKeyNotNull = (s: string) => {
const m = DROP_NOT_NULL.exec(s);
return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`);
};
const isPortedTrigger = (s: string) => {
const trigger = DROP_TRIGGER.exec(s);
if (trigger) return PORTED.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`);
const fn = DROP_FUNCTION.exec(s);
return fn !== null && PORTED.some((name) => fn[1].endsWith(`_${name}_fn`));
};
const spelling = statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && SPELLING.test(s));
expect(statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && !SPELLING.test(s))).toEqual([]);
const { before, after } = await changesNothing(orm, spelling);
expect(after).toBe(before);
// Every hand-ported trigger is in the database, on its table.
expect(PORTED.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`)))).toEqual([]);
} finally {
await orm.close(true);
}
}, 60_000);
});
+66
View File
@@ -0,0 +1,66 @@
/**
* PGSEM-001..002 — what SQLite gives for free and Postgres only through the
* engine, against the real app on the lane's Postgres database:
* writes in a UnitOfWork run one at a time, as on SQLite's single writer, so
* a read-then-insert never races; counts come back as numbers; and a search
* ignores case as SQLite's LIKE does.
*/
import { buildApp } from '../../src/bootstrap';
import { ilike } from '../../src/db/dialect/sql-functions';
import { withRequestContext } from '../../src/nest/database/request-context';
import { UnitOfWork } from '../../src/nest/database/unit-of-work';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
let app: INestApplication;
let orm: MikroORM;
beforeAll(async () => {
app = await buildApp();
orm = app.get(MikroORM);
}, 60_000);
afterAll(async () => {
await app?.close();
});
/** Rows of `sql` read through the current (transactional) EntityManager. */
function query<T>(sql: string, params: unknown[] = []): Promise<T[]> {
return orm.em.getConnection().execute<T[]>(sql, params, 'all', orm.em.getTransactionContext());
}
describe('Postgres keeps SQLite semantics', () => {
it('PGSEM-001: concurrent read-then-insert units of work never see the same count, and counts are numbers', async () => {
const uow = app.get(UnitOfWork, { strict: false });
const claim = () =>
withRequestContext(orm, () =>
uow.transactional(async () => {
const [{ n }] = await query<{ n: number }>(
"select count(*) as n from categories where name like 'pg-lock-%'",
);
// Long enough for every other unit of work to have started and read too, were nothing holding them.
await new Promise((resolve) => setTimeout(resolve, 50));
await query('insert into categories (name, color, icon) values (?, ?, ?)', [`pg-lock-${n}`, '#000000', 'x']);
return n;
}),
);
const claimed = await Promise.all([claim(), claim(), claim(), claim()]);
expect(claimed.every((n) => typeof n === 'number')).toBe(true);
expect([...claimed].sort((a, b) => a - b)).toEqual([0, 1, 2, 3]);
});
it('PGSEM-002: a search ignores case, as LIKE does on SQLite', async () => {
const rows = await withRequestContext(orm, () =>
query<{ name: string }>(`select name from categories where ${ilike(orm.em.getPlatform(), ['name'])}`, [
'%HOTEL%',
]),
);
const names = rows.map((row) => row.name);
expect(names).toContain('Hotel');
expect(names.every((name) => /hotel/i.test(name))).toBe(true);
});
});
+44
View File
@@ -0,0 +1,44 @@
// The Postgres lane's per-file setup (vitest.pg.config.ts), after tests/setup.ts:
// every test file runs on a fresh copy of the migrated template, under a name of
// its worker's own, so files never see each other's rows and workers never share
// a database. TREK_DB_URL is set here, before the test file imports anything
// that reads it (src/mikro-orm.config.ts reads it on import); the copy itself is
// made in a beforeAll, which runs before the test file's own (where the app
// connects).
import { databaseUrl, TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, withAdmin } from './databases';
import { beforeAll } from 'vitest';
/** Copies of the template are made one at a time: Postgres refuses to copy a template another copy is reading. */
const COPY_LOCK = 0x54524b54;
const COPY_ATTEMPTS = 3;
const COPY_RETRY_MS = 500;
const COPY_TIMEOUT_MS = 60_000;
const adminUrl = process.env.TREK_TEST_DB_URL as string;
const poolId = process.env.VITEST_POOL_ID ?? '0';
if (!/^\d+$/.test(poolId)) throw new Error(`unexpected VITEST_POOL_ID ${poolId}`);
const database = `${WORKER_DATABASE_PREFIX}${poolId}`;
process.env.TREK_DB_URL = databaseUrl(adminUrl, database);
async function copyTemplate(attempt = 1): Promise<void> {
try {
await withAdmin(adminUrl, async (run) => {
// One connection (withAdmin's pool holds one), so the session lock covers both statements.
await run('select pg_advisory_lock(?)', [COPY_LOCK]);
try {
await run(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`);
await run(`CREATE DATABASE "${database}" TEMPLATE "${TEMPLATE_DATABASE}"`);
} finally {
await run('select pg_advisory_unlock(?)', [COPY_LOCK]);
}
});
} catch (err) {
if (attempt >= COPY_ATTEMPTS) throw err;
await new Promise((resolve) => setTimeout(resolve, COPY_RETRY_MS * attempt));
await copyTemplate(attempt + 1);
}
}
beforeAll(() => copyTemplate(), COPY_TIMEOUT_MS);