mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
feat(db): run TREK on PostgreSQL
TREK runs on PostgreSQL when TREK_DB_URL is set, and on SQLite as before otherwise: an engine port under src/db/engine, the dialect helpers under src/db/dialect, a Postgres baseline for the migrations up to the cut, pg_dump/pg_restore backups, a probe of every repository method against a real server and a Postgres test lane in CI.
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
import { buildOrmConfig, resolveEngine } from '../../src/db/engine';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
|
||||
/**
|
||||
* Brings the template database TREK_DB_URL names to the current schema, the
|
||||
* way a first boot on an empty Postgres database does: the engine's own
|
||||
* bootstrap (baseline, the migrations after its cut, seeders), with no
|
||||
* pre-migrate copy, since the database is new.
|
||||
*/
|
||||
async function main(): Promise<void> {
|
||||
const engine = resolveEngine();
|
||||
if (engine.id !== 'postgres') throw new Error('build-template.ts runs on the Postgres engine only (TREK_DB_URL).');
|
||||
const config = buildOrmConfig(engine);
|
||||
const orm = await MikroORM.init({
|
||||
...config,
|
||||
// As src/db/standalone-orm.ts: the bootstrap runs SQL and seeders, which
|
||||
// need no entity metadata.
|
||||
entities: [],
|
||||
entitiesTs: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
migrations: { ...config.migrations, silent: true },
|
||||
});
|
||||
try {
|
||||
await engine.schemaBootstrap(orm, { snapshot: false });
|
||||
process.stdout.write(`[test-db] Postgres template ready: ${engine.describe()}\n`);
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
process.stderr.write(
|
||||
`[test-db] failed to build the Postgres template: ${err instanceof Error ? err.stack : String(err)}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import { MikroORM } from '@mikro-orm/postgresql';
|
||||
|
||||
/**
|
||||
* The databases the Postgres lane creates on the server TREK_TEST_DB_URL names:
|
||||
* one migrated template per run, and one copy of it per test file.
|
||||
*/
|
||||
export const TEMPLATE_DATABASE = 'trek_test_template';
|
||||
export const WORKER_DATABASE_PREFIX = 'trek_test_w';
|
||||
|
||||
/** `adminUrl` pointed at `database`, everything else (credentials, host, sslmode) kept. */
|
||||
export function databaseUrl(adminUrl: string, database: string): string {
|
||||
const url = new URL(adminUrl);
|
||||
url.pathname = `/${database}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
type Run = <T = unknown>(sql: string, params?: unknown[]) => Promise<T[]>;
|
||||
|
||||
/**
|
||||
* Runs `fn` on a connection to the admin database, outside any transaction
|
||||
* (CREATE and DROP DATABASE refuse to run in one), through the same Postgres
|
||||
* driver the app uses.
|
||||
*/
|
||||
export async function withAdmin<T>(adminUrl: string, fn: (run: Run) => Promise<T>): Promise<T> {
|
||||
const orm = await MikroORM.init({
|
||||
clientUrl: adminUrl,
|
||||
entities: [],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
pool: { min: 0, max: 1 },
|
||||
});
|
||||
try {
|
||||
const connection = orm.em.getConnection();
|
||||
return await fn((sql, params = []) => connection.execute(sql, params, 'all'));
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
import { databaseUrl, TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, withAdmin } from './databases';
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* Runs once per `npm run test:pg`, before any worker starts: the Postgres
|
||||
* counterpart of tests/global-setup.ts's schema snapshot. One template
|
||||
* database is created and brought to the current schema the way a first boot
|
||||
* does (baseline, migrations after its cut, seeders); every test file then
|
||||
* gets a copy of it (tests/postgres/setup.ts), which Postgres makes at the
|
||||
* file level, far faster than migrating per file.
|
||||
*
|
||||
* TREK_TEST_DB_URL is required: a lane that skipped without it would pass
|
||||
* having tested nothing.
|
||||
*/
|
||||
export async function setup(): Promise<void> {
|
||||
const adminUrl = process.env.TREK_TEST_DB_URL;
|
||||
if (!adminUrl) {
|
||||
throw new Error(
|
||||
'The Postgres lane needs TREK_TEST_DB_URL: a database it may create and drop databases from, ' +
|
||||
'e.g. postgres://postgres:trek@127.0.0.1:55432/postgres.',
|
||||
);
|
||||
}
|
||||
|
||||
// Left behind by an earlier run that was killed; nothing else uses these names.
|
||||
await dropLaneDatabases(adminUrl);
|
||||
await withAdmin(adminUrl, (run) => run(`CREATE DATABASE "${TEMPLATE_DATABASE}"`));
|
||||
|
||||
// A child process under tsx, as tests/global-setup.ts does, for the same
|
||||
// reason: globalSetup runs outside vitest's transform pipeline.
|
||||
execFileSync(process.execPath, ['--import', 'tsx', path.join(__dirname, 'build-template.ts')], {
|
||||
cwd: path.join(__dirname, '..', '..'),
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_ENV: 'test',
|
||||
ENCRYPTION_KEY: 'a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2a3b4c5d6a7b8c9d0e1f2',
|
||||
LOG_LEVEL: 'error',
|
||||
TREK_DB_URL: databaseUrl(adminUrl, TEMPLATE_DATABASE),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** The template and every worker's copy: names the lane alone uses (a prefix compared as text, not a LIKE pattern). */
|
||||
async function dropLaneDatabases(adminUrl: string): Promise<void> {
|
||||
await withAdmin(adminUrl, async (run) => {
|
||||
const lane = await run<{ datname: string }>(
|
||||
'select datname from pg_database where datname = ? or left(datname, length(?)) = ?',
|
||||
[TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, WORKER_DATABASE_PREFIX],
|
||||
);
|
||||
await Promise.all(lane.map((row) => run(`DROP DATABASE IF EXISTS "${row.datname}" WITH (FORCE)`)));
|
||||
});
|
||||
}
|
||||
|
||||
/** After the run: the lane leaves no database behind. */
|
||||
export async function teardown(): Promise<void> {
|
||||
const adminUrl = process.env.TREK_TEST_DB_URL;
|
||||
if (adminUrl) await dropLaneDatabases(adminUrl);
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
/**
|
||||
* PARITY-PG — the schema a Postgres database ends with after its bootstrap
|
||||
* (the baseline, then every migration after its cut) is the schema the
|
||||
* entities describe, as scripts/generate-pg-baseline.ts writes it for
|
||||
* Postgres (src/db/engine/postgres/schema-adjust.ts). MikroORM's comparator
|
||||
* reads the live database and must find nothing to change: a migration after
|
||||
* the cut that left Postgres behind SQLite, or a baseline that drifted from
|
||||
* the entities, fails here. It runs on the lane's database, never skipped.
|
||||
*
|
||||
* The comparator also reports statements that change nothing, and each kind
|
||||
* is proven so rather than recognised by its text:
|
||||
* - `drop not null` on a primary key column: SQLite lets a key column be null
|
||||
* and the entities say so; Postgres keeps every primary key NOT NULL (and
|
||||
* refuses the statement), checked against the live keys.
|
||||
* - a column default or a check constraint Postgres stores in its own spelling
|
||||
* (casts added, EXTRACT rewritten, BETWEEN as two comparisons): such
|
||||
* statements are applied in a transaction that is rolled back, and the
|
||||
* catalogue's own rendering of every default and check must read the same
|
||||
* after as before. A default or a bound that really differs changes it.
|
||||
* - dropping a trigger the entities cannot carry: only the triggers ported by
|
||||
* hand (POSTGRES_TRIGGERS), by exact name, and each of them must exist.
|
||||
*/
|
||||
import { POSTGRES_TRIGGERS } from '../../src/db/engine/postgres/postgres-triggers';
|
||||
import { applyPostgresColumnAdjustments } from '../../src/db/engine/postgres/schema-adjust';
|
||||
import { ALL_ENTITIES } from '../../src/db/entities';
|
||||
import type { Transaction } from '@mikro-orm/core';
|
||||
import { MikroORM } from '@mikro-orm/postgresql';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const PORTED = Object.keys(POSTGRES_TRIGGERS);
|
||||
const DROP_NOT_NULL = /^alter table "(\w+)" alter column "(\w+)" drop not null$/;
|
||||
const SPELLING =
|
||||
/^alter table "\w+" (?:alter column "\w+" set default .+|drop constraint "\w+"|add constraint "\w+" check \(.+\))$/s;
|
||||
/** MikroORM drops a trigger by its name, and the function it would have made for it as `<table>_<trigger>_fn`. */
|
||||
const DROP_TRIGGER = /^drop trigger if exists "(\w+)" on "(\w+)"$/;
|
||||
const DROP_FUNCTION = /^drop function if exists "(\w+)"\(\)$/;
|
||||
|
||||
/** Every column default and check constraint of the schema, as the catalogue renders them. */
|
||||
const FINGERPRINT_SQL = `select string_agg(item, E'\\n' order by item) as fingerprint from (
|
||||
select 'default ' || c.relname || '.' || a.attname || ' = ' || pg_get_expr(d.adbin, d.adrelid) as item
|
||||
from pg_attrdef d join pg_class c on c.oid = d.adrelid join pg_attribute a on a.attrelid = d.adrelid and a.attnum = d.adnum
|
||||
where c.relnamespace = current_schema()::regnamespace
|
||||
union all
|
||||
select 'check ' || conrelid::regclass || '.' || conname || ' = ' || pg_get_constraintdef(oid)
|
||||
from pg_constraint where contype = 'c' and connamespace = current_schema()::regnamespace
|
||||
) items`;
|
||||
|
||||
const ROLLBACK = new Error('roll back the spelling check');
|
||||
|
||||
async function liveSets(orm: MikroORM): Promise<{ primaryKeys: Set<string>; triggers: Set<string> }> {
|
||||
const connection = orm.em.getConnection();
|
||||
const names = (sql: string) =>
|
||||
connection.execute<Array<{ name: string }>>(sql).then((rows) => new Set(rows.map((row) => row.name)));
|
||||
const [primaryKeys, triggers] = await Promise.all([
|
||||
names(
|
||||
`select c.relname || '.' || a.attname as name from pg_index i
|
||||
join pg_class c on c.oid = i.indrelid join pg_attribute a on a.attrelid = i.indrelid and a.attnum = any(i.indkey)
|
||||
where i.indisprimary and c.relnamespace = current_schema()::regnamespace`,
|
||||
),
|
||||
names(
|
||||
`select c.relname || '.' || t.tgname as name from pg_trigger t join pg_class c on c.oid = t.tgrelid
|
||||
where not t.tgisinternal and c.relnamespace = current_schema()::regnamespace`,
|
||||
),
|
||||
]);
|
||||
return { primaryKeys, triggers };
|
||||
}
|
||||
|
||||
/** Whether applying `statements` leaves every default and check constraint reading as it did; nothing is kept. */
|
||||
async function changesNothing(
|
||||
orm: MikroORM,
|
||||
statements: readonly string[],
|
||||
): Promise<{ before: string; after: string }> {
|
||||
const connection = orm.em.getConnection();
|
||||
const fingerprint = async (trx?: Transaction) =>
|
||||
(await connection.execute<Array<{ fingerprint: string }>>(FINGERPRINT_SQL, [], 'all', trx))[0].fingerprint;
|
||||
const before = await fingerprint();
|
||||
let after = '';
|
||||
await connection
|
||||
.transactional(async (trx) => {
|
||||
// One after the other: a drop and the add that follows name the same constraint.
|
||||
await statements.reduce<Promise<unknown>>(
|
||||
(previous, statement) => previous.then(() => connection.execute(statement, [], 'run', trx)),
|
||||
Promise.resolve(),
|
||||
);
|
||||
after = await fingerprint(trx);
|
||||
throw ROLLBACK;
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (err !== ROLLBACK) throw err;
|
||||
});
|
||||
return { before, after };
|
||||
}
|
||||
|
||||
describe('PARITY-PG', () => {
|
||||
it('PARITY-PG-001: the bootstrapped database holds exactly the schema the entities generate for Postgres', async () => {
|
||||
const orm = await MikroORM.init({
|
||||
clientUrl: process.env.TREK_DB_URL,
|
||||
entities: [...ALL_ENTITIES],
|
||||
discovery: { warnWhenNoEntities: false },
|
||||
});
|
||||
try {
|
||||
for (const meta of orm.getMetadata().getAll().values()) {
|
||||
applyPostgresColumnAdjustments(meta.props);
|
||||
}
|
||||
const live = await liveSets(orm);
|
||||
const statements = (await orm.schema.getUpdateSchemaSQL({ wrap: false }))
|
||||
.split(';\n')
|
||||
.map((statement) => statement.trim().replace(/;$/, ''))
|
||||
.filter((statement) => statement !== '');
|
||||
|
||||
const isKeyNotNull = (s: string) => {
|
||||
const m = DROP_NOT_NULL.exec(s);
|
||||
return m !== null && live.primaryKeys.has(`${m[1]}.${m[2]}`);
|
||||
};
|
||||
const isPortedTrigger = (s: string) => {
|
||||
const trigger = DROP_TRIGGER.exec(s);
|
||||
if (trigger) return PORTED.includes(trigger[1]) && live.triggers.has(`${trigger[2]}.${trigger[1]}`);
|
||||
const fn = DROP_FUNCTION.exec(s);
|
||||
return fn !== null && PORTED.some((name) => fn[1].endsWith(`_${name}_fn`));
|
||||
};
|
||||
const spelling = statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && SPELLING.test(s));
|
||||
|
||||
expect(statements.filter((s) => !isKeyNotNull(s) && !isPortedTrigger(s) && !SPELLING.test(s))).toEqual([]);
|
||||
const { before, after } = await changesNothing(orm, spelling);
|
||||
expect(after).toBe(before);
|
||||
// Every hand-ported trigger is in the database, on its table.
|
||||
expect(PORTED.filter((name) => ![...live.triggers].some((t) => t.endsWith(`.${name}`)))).toEqual([]);
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* PGSEM-001..002 — what SQLite gives for free and Postgres only through the
|
||||
* engine, against the real app on the lane's Postgres database:
|
||||
* writes in a UnitOfWork run one at a time, as on SQLite's single writer, so
|
||||
* a read-then-insert never races; counts come back as numbers; and a search
|
||||
* ignores case as SQLite's LIKE does.
|
||||
*/
|
||||
import { buildApp } from '../../src/bootstrap';
|
||||
import { ilike } from '../../src/db/dialect/sql-functions';
|
||||
import { withRequestContext } from '../../src/nest/database/request-context';
|
||||
import { UnitOfWork } from '../../src/nest/database/unit-of-work';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import type { INestApplication } from '@nestjs/common';
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
let app: INestApplication;
|
||||
let orm: MikroORM;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
orm = app.get(MikroORM);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
});
|
||||
|
||||
/** Rows of `sql` read through the current (transactional) EntityManager. */
|
||||
function query<T>(sql: string, params: unknown[] = []): Promise<T[]> {
|
||||
return orm.em.getConnection().execute<T[]>(sql, params, 'all', orm.em.getTransactionContext());
|
||||
}
|
||||
|
||||
describe('Postgres keeps SQLite semantics', () => {
|
||||
it('PGSEM-001: concurrent read-then-insert units of work never see the same count, and counts are numbers', async () => {
|
||||
const uow = app.get(UnitOfWork, { strict: false });
|
||||
const claim = () =>
|
||||
withRequestContext(orm, () =>
|
||||
uow.transactional(async () => {
|
||||
const [{ n }] = await query<{ n: number }>(
|
||||
"select count(*) as n from categories where name like 'pg-lock-%'",
|
||||
);
|
||||
// Long enough for every other unit of work to have started and read too, were nothing holding them.
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await query('insert into categories (name, color, icon) values (?, ?, ?)', [`pg-lock-${n}`, '#000000', 'x']);
|
||||
return n;
|
||||
}),
|
||||
);
|
||||
|
||||
const claimed = await Promise.all([claim(), claim(), claim(), claim()]);
|
||||
|
||||
expect(claimed.every((n) => typeof n === 'number')).toBe(true);
|
||||
expect([...claimed].sort((a, b) => a - b)).toEqual([0, 1, 2, 3]);
|
||||
});
|
||||
|
||||
it('PGSEM-002: a search ignores case, as LIKE does on SQLite', async () => {
|
||||
const rows = await withRequestContext(orm, () =>
|
||||
query<{ name: string }>(`select name from categories where ${ilike(orm.em.getPlatform(), ['name'])}`, [
|
||||
'%HOTEL%',
|
||||
]),
|
||||
);
|
||||
const names = rows.map((row) => row.name);
|
||||
expect(names).toContain('Hotel');
|
||||
expect(names.every((name) => /hotel/i.test(name))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
// The Postgres lane's per-file setup (vitest.pg.config.ts), after tests/setup.ts:
|
||||
// every test file runs on a fresh copy of the migrated template, under a name of
|
||||
// its worker's own, so files never see each other's rows and workers never share
|
||||
// a database. TREK_DB_URL is set here, before the test file imports anything
|
||||
// that reads it (src/mikro-orm.config.ts reads it on import); the copy itself is
|
||||
// made in a beforeAll, which runs before the test file's own (where the app
|
||||
// connects).
|
||||
import { databaseUrl, TEMPLATE_DATABASE, WORKER_DATABASE_PREFIX, withAdmin } from './databases';
|
||||
|
||||
import { beforeAll } from 'vitest';
|
||||
|
||||
/** Copies of the template are made one at a time: Postgres refuses to copy a template another copy is reading. */
|
||||
const COPY_LOCK = 0x54524b54;
|
||||
const COPY_ATTEMPTS = 3;
|
||||
const COPY_RETRY_MS = 500;
|
||||
const COPY_TIMEOUT_MS = 60_000;
|
||||
|
||||
const adminUrl = process.env.TREK_TEST_DB_URL as string;
|
||||
const poolId = process.env.VITEST_POOL_ID ?? '0';
|
||||
if (!/^\d+$/.test(poolId)) throw new Error(`unexpected VITEST_POOL_ID ${poolId}`);
|
||||
const database = `${WORKER_DATABASE_PREFIX}${poolId}`;
|
||||
|
||||
process.env.TREK_DB_URL = databaseUrl(adminUrl, database);
|
||||
|
||||
async function copyTemplate(attempt = 1): Promise<void> {
|
||||
try {
|
||||
await withAdmin(adminUrl, async (run) => {
|
||||
// One connection (withAdmin's pool holds one), so the session lock covers both statements.
|
||||
await run('select pg_advisory_lock(?)', [COPY_LOCK]);
|
||||
try {
|
||||
await run(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`);
|
||||
await run(`CREATE DATABASE "${database}" TEMPLATE "${TEMPLATE_DATABASE}"`);
|
||||
} finally {
|
||||
await run('select pg_advisory_unlock(?)', [COPY_LOCK]);
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
if (attempt >= COPY_ATTEMPTS) throw err;
|
||||
await new Promise((resolve) => setTimeout(resolve, COPY_RETRY_MS * attempt));
|
||||
await copyTemplate(attempt + 1);
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(() => copyTemplate(), COPY_TIMEOUT_MS);
|
||||
Reference in New Issue
Block a user