fix(db): sort NULLs where SQLite sorts them on Postgres

SQLite takes NULL as the smallest value and Postgres as the largest, so any
list ordered by a nullable column (bookings without a time, trips without
dates) came back in another order on Postgres. Instead of every ORDER BY
spelling its NULL position, the Postgres engine renders SQLite's wherever a
statement names none: a Kysely query compiler for this.kysely() statements
and a MikroORM platform for find() and the query builder. An explicit
position is kept.

A new Postgres-lane suite checks the answers the probe cannot see: the NULL
order, a stay linked as '14.0' in the public reads, the traveller owner
flag, the identity errors for a lost race, and the plugin caps under a race
on the pool.
This commit is contained in:
Maurice
2026-11-17 12:12:55 +01:00
parent 60642cc18b
commit c3f6177be1
4 changed files with 288 additions and 5 deletions
@@ -0,0 +1,68 @@
import { PostgreSqlPlatform } from '@mikro-orm/postgresql';
import {
PostgresQueryCompiler,
RawNode,
type DatabaseIntrospector,
type Dialect,
type Kysely,
type OperationNode,
type OrderByItemNode,
} from 'kysely';
/**
* Where NULLs sort, as SQLite sorts them, for every ORDER BY TREK sends to
* Postgres.
*
* SQLite takes NULL as the smallest value: first in ascending order, last in
* descending order. Postgres takes it as the largest, the other way round, so
* a list ordered by a nullable column (bookings without a time, trips without
* dates, items without a position) would come back in another order on each
* engine. Rather than every ORDER BY spelling its NULL position, which a new
* one would forget, the Postgres engine renders the position SQLite implies
* wherever a statement does not name one: Kysely statements through the query
* compiler below, MikroORM's own (`find`, the query builder) through the
* platform's ORDER BY rendering. An explicit `nulls first`/`nulls last` is
* left as written. ORDER BY spelled inside raw SQL text is outside both.
*/
/** The NULL position SQLite gives an ORDER BY item of this direction. */
function sqliteNulls(descending: boolean): 'first' | 'last' {
return descending ? 'last' : 'first';
}
/** Whether a Kysely ORDER BY item's direction is descending (`desc`); no direction is ascending. */
function isDescending(direction: OperationNode | undefined): boolean {
if (!direction || !RawNode.is(direction)) return false;
return direction.sqlFragments.join('').trim().toLowerCase() === 'desc';
}
/** Postgres's query compiler, with SQLite's NULL position on every ORDER BY item that names none. */
export class SqliteNullOrderCompiler extends PostgresQueryCompiler {
protected override visitOrderByItem(node: OrderByItemNode): void {
super.visitOrderByItem(node.nulls ? node : { ...node, nulls: sqliteNulls(isDescending(node.direction)) });
}
}
/** `dialect` with {@link SqliteNullOrderCompiler} as its query compiler; the rest is its own. */
export function withSqliteNullOrder(dialect: Dialect): Dialect {
return {
createAdapter: () => dialect.createAdapter(),
createDriver: () => dialect.createDriver(),
createIntrospector: (db: Kysely<unknown>): DatabaseIntrospector => dialect.createIntrospector(db),
createQueryCompiler: () => new SqliteNullOrderCompiler(),
};
}
/** One rendered MikroORM ORDER BY item (`"t"."start_date" desc`), with SQLite's NULL position when it names none. */
export function withSqliteNullPosition(item: string): string {
if (/\snulls\s+(first|last)\s*$/i.test(item)) return item;
return `${item} nulls ${sqliteNulls(/\sdesc\s*$/i.test(item))}`;
}
/** MikroORM's Postgres platform, rendering ORDER BY items with SQLite's NULL position. */
export class SqliteNullOrderPlatform extends PostgreSqlPlatform {
override getOrderByExpression(column: string, direction: string, collation?: string): string[] {
return super.getOrderByExpression(column, direction, collation).map(withSqliteNullPosition);
}
}
@@ -1,3 +1,4 @@
import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order';
import { type Configuration } from '@mikro-orm/core';
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
@@ -49,19 +50,29 @@ class NumberTypesConnection extends PostgreSqlConnection {
types.setTypeParser(NUMERIC_OID, parseNumeric);
return options;
}
/** The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL position (`null-order.ts`). */
override createKyselyDialect(overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0]) {
return withSqliteNullOrder(super.createKyselyDialect(overrides)) as ReturnType<
PostgreSqlConnection['createKyselyDialect']
>;
}
}
/**
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
* as numbers. The parsers belong to this pool only: nothing global in `pg`
* changes.
* as numbers, and on a platform and a query compiler that sort NULLs where
* SQLite does (`null-order.ts`). The parsers belong to this pool only:
* nothing global in `pg` changes.
*/
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
constructor(config: Configuration) {
super(config);
// PostgreSqlDriver's constructor hardcodes its connection class, as
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
// field, which is not public, so it is replaced here, before anything connects.
// PostgreSqlDriver's constructor hardcodes its connection and platform
// classes, as SqliteDriver's does (`db/orm-driver.ts`). It keeps both in
// fields, which are not public, so they are replaced here, before anything
// connects and before the configuration reads the platform.
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
(this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform();
}
}
@@ -0,0 +1,129 @@
/**
* PGSEM-101..105 — answers that differ by engine although every statement is
* accepted, so the probe cannot see them; checked here against the real app
* on the lane's Postgres database:
* a booking linked to a stay as '14.0' still shows the stay in the public
* reads; the traveller owner flag comes back as a number; a lost race for an
* email or a username is the identity error, not a raw driver error; NULLs
* sort where SQLite sorts them; and a plugin's capped writes, run in a unit of
* work, never pass their cap when they race on the pool.
*/
import { buildApp } from '../../src/bootstrap';
import { PluginEntityMetadata } from '../../src/db/entities/PluginEntityMetadata.entity';
import { Reservations } from '../../src/db/entities/Reservations.entity';
import { Trips } from '../../src/db/entities/Trips.entity';
import { Users } from '../../src/db/entities/Users.entity';
import type { PluginEntityMetadataRepository } from '../../src/db/repositories/PluginEntityMetadata.repository';
import type { ReservationsRepository } from '../../src/db/repositories/Reservations.repository';
import type { TripsRepository } from '../../src/db/repositories/Trips.repository';
import { UserIdentityTakenError, type UsersRepository } from '../../src/db/repositories/Users.repository';
import { withRequestContext } from '../../src/nest/database/request-context';
import { UnitOfWork } from '../../src/nest/database/unit-of-work';
import { makeDayAccommodation } from '../helpers/factories/itinerary';
import { makePlace } from '../helpers/factories/places';
import { makeReservation } from '../helpers/factories/reservations';
import { addTripMember, makeTrip, readTripDays } from '../helpers/factories/trips';
import { makeUser } from '../helpers/factories/users';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
let app: INestApplication;
let orm: MikroORM;
beforeAll(async () => {
app = await buildApp();
orm = app.get(MikroORM);
}, 60_000);
afterAll(async () => {
await app?.close();
});
/** `fn` on a fresh request context, as a request runs it. */
function inRequest<T>(fn: () => Promise<T>): Promise<T> {
return withRequestContext(orm, fn);
}
const trips = () => orm.em.getRepository(Trips) as unknown as TripsRepository;
const reservations = () => orm.em.getRepository(Reservations) as unknown as ReservationsRepository;
const users = () => orm.em.getRepository(Users) as unknown as UsersRepository;
describe('Postgres answers as SQLite does', () => {
it("PGSEM-101: a stay a booking points at as '14.0' is public in the calendar and share reads, for its trip and others", async () => {
const { user } = await makeUser(orm);
const linked = await makeTrip(orm, user.id, { start_date: '2026-09-01', end_date: '2026-09-03' });
const other = await makeTrip(orm, user.id, { start_date: '2026-10-01', end_date: '2026-10-03' });
const stays: number[] = [];
for (const trip of [linked, other]) {
const place = await makePlace(orm, trip.id);
const days = await readTripDays(orm, trip.id);
const stay = await makeDayAccommodation(orm, trip.id, place.id, days[0].id, days[1].id);
stays.push(stay.id);
}
await makeReservation(orm, linked.id, { type: 'hotel', title: 'Hotel', accommodation_id: `${stays[0]}.0` });
for (const [trip, stay] of [
[linked.id, stays[0]],
[other.id, stays[1]],
] as const) {
const calendar = await inRequest(() => reservations().listPublicStaysForCalendar(trip));
expect(calendar.map((row) => row.id)).toEqual([stay]);
const shared = await inRequest(() => reservations().listPublicAccommodationsForShare(trip));
expect(shared.map((row) => row.id)).toEqual([stay]);
await expect(inRequest(() => reservations().listPublicStayPlaceIdsForShare(trip))).resolves.toHaveLength(1);
}
});
it('PGSEM-102: the traveller list reports the owner flag as the numbers 1 and 0', async () => {
const { user: owner } = await makeUser(orm, { username: 'pgsem-owner' });
const { user: member } = await makeUser(orm, { username: 'pgsem-member' });
const trip = await makeTrip(orm, owner.id);
await addTripMember(orm, trip.id, member.id, owner.id);
await expect(inRequest(() => trips().listTravellerUsernames(trip.id))).resolves.toEqual([
{ username: 'pgsem-owner', is_owner: 1 },
{ username: 'pgsem-member', is_owner: 0 },
]);
});
it('PGSEM-103: a taken email or username is refused as that field, whichever constraint Postgres names', async () => {
await makeUser(orm, { username: 'pgsem-anna', email: 'pgsem-anna@example.com' });
const row = (username: string, email: string) => ({ username, email, password_hash: 'x', role: 'user' });
const sameEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna2', 'pgsem-anna@example.com')));
await expect(sameEmail).rejects.toMatchObject({ field: 'email' });
const caseEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna3', 'PGSEM-Anna@example.com')));
await expect(caseEmail).rejects.toMatchObject({ field: 'email' });
const sameName = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna', 'pgsem-other@example.com')));
await expect(sameName).rejects.toBeInstanceOf(UserIdentityTakenError);
await expect(sameName).rejects.toMatchObject({ field: 'username' });
});
it('PGSEM-104: NULLs sort first ascending and last descending, as on SQLite', async () => {
const { user } = await makeUser(orm);
const dated = await makeTrip(orm, user.id, { start_date: '2026-05-01', end_date: '2026-05-02' });
const undated = await makeTrip(orm, user.id);
const summaries = await inRequest(() => trips().listSummariesByIds([dated.id, undated.id]));
expect(summaries.map((row) => row.id)).toEqual([dated.id, undated.id]);
const trip = await makeTrip(orm, user.id);
const timed = await makeReservation(orm, trip.id, { reservation_time: '2026-05-01T10:00' });
const untimed = await makeReservation(orm, trip.id, { reservation_time: null });
const listed = await inRequest(() => reservations().listForTrip(trip.id));
expect(listed.map((row) => row.id)).toEqual([untimed.id, timed.id]);
});
it('PGSEM-105: capped plugin writes racing on the pool, each in a unit of work, never pass the cap', async () => {
const uow = app.get(UnitOfWork, { strict: false });
const meta = () => orm.em.getRepository(PluginEntityMetadata) as unknown as PluginEntityMetadataRepository;
const cap = 5;
const written = await Promise.all(
Array.from({ length: 12 }, (_, i) =>
inRequest(() => uow.transactional(() => meta().upsertValueCapped('pgsem', 'trip', 1, `k${i}`, '1', cap))),
),
);
expect(written.filter(Boolean)).toHaveLength(cap);
const stored = await inRequest(() => meta().countForEntity('pgsem', 'trip', 1));
expect(stored).toBe(cap);
});
});
@@ -0,0 +1,75 @@
/**
* SQLite's NULL position on Postgres (src/db/engine/postgres/null-order.ts):
* the compiled ORDER BY of a Kysely statement and MikroORM's rendered ORDER BY
* items. Nothing here reaches a server; the Postgres lane runs the orders
* against a real one (PGSEM-004).
*/
import {
SqliteNullOrderCompiler,
SqliteNullOrderPlatform,
withSqliteNullOrder,
withSqliteNullPosition,
} from '../../../../../src/db/engine/postgres/null-order';
import { DummyDriver, Kysely, PostgresAdapter, PostgresDialect, PostgresIntrospector, sql } from 'kysely';
import { describe, expect, it } from 'vitest';
interface ProbeDB {
trips: { id: number; start_date: string | null; title: string };
}
const db = new Kysely<ProbeDB>({
dialect: withSqliteNullOrder({
createAdapter: () => new PostgresAdapter(),
createDriver: () => new DummyDriver(),
createIntrospector: (kysely) => new PostgresIntrospector(kysely),
createQueryCompiler: () => {
throw new Error('the wrapped dialect brings its own compiler');
},
}),
});
describe('SQLite NULL order on Postgres', () => {
it('NULLORDER-001: an ascending item sorts NULLs first and a descending one last, as SQLite does', () => {
const compiled = db
.selectFrom('trips')
.select('id')
.orderBy('start_date', 'desc')
.orderBy('title')
.orderBy('id', 'asc')
.compile();
expect(compiled.sql).toBe(
'select "id" from "trips" order by "start_date" desc nulls last, "title" nulls first, "id" asc nulls first',
);
});
it('NULLORDER-002: an explicit NULL position and an expression item are kept as written', () => {
const compiled = db
.selectFrom('trips')
.select('id')
.orderBy('start_date', (ob) => ob.asc().nullsLast())
.orderBy((eb) => eb('start_date', 'is', null), 'desc')
.orderBy(sql`lower(title)`, (ob) => ob.desc())
.compile();
expect(compiled.sql).toBe(
'select "id" from "trips" order by "start_date" asc nulls last, "start_date" is null desc nulls last, lower(title) desc nulls last',
);
});
it('NULLORDER-003: the wrapped dialect keeps its adapter, driver and introspector, and compiles with the null-order compiler', () => {
const inner = new PostgresDialect({ pool: {} as never });
const wrapped = withSqliteNullOrder(inner);
expect(wrapped.createAdapter()).toBeInstanceOf(PostgresAdapter);
expect(wrapped.createQueryCompiler()).toBeInstanceOf(SqliteNullOrderCompiler);
});
it("NULLORDER-004: MikroORM's ORDER BY items get SQLite's NULL position unless they name one", () => {
expect(withSqliteNullPosition('"t"."start_date" asc')).toBe('"t"."start_date" asc nulls first');
expect(withSqliteNullPosition('"t"."start_date" desc')).toBe('"t"."start_date" desc nulls last');
expect(withSqliteNullPosition('"t"."start_date" desc nulls first')).toBe('"t"."start_date" desc nulls first');
const platform = new SqliteNullOrderPlatform();
expect(platform.getOrderByExpression('"name"', 'ASC')).toEqual(['"name" asc nulls first']);
expect(platform.getOrderByExpression('"name"', 'desc', 'nocase')).toEqual(['"name" collate "nocase" desc nulls last']);
expect(platform.getOrderByExpression('"name"', 'asc nulls last')).toEqual(['"name" asc nulls last']);
});
});