From c3f6177be16a4f87d2aec757d0053dc3e1e42403 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 17 Nov 2026 12:12:55 +0100 Subject: [PATCH] fix(db): sort NULLs where SQLite sorts them on Postgres SQLite takes NULL as the smallest value and Postgres as the largest, so any list ordered by a nullable column (bookings without a time, trips without dates) came back in another order on Postgres. Instead of every ORDER BY spelling its NULL position, the Postgres engine renders SQLite's wherever a statement names none: a Kysely query compiler for this.kysely() statements and a MikroORM platform for find() and the query builder. An explicit position is kept. A new Postgres-lane suite checks the answers the probe cannot see: the NULL order, a stay linked as '14.0' in the public reads, the traveller owner flag, the identity errors for a lost race, and the plugin caps under a race on the pool. --- server/src/db/engine/postgres/null-order.ts | 68 +++++++++ .../src/db/engine/postgres/postgres-driver.ts | 21 ++- .../tests/postgres/result-semantics.test.ts | 129 ++++++++++++++++++ .../db/engine/postgres/null-order.test.ts | 75 ++++++++++ 4 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 server/src/db/engine/postgres/null-order.ts create mode 100644 server/tests/postgres/result-semantics.test.ts create mode 100644 server/tests/unit/db/engine/postgres/null-order.test.ts diff --git a/server/src/db/engine/postgres/null-order.ts b/server/src/db/engine/postgres/null-order.ts new file mode 100644 index 000000000..f1c76957a --- /dev/null +++ b/server/src/db/engine/postgres/null-order.ts @@ -0,0 +1,68 @@ +import { PostgreSqlPlatform } from '@mikro-orm/postgresql'; + +import { + PostgresQueryCompiler, + RawNode, + type DatabaseIntrospector, + type Dialect, + type Kysely, + type OperationNode, + type OrderByItemNode, +} from 'kysely'; + +/** + * Where NULLs sort, as SQLite sorts them, for every ORDER BY TREK sends to + * Postgres. + * + * SQLite takes NULL as the smallest value: first in ascending order, last in + * descending order. Postgres takes it as the largest, the other way round, so + * a list ordered by a nullable column (bookings without a time, trips without + * dates, items without a position) would come back in another order on each + * engine. Rather than every ORDER BY spelling its NULL position, which a new + * one would forget, the Postgres engine renders the position SQLite implies + * wherever a statement does not name one: Kysely statements through the query + * compiler below, MikroORM's own (`find`, the query builder) through the + * platform's ORDER BY rendering. An explicit `nulls first`/`nulls last` is + * left as written. ORDER BY spelled inside raw SQL text is outside both. + */ + +/** The NULL position SQLite gives an ORDER BY item of this direction. */ +function sqliteNulls(descending: boolean): 'first' | 'last' { + return descending ? 'last' : 'first'; +} + +/** Whether a Kysely ORDER BY item's direction is descending (`desc`); no direction is ascending. */ +function isDescending(direction: OperationNode | undefined): boolean { + if (!direction || !RawNode.is(direction)) return false; + return direction.sqlFragments.join('').trim().toLowerCase() === 'desc'; +} + +/** Postgres's query compiler, with SQLite's NULL position on every ORDER BY item that names none. */ +export class SqliteNullOrderCompiler extends PostgresQueryCompiler { + protected override visitOrderByItem(node: OrderByItemNode): void { + super.visitOrderByItem(node.nulls ? node : { ...node, nulls: sqliteNulls(isDescending(node.direction)) }); + } +} + +/** `dialect` with {@link SqliteNullOrderCompiler} as its query compiler; the rest is its own. */ +export function withSqliteNullOrder(dialect: Dialect): Dialect { + return { + createAdapter: () => dialect.createAdapter(), + createDriver: () => dialect.createDriver(), + createIntrospector: (db: Kysely): DatabaseIntrospector => dialect.createIntrospector(db), + createQueryCompiler: () => new SqliteNullOrderCompiler(), + }; +} + +/** One rendered MikroORM ORDER BY item (`"t"."start_date" desc`), with SQLite's NULL position when it names none. */ +export function withSqliteNullPosition(item: string): string { + if (/\snulls\s+(first|last)\s*$/i.test(item)) return item; + return `${item} nulls ${sqliteNulls(/\sdesc\s*$/i.test(item))}`; +} + +/** MikroORM's Postgres platform, rendering ORDER BY items with SQLite's NULL position. */ +export class SqliteNullOrderPlatform extends PostgreSqlPlatform { + override getOrderByExpression(column: string, direction: string, collation?: string): string[] { + return super.getOrderByExpression(column, direction, collation).map(withSqliteNullPosition); + } +} diff --git a/server/src/db/engine/postgres/postgres-driver.ts b/server/src/db/engine/postgres/postgres-driver.ts index 5dde04db2..272c6ae96 100644 --- a/server/src/db/engine/postgres/postgres-driver.ts +++ b/server/src/db/engine/postgres/postgres-driver.ts @@ -1,3 +1,4 @@ +import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order'; import { type Configuration } from '@mikro-orm/core'; import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql'; @@ -49,19 +50,29 @@ class NumberTypesConnection extends PostgreSqlConnection { types.setTypeParser(NUMERIC_OID, parseNumeric); return options; } + + /** The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL position (`null-order.ts`). */ + override createKyselyDialect(overrides: Parameters[0]) { + return withSqliteNullOrder(super.createKyselyDialect(overrides)) as ReturnType< + PostgreSqlConnection['createKyselyDialect'] + >; + } } /** * MikroORM's Postgres driver on a connection that reads `bigint` and `numeric` - * as numbers. The parsers belong to this pool only: nothing global in `pg` - * changes. + * as numbers, and on a platform and a query compiler that sort NULLs where + * SQLite does (`null-order.ts`). The parsers belong to this pool only: + * nothing global in `pg` changes. */ export class TrekPostgreSqlDriver extends PostgreSqlDriver { constructor(config: Configuration) { super(config); - // PostgreSqlDriver's constructor hardcodes its connection class, as - // SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a - // field, which is not public, so it is replaced here, before anything connects. + // PostgreSqlDriver's constructor hardcodes its connection and platform + // classes, as SqliteDriver's does (`db/orm-driver.ts`). It keeps both in + // fields, which are not public, so they are replaced here, before anything + // connects and before the configuration reads the platform. (this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config); + (this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform(); } } diff --git a/server/tests/postgres/result-semantics.test.ts b/server/tests/postgres/result-semantics.test.ts new file mode 100644 index 000000000..a3e92a6b7 --- /dev/null +++ b/server/tests/postgres/result-semantics.test.ts @@ -0,0 +1,129 @@ +/** + * PGSEM-101..105 — answers that differ by engine although every statement is + * accepted, so the probe cannot see them; checked here against the real app + * on the lane's Postgres database: + * a booking linked to a stay as '14.0' still shows the stay in the public + * reads; the traveller owner flag comes back as a number; a lost race for an + * email or a username is the identity error, not a raw driver error; NULLs + * sort where SQLite sorts them; and a plugin's capped writes, run in a unit of + * work, never pass their cap when they race on the pool. + */ +import { buildApp } from '../../src/bootstrap'; +import { PluginEntityMetadata } from '../../src/db/entities/PluginEntityMetadata.entity'; +import { Reservations } from '../../src/db/entities/Reservations.entity'; +import { Trips } from '../../src/db/entities/Trips.entity'; +import { Users } from '../../src/db/entities/Users.entity'; +import type { PluginEntityMetadataRepository } from '../../src/db/repositories/PluginEntityMetadata.repository'; +import type { ReservationsRepository } from '../../src/db/repositories/Reservations.repository'; +import type { TripsRepository } from '../../src/db/repositories/Trips.repository'; +import { UserIdentityTakenError, type UsersRepository } from '../../src/db/repositories/Users.repository'; +import { withRequestContext } from '../../src/nest/database/request-context'; +import { UnitOfWork } from '../../src/nest/database/unit-of-work'; +import { makeDayAccommodation } from '../helpers/factories/itinerary'; +import { makePlace } from '../helpers/factories/places'; +import { makeReservation } from '../helpers/factories/reservations'; +import { addTripMember, makeTrip, readTripDays } from '../helpers/factories/trips'; +import { makeUser } from '../helpers/factories/users'; +import { MikroORM } from '@mikro-orm/core'; +import type { INestApplication } from '@nestjs/common'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +let app: INestApplication; +let orm: MikroORM; + +beforeAll(async () => { + app = await buildApp(); + orm = app.get(MikroORM); +}, 60_000); + +afterAll(async () => { + await app?.close(); +}); + +/** `fn` on a fresh request context, as a request runs it. */ +function inRequest(fn: () => Promise): Promise { + return withRequestContext(orm, fn); +} + +const trips = () => orm.em.getRepository(Trips) as unknown as TripsRepository; +const reservations = () => orm.em.getRepository(Reservations) as unknown as ReservationsRepository; +const users = () => orm.em.getRepository(Users) as unknown as UsersRepository; + +describe('Postgres answers as SQLite does', () => { + it("PGSEM-101: a stay a booking points at as '14.0' is public in the calendar and share reads, for its trip and others", async () => { + const { user } = await makeUser(orm); + const linked = await makeTrip(orm, user.id, { start_date: '2026-09-01', end_date: '2026-09-03' }); + const other = await makeTrip(orm, user.id, { start_date: '2026-10-01', end_date: '2026-10-03' }); + const stays: number[] = []; + for (const trip of [linked, other]) { + const place = await makePlace(orm, trip.id); + const days = await readTripDays(orm, trip.id); + const stay = await makeDayAccommodation(orm, trip.id, place.id, days[0].id, days[1].id); + stays.push(stay.id); + } + await makeReservation(orm, linked.id, { type: 'hotel', title: 'Hotel', accommodation_id: `${stays[0]}.0` }); + + for (const [trip, stay] of [ + [linked.id, stays[0]], + [other.id, stays[1]], + ] as const) { + const calendar = await inRequest(() => reservations().listPublicStaysForCalendar(trip)); + expect(calendar.map((row) => row.id)).toEqual([stay]); + const shared = await inRequest(() => reservations().listPublicAccommodationsForShare(trip)); + expect(shared.map((row) => row.id)).toEqual([stay]); + await expect(inRequest(() => reservations().listPublicStayPlaceIdsForShare(trip))).resolves.toHaveLength(1); + } + }); + + it('PGSEM-102: the traveller list reports the owner flag as the numbers 1 and 0', async () => { + const { user: owner } = await makeUser(orm, { username: 'pgsem-owner' }); + const { user: member } = await makeUser(orm, { username: 'pgsem-member' }); + const trip = await makeTrip(orm, owner.id); + await addTripMember(orm, trip.id, member.id, owner.id); + await expect(inRequest(() => trips().listTravellerUsernames(trip.id))).resolves.toEqual([ + { username: 'pgsem-owner', is_owner: 1 }, + { username: 'pgsem-member', is_owner: 0 }, + ]); + }); + + it('PGSEM-103: a taken email or username is refused as that field, whichever constraint Postgres names', async () => { + await makeUser(orm, { username: 'pgsem-anna', email: 'pgsem-anna@example.com' }); + const row = (username: string, email: string) => ({ username, email, password_hash: 'x', role: 'user' }); + const sameEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna2', 'pgsem-anna@example.com'))); + await expect(sameEmail).rejects.toMatchObject({ field: 'email' }); + const caseEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna3', 'PGSEM-Anna@example.com'))); + await expect(caseEmail).rejects.toMatchObject({ field: 'email' }); + const sameName = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna', 'pgsem-other@example.com'))); + await expect(sameName).rejects.toBeInstanceOf(UserIdentityTakenError); + await expect(sameName).rejects.toMatchObject({ field: 'username' }); + }); + + it('PGSEM-104: NULLs sort first ascending and last descending, as on SQLite', async () => { + const { user } = await makeUser(orm); + const dated = await makeTrip(orm, user.id, { start_date: '2026-05-01', end_date: '2026-05-02' }); + const undated = await makeTrip(orm, user.id); + const summaries = await inRequest(() => trips().listSummariesByIds([dated.id, undated.id])); + expect(summaries.map((row) => row.id)).toEqual([dated.id, undated.id]); + + const trip = await makeTrip(orm, user.id); + const timed = await makeReservation(orm, trip.id, { reservation_time: '2026-05-01T10:00' }); + const untimed = await makeReservation(orm, trip.id, { reservation_time: null }); + const listed = await inRequest(() => reservations().listForTrip(trip.id)); + expect(listed.map((row) => row.id)).toEqual([untimed.id, timed.id]); + }); + + it('PGSEM-105: capped plugin writes racing on the pool, each in a unit of work, never pass the cap', async () => { + const uow = app.get(UnitOfWork, { strict: false }); + const meta = () => orm.em.getRepository(PluginEntityMetadata) as unknown as PluginEntityMetadataRepository; + const cap = 5; + const written = await Promise.all( + Array.from({ length: 12 }, (_, i) => + inRequest(() => uow.transactional(() => meta().upsertValueCapped('pgsem', 'trip', 1, `k${i}`, '1', cap))), + ), + ); + expect(written.filter(Boolean)).toHaveLength(cap); + const stored = await inRequest(() => meta().countForEntity('pgsem', 'trip', 1)); + expect(stored).toBe(cap); + }); +}); diff --git a/server/tests/unit/db/engine/postgres/null-order.test.ts b/server/tests/unit/db/engine/postgres/null-order.test.ts new file mode 100644 index 000000000..c550b5600 --- /dev/null +++ b/server/tests/unit/db/engine/postgres/null-order.test.ts @@ -0,0 +1,75 @@ +/** + * SQLite's NULL position on Postgres (src/db/engine/postgres/null-order.ts): + * the compiled ORDER BY of a Kysely statement and MikroORM's rendered ORDER BY + * items. Nothing here reaches a server; the Postgres lane runs the orders + * against a real one (PGSEM-004). + */ +import { + SqliteNullOrderCompiler, + SqliteNullOrderPlatform, + withSqliteNullOrder, + withSqliteNullPosition, +} from '../../../../../src/db/engine/postgres/null-order'; + +import { DummyDriver, Kysely, PostgresAdapter, PostgresDialect, PostgresIntrospector, sql } from 'kysely'; +import { describe, expect, it } from 'vitest'; + +interface ProbeDB { + trips: { id: number; start_date: string | null; title: string }; +} + +const db = new Kysely({ + dialect: withSqliteNullOrder({ + createAdapter: () => new PostgresAdapter(), + createDriver: () => new DummyDriver(), + createIntrospector: (kysely) => new PostgresIntrospector(kysely), + createQueryCompiler: () => { + throw new Error('the wrapped dialect brings its own compiler'); + }, + }), +}); + +describe('SQLite NULL order on Postgres', () => { + it('NULLORDER-001: an ascending item sorts NULLs first and a descending one last, as SQLite does', () => { + const compiled = db + .selectFrom('trips') + .select('id') + .orderBy('start_date', 'desc') + .orderBy('title') + .orderBy('id', 'asc') + .compile(); + expect(compiled.sql).toBe( + 'select "id" from "trips" order by "start_date" desc nulls last, "title" nulls first, "id" asc nulls first', + ); + }); + + it('NULLORDER-002: an explicit NULL position and an expression item are kept as written', () => { + const compiled = db + .selectFrom('trips') + .select('id') + .orderBy('start_date', (ob) => ob.asc().nullsLast()) + .orderBy((eb) => eb('start_date', 'is', null), 'desc') + .orderBy(sql`lower(title)`, (ob) => ob.desc()) + .compile(); + expect(compiled.sql).toBe( + 'select "id" from "trips" order by "start_date" asc nulls last, "start_date" is null desc nulls last, lower(title) desc nulls last', + ); + }); + + it('NULLORDER-003: the wrapped dialect keeps its adapter, driver and introspector, and compiles with the null-order compiler', () => { + const inner = new PostgresDialect({ pool: {} as never }); + const wrapped = withSqliteNullOrder(inner); + expect(wrapped.createAdapter()).toBeInstanceOf(PostgresAdapter); + expect(wrapped.createQueryCompiler()).toBeInstanceOf(SqliteNullOrderCompiler); + }); + + it("NULLORDER-004: MikroORM's ORDER BY items get SQLite's NULL position unless they name one", () => { + expect(withSqliteNullPosition('"t"."start_date" asc')).toBe('"t"."start_date" asc nulls first'); + expect(withSqliteNullPosition('"t"."start_date" desc')).toBe('"t"."start_date" desc nulls last'); + expect(withSqliteNullPosition('"t"."start_date" desc nulls first')).toBe('"t"."start_date" desc nulls first'); + const platform = new SqliteNullOrderPlatform(); + expect(platform.getOrderByExpression('"name"', 'ASC')).toEqual(['"name" asc nulls first']); + expect(platform.getOrderByExpression('"name"', 'desc', 'nocase')).toEqual(['"name" collate "nocase" desc nulls last']); + expect(platform.getOrderByExpression('"name"', 'asc nulls last')).toEqual(['"name" asc nulls last']); + }); +});