mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 02:56:10 +00:00
fix(db): sort NULLs where SQLite sorts them on Postgres
SQLite takes NULL as the smallest value and Postgres as the largest, so any list ordered by a nullable column (bookings without a time, trips without dates) came back in another order on Postgres. Instead of every ORDER BY spelling its NULL position, the Postgres engine renders SQLite's wherever a statement names none: a Kysely query compiler for this.kysely() statements and a MikroORM platform for find() and the query builder. An explicit position is kept. A new Postgres-lane suite checks the answers the probe cannot see: the NULL order, a stay linked as '14.0' in the public reads, the traveller owner flag, the identity errors for a lost race, and the plugin caps under a race on the pool.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
import { PostgreSqlPlatform } from '@mikro-orm/postgresql';
|
||||
|
||||
import {
|
||||
PostgresQueryCompiler,
|
||||
RawNode,
|
||||
type DatabaseIntrospector,
|
||||
type Dialect,
|
||||
type Kysely,
|
||||
type OperationNode,
|
||||
type OrderByItemNode,
|
||||
} from 'kysely';
|
||||
|
||||
/**
|
||||
* Where NULLs sort, as SQLite sorts them, for every ORDER BY TREK sends to
|
||||
* Postgres.
|
||||
*
|
||||
* SQLite takes NULL as the smallest value: first in ascending order, last in
|
||||
* descending order. Postgres takes it as the largest, the other way round, so
|
||||
* a list ordered by a nullable column (bookings without a time, trips without
|
||||
* dates, items without a position) would come back in another order on each
|
||||
* engine. Rather than every ORDER BY spelling its NULL position, which a new
|
||||
* one would forget, the Postgres engine renders the position SQLite implies
|
||||
* wherever a statement does not name one: Kysely statements through the query
|
||||
* compiler below, MikroORM's own (`find`, the query builder) through the
|
||||
* platform's ORDER BY rendering. An explicit `nulls first`/`nulls last` is
|
||||
* left as written. ORDER BY spelled inside raw SQL text is outside both.
|
||||
*/
|
||||
|
||||
/** The NULL position SQLite gives an ORDER BY item of this direction. */
|
||||
function sqliteNulls(descending: boolean): 'first' | 'last' {
|
||||
return descending ? 'last' : 'first';
|
||||
}
|
||||
|
||||
/** Whether a Kysely ORDER BY item's direction is descending (`desc`); no direction is ascending. */
|
||||
function isDescending(direction: OperationNode | undefined): boolean {
|
||||
if (!direction || !RawNode.is(direction)) return false;
|
||||
return direction.sqlFragments.join('').trim().toLowerCase() === 'desc';
|
||||
}
|
||||
|
||||
/** Postgres's query compiler, with SQLite's NULL position on every ORDER BY item that names none. */
|
||||
export class SqliteNullOrderCompiler extends PostgresQueryCompiler {
|
||||
protected override visitOrderByItem(node: OrderByItemNode): void {
|
||||
super.visitOrderByItem(node.nulls ? node : { ...node, nulls: sqliteNulls(isDescending(node.direction)) });
|
||||
}
|
||||
}
|
||||
|
||||
/** `dialect` with {@link SqliteNullOrderCompiler} as its query compiler; the rest is its own. */
|
||||
export function withSqliteNullOrder(dialect: Dialect): Dialect {
|
||||
return {
|
||||
createAdapter: () => dialect.createAdapter(),
|
||||
createDriver: () => dialect.createDriver(),
|
||||
createIntrospector: (db: Kysely<unknown>): DatabaseIntrospector => dialect.createIntrospector(db),
|
||||
createQueryCompiler: () => new SqliteNullOrderCompiler(),
|
||||
};
|
||||
}
|
||||
|
||||
/** One rendered MikroORM ORDER BY item (`"t"."start_date" desc`), with SQLite's NULL position when it names none. */
|
||||
export function withSqliteNullPosition(item: string): string {
|
||||
if (/\snulls\s+(first|last)\s*$/i.test(item)) return item;
|
||||
return `${item} nulls ${sqliteNulls(/\sdesc\s*$/i.test(item))}`;
|
||||
}
|
||||
|
||||
/** MikroORM's Postgres platform, rendering ORDER BY items with SQLite's NULL position. */
|
||||
export class SqliteNullOrderPlatform extends PostgreSqlPlatform {
|
||||
override getOrderByExpression(column: string, direction: string, collation?: string): string[] {
|
||||
return super.getOrderByExpression(column, direction, collation).map(withSqliteNullPosition);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { SqliteNullOrderPlatform, withSqliteNullOrder } from './null-order';
|
||||
import { type Configuration } from '@mikro-orm/core';
|
||||
import { PostgreSqlConnection, PostgreSqlDriver } from '@mikro-orm/postgresql';
|
||||
|
||||
@@ -49,19 +50,29 @@ class NumberTypesConnection extends PostgreSqlConnection {
|
||||
types.setTypeParser(NUMERIC_OID, parseNumeric);
|
||||
return options;
|
||||
}
|
||||
|
||||
/** The pool's Kysely dialect, compiling every ORDER BY with SQLite's NULL position (`null-order.ts`). */
|
||||
override createKyselyDialect(overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0]) {
|
||||
return withSqliteNullOrder(super.createKyselyDialect(overrides)) as ReturnType<
|
||||
PostgreSqlConnection['createKyselyDialect']
|
||||
>;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* MikroORM's Postgres driver on a connection that reads `bigint` and `numeric`
|
||||
* as numbers. The parsers belong to this pool only: nothing global in `pg`
|
||||
* changes.
|
||||
* as numbers, and on a platform and a query compiler that sort NULLs where
|
||||
* SQLite does (`null-order.ts`). The parsers belong to this pool only:
|
||||
* nothing global in `pg` changes.
|
||||
*/
|
||||
export class TrekPostgreSqlDriver extends PostgreSqlDriver {
|
||||
constructor(config: Configuration) {
|
||||
super(config);
|
||||
// PostgreSqlDriver's constructor hardcodes its connection class, as
|
||||
// SqliteDriver's does (`db/orm-driver.ts`). It keeps the connection in a
|
||||
// field, which is not public, so it is replaced here, before anything connects.
|
||||
// PostgreSqlDriver's constructor hardcodes its connection and platform
|
||||
// classes, as SqliteDriver's does (`db/orm-driver.ts`). It keeps both in
|
||||
// fields, which are not public, so they are replaced here, before anything
|
||||
// connects and before the configuration reads the platform.
|
||||
(this as unknown as { connection: PostgreSqlConnection }).connection = new NumberTypesConnection(config);
|
||||
(this as unknown as { platform: SqliteNullOrderPlatform }).platform = new SqliteNullOrderPlatform();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* PGSEM-101..105 — answers that differ by engine although every statement is
|
||||
* accepted, so the probe cannot see them; checked here against the real app
|
||||
* on the lane's Postgres database:
|
||||
* a booking linked to a stay as '14.0' still shows the stay in the public
|
||||
* reads; the traveller owner flag comes back as a number; a lost race for an
|
||||
* email or a username is the identity error, not a raw driver error; NULLs
|
||||
* sort where SQLite sorts them; and a plugin's capped writes, run in a unit of
|
||||
* work, never pass their cap when they race on the pool.
|
||||
*/
|
||||
import { buildApp } from '../../src/bootstrap';
|
||||
import { PluginEntityMetadata } from '../../src/db/entities/PluginEntityMetadata.entity';
|
||||
import { Reservations } from '../../src/db/entities/Reservations.entity';
|
||||
import { Trips } from '../../src/db/entities/Trips.entity';
|
||||
import { Users } from '../../src/db/entities/Users.entity';
|
||||
import type { PluginEntityMetadataRepository } from '../../src/db/repositories/PluginEntityMetadata.repository';
|
||||
import type { ReservationsRepository } from '../../src/db/repositories/Reservations.repository';
|
||||
import type { TripsRepository } from '../../src/db/repositories/Trips.repository';
|
||||
import { UserIdentityTakenError, type UsersRepository } from '../../src/db/repositories/Users.repository';
|
||||
import { withRequestContext } from '../../src/nest/database/request-context';
|
||||
import { UnitOfWork } from '../../src/nest/database/unit-of-work';
|
||||
import { makeDayAccommodation } from '../helpers/factories/itinerary';
|
||||
import { makePlace } from '../helpers/factories/places';
|
||||
import { makeReservation } from '../helpers/factories/reservations';
|
||||
import { addTripMember, makeTrip, readTripDays } from '../helpers/factories/trips';
|
||||
import { makeUser } from '../helpers/factories/users';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import type { INestApplication } from '@nestjs/common';
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
let app: INestApplication;
|
||||
let orm: MikroORM;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildApp();
|
||||
orm = app.get(MikroORM);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await app?.close();
|
||||
});
|
||||
|
||||
/** `fn` on a fresh request context, as a request runs it. */
|
||||
function inRequest<T>(fn: () => Promise<T>): Promise<T> {
|
||||
return withRequestContext(orm, fn);
|
||||
}
|
||||
|
||||
const trips = () => orm.em.getRepository(Trips) as unknown as TripsRepository;
|
||||
const reservations = () => orm.em.getRepository(Reservations) as unknown as ReservationsRepository;
|
||||
const users = () => orm.em.getRepository(Users) as unknown as UsersRepository;
|
||||
|
||||
describe('Postgres answers as SQLite does', () => {
|
||||
it("PGSEM-101: a stay a booking points at as '14.0' is public in the calendar and share reads, for its trip and others", async () => {
|
||||
const { user } = await makeUser(orm);
|
||||
const linked = await makeTrip(orm, user.id, { start_date: '2026-09-01', end_date: '2026-09-03' });
|
||||
const other = await makeTrip(orm, user.id, { start_date: '2026-10-01', end_date: '2026-10-03' });
|
||||
const stays: number[] = [];
|
||||
for (const trip of [linked, other]) {
|
||||
const place = await makePlace(orm, trip.id);
|
||||
const days = await readTripDays(orm, trip.id);
|
||||
const stay = await makeDayAccommodation(orm, trip.id, place.id, days[0].id, days[1].id);
|
||||
stays.push(stay.id);
|
||||
}
|
||||
await makeReservation(orm, linked.id, { type: 'hotel', title: 'Hotel', accommodation_id: `${stays[0]}.0` });
|
||||
|
||||
for (const [trip, stay] of [
|
||||
[linked.id, stays[0]],
|
||||
[other.id, stays[1]],
|
||||
] as const) {
|
||||
const calendar = await inRequest(() => reservations().listPublicStaysForCalendar(trip));
|
||||
expect(calendar.map((row) => row.id)).toEqual([stay]);
|
||||
const shared = await inRequest(() => reservations().listPublicAccommodationsForShare(trip));
|
||||
expect(shared.map((row) => row.id)).toEqual([stay]);
|
||||
await expect(inRequest(() => reservations().listPublicStayPlaceIdsForShare(trip))).resolves.toHaveLength(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('PGSEM-102: the traveller list reports the owner flag as the numbers 1 and 0', async () => {
|
||||
const { user: owner } = await makeUser(orm, { username: 'pgsem-owner' });
|
||||
const { user: member } = await makeUser(orm, { username: 'pgsem-member' });
|
||||
const trip = await makeTrip(orm, owner.id);
|
||||
await addTripMember(orm, trip.id, member.id, owner.id);
|
||||
await expect(inRequest(() => trips().listTravellerUsernames(trip.id))).resolves.toEqual([
|
||||
{ username: 'pgsem-owner', is_owner: 1 },
|
||||
{ username: 'pgsem-member', is_owner: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('PGSEM-103: a taken email or username is refused as that field, whichever constraint Postgres names', async () => {
|
||||
await makeUser(orm, { username: 'pgsem-anna', email: 'pgsem-anna@example.com' });
|
||||
const row = (username: string, email: string) => ({ username, email, password_hash: 'x', role: 'user' });
|
||||
const sameEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna2', 'pgsem-anna@example.com')));
|
||||
await expect(sameEmail).rejects.toMatchObject({ field: 'email' });
|
||||
const caseEmail = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna3', 'PGSEM-Anna@example.com')));
|
||||
await expect(caseEmail).rejects.toMatchObject({ field: 'email' });
|
||||
const sameName = inRequest(() => users().insertAdminCreatedUser(row('pgsem-anna', 'pgsem-other@example.com')));
|
||||
await expect(sameName).rejects.toBeInstanceOf(UserIdentityTakenError);
|
||||
await expect(sameName).rejects.toMatchObject({ field: 'username' });
|
||||
});
|
||||
|
||||
it('PGSEM-104: NULLs sort first ascending and last descending, as on SQLite', async () => {
|
||||
const { user } = await makeUser(orm);
|
||||
const dated = await makeTrip(orm, user.id, { start_date: '2026-05-01', end_date: '2026-05-02' });
|
||||
const undated = await makeTrip(orm, user.id);
|
||||
const summaries = await inRequest(() => trips().listSummariesByIds([dated.id, undated.id]));
|
||||
expect(summaries.map((row) => row.id)).toEqual([dated.id, undated.id]);
|
||||
|
||||
const trip = await makeTrip(orm, user.id);
|
||||
const timed = await makeReservation(orm, trip.id, { reservation_time: '2026-05-01T10:00' });
|
||||
const untimed = await makeReservation(orm, trip.id, { reservation_time: null });
|
||||
const listed = await inRequest(() => reservations().listForTrip(trip.id));
|
||||
expect(listed.map((row) => row.id)).toEqual([untimed.id, timed.id]);
|
||||
});
|
||||
|
||||
it('PGSEM-105: capped plugin writes racing on the pool, each in a unit of work, never pass the cap', async () => {
|
||||
const uow = app.get(UnitOfWork, { strict: false });
|
||||
const meta = () => orm.em.getRepository(PluginEntityMetadata) as unknown as PluginEntityMetadataRepository;
|
||||
const cap = 5;
|
||||
const written = await Promise.all(
|
||||
Array.from({ length: 12 }, (_, i) =>
|
||||
inRequest(() => uow.transactional(() => meta().upsertValueCapped('pgsem', 'trip', 1, `k${i}`, '1', cap))),
|
||||
),
|
||||
);
|
||||
expect(written.filter(Boolean)).toHaveLength(cap);
|
||||
const stored = await inRequest(() => meta().countForEntity('pgsem', 'trip', 1));
|
||||
expect(stored).toBe(cap);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* SQLite's NULL position on Postgres (src/db/engine/postgres/null-order.ts):
|
||||
* the compiled ORDER BY of a Kysely statement and MikroORM's rendered ORDER BY
|
||||
* items. Nothing here reaches a server; the Postgres lane runs the orders
|
||||
* against a real one (PGSEM-004).
|
||||
*/
|
||||
import {
|
||||
SqliteNullOrderCompiler,
|
||||
SqliteNullOrderPlatform,
|
||||
withSqliteNullOrder,
|
||||
withSqliteNullPosition,
|
||||
} from '../../../../../src/db/engine/postgres/null-order';
|
||||
|
||||
import { DummyDriver, Kysely, PostgresAdapter, PostgresDialect, PostgresIntrospector, sql } from 'kysely';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
interface ProbeDB {
|
||||
trips: { id: number; start_date: string | null; title: string };
|
||||
}
|
||||
|
||||
const db = new Kysely<ProbeDB>({
|
||||
dialect: withSqliteNullOrder({
|
||||
createAdapter: () => new PostgresAdapter(),
|
||||
createDriver: () => new DummyDriver(),
|
||||
createIntrospector: (kysely) => new PostgresIntrospector(kysely),
|
||||
createQueryCompiler: () => {
|
||||
throw new Error('the wrapped dialect brings its own compiler');
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
describe('SQLite NULL order on Postgres', () => {
|
||||
it('NULLORDER-001: an ascending item sorts NULLs first and a descending one last, as SQLite does', () => {
|
||||
const compiled = db
|
||||
.selectFrom('trips')
|
||||
.select('id')
|
||||
.orderBy('start_date', 'desc')
|
||||
.orderBy('title')
|
||||
.orderBy('id', 'asc')
|
||||
.compile();
|
||||
expect(compiled.sql).toBe(
|
||||
'select "id" from "trips" order by "start_date" desc nulls last, "title" nulls first, "id" asc nulls first',
|
||||
);
|
||||
});
|
||||
|
||||
it('NULLORDER-002: an explicit NULL position and an expression item are kept as written', () => {
|
||||
const compiled = db
|
||||
.selectFrom('trips')
|
||||
.select('id')
|
||||
.orderBy('start_date', (ob) => ob.asc().nullsLast())
|
||||
.orderBy((eb) => eb('start_date', 'is', null), 'desc')
|
||||
.orderBy(sql`lower(title)`, (ob) => ob.desc())
|
||||
.compile();
|
||||
expect(compiled.sql).toBe(
|
||||
'select "id" from "trips" order by "start_date" asc nulls last, "start_date" is null desc nulls last, lower(title) desc nulls last',
|
||||
);
|
||||
});
|
||||
|
||||
it('NULLORDER-003: the wrapped dialect keeps its adapter, driver and introspector, and compiles with the null-order compiler', () => {
|
||||
const inner = new PostgresDialect({ pool: {} as never });
|
||||
const wrapped = withSqliteNullOrder(inner);
|
||||
expect(wrapped.createAdapter()).toBeInstanceOf(PostgresAdapter);
|
||||
expect(wrapped.createQueryCompiler()).toBeInstanceOf(SqliteNullOrderCompiler);
|
||||
});
|
||||
|
||||
it("NULLORDER-004: MikroORM's ORDER BY items get SQLite's NULL position unless they name one", () => {
|
||||
expect(withSqliteNullPosition('"t"."start_date" asc')).toBe('"t"."start_date" asc nulls first');
|
||||
expect(withSqliteNullPosition('"t"."start_date" desc')).toBe('"t"."start_date" desc nulls last');
|
||||
expect(withSqliteNullPosition('"t"."start_date" desc nulls first')).toBe('"t"."start_date" desc nulls first');
|
||||
const platform = new SqliteNullOrderPlatform();
|
||||
expect(platform.getOrderByExpression('"name"', 'ASC')).toEqual(['"name" asc nulls first']);
|
||||
expect(platform.getOrderByExpression('"name"', 'desc', 'nocase')).toEqual(['"name" collate "nocase" desc nulls last']);
|
||||
expect(platform.getOrderByExpression('"name"', 'asc nulls last')).toEqual(['"name" asc nulls last']);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user