mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-19 13:36:48 +00:00
feat(ferratas): keep loaded via ferratas in a tile cache on server and device, group segments, own look and count badge
This commit is contained in:
Maurice
@@ -31,6 +31,7 @@
|
||||
"DocumentProviderFields": "doc-sync",
|
||||
"DocumentProviders": "doc-sync",
|
||||
"DocumentSyncItems": "doc-sync",
|
||||
"FerrataTiles": "ferratas",
|
||||
"FileLinks": "files",
|
||||
"GoogleApiUsage": "google-quota",
|
||||
"GooglePlacePhotoMeta": "place-photos",
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { EntityRepositoryType, PrimaryKeyProp, defineEntity, p } from '@mikro-orm/core';
|
||||
import { FerrataTilesRepository } from '../repositories/FerrataTiles.repository';
|
||||
|
||||
export class FerrataTiles {
|
||||
[EntityRepositoryType]?: FerrataTilesRepository;
|
||||
[PrimaryKeyProp]?: 'tile_key';
|
||||
tile_key!: string;
|
||||
payload!: string;
|
||||
fetched_at!: number;
|
||||
}
|
||||
|
||||
export const FerrataTilesSchema = defineEntity({
|
||||
class: FerrataTiles,
|
||||
repository: () => FerrataTilesRepository,
|
||||
properties: {
|
||||
tile_key: p.text().primary(),
|
||||
payload: p.text(),
|
||||
fetched_at: p.integer(),
|
||||
},
|
||||
});
|
||||
@@ -31,6 +31,7 @@ import { DocumentConnectionsSchema } from './DocumentConnections.entity';
|
||||
import { DocumentProviderFieldsSchema } from './DocumentProviderFields.entity';
|
||||
import { DocumentProvidersSchema } from './DocumentProviders.entity';
|
||||
import { DocumentSyncItemsSchema } from './DocumentSyncItems.entity';
|
||||
import { FerrataTilesSchema } from './FerrataTiles.entity';
|
||||
import { FileLinksSchema } from './FileLinks.entity';
|
||||
import { GoogleApiUsageSchema } from './GoogleApiUsage.entity';
|
||||
import { GooglePlacePhotoMetaSchema } from './GooglePlacePhotoMeta.entity';
|
||||
@@ -165,6 +166,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [
|
||||
DocumentProviderFieldsSchema,
|
||||
DocumentProvidersSchema,
|
||||
DocumentSyncItemsSchema,
|
||||
FerrataTilesSchema,
|
||||
FileLinksSchema,
|
||||
GoogleApiUsageSchema,
|
||||
GooglePlacePhotoMetaSchema,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { JsonColumn } from '../utils/json-column';
|
||||
import { FERRATAS_RESULT_CAP, ferrataSchema, type Ferrata } from '@trek/shared';
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -106,3 +107,20 @@ export const BUDGET_ITEM_TICKET: JsonColumn<{ items: { price?: string | number;
|
||||
.loose(),
|
||||
fallback: () => null,
|
||||
};
|
||||
|
||||
/** What one cell of the via ferrata grid holds: its ferratas, and whether the Overpass answer they came from was cut. */
|
||||
export interface FerrataTilePayload {
|
||||
ferratas: Ferrata[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* One via ferrata grid cell, as Overpass answered it. Checked against the wire
|
||||
* contract, so a stored row can go out as it is. A broken row reads as a cell
|
||||
* never fetched (null), and the next request asks Overpass for it again.
|
||||
*/
|
||||
export const FERRATA_TILE_PAYLOAD: JsonColumn<FerrataTilePayload | null> = {
|
||||
column: 'ferrata_tiles.payload',
|
||||
schema: z.object({ ferratas: z.array(ferrataSchema).max(FERRATAS_RESULT_CAP), truncated: z.boolean() }).nullable(),
|
||||
fallback: () => null,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:
|
||||
// change the migration, then run `npm run gen:db-types`.
|
||||
|
||||
export interface FerrataTilesTable {
|
||||
tile_key: string;
|
||||
payload: string;
|
||||
fetched_at: number;
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import type { DocumentConnectionsTable } from './DocumentConnections.table';
|
||||
import type { DocumentProviderFieldsTable } from './DocumentProviderFields.table';
|
||||
import type { DocumentProvidersTable } from './DocumentProviders.table';
|
||||
import type { DocumentSyncItemsTable } from './DocumentSyncItems.table';
|
||||
import type { FerrataTilesTable } from './FerrataTiles.table';
|
||||
import type { FileLinksTable } from './FileLinks.table';
|
||||
import type { GoogleApiUsageTable } from './GoogleApiUsage.table';
|
||||
import type { GooglePlacePhotoMetaTable } from './GooglePlacePhotoMeta.table';
|
||||
@@ -174,6 +175,7 @@ export interface DB {
|
||||
document_provider_fields: DocumentProviderFieldsTable;
|
||||
document_providers: DocumentProvidersTable;
|
||||
document_sync_items: DocumentSyncItemsTable;
|
||||
ferrata_tiles: FerrataTilesTable;
|
||||
file_links: FileLinksTable;
|
||||
google_api_usage: GoogleApiUsageTable;
|
||||
google_place_photo_meta: GooglePlacePhotoMetaTable;
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
import { Migration } from '@mikro-orm/migrations';
|
||||
|
||||
/**
|
||||
* `ferrata_tiles` keeps what Overpass answered for one cell of the via
|
||||
* ferrata grid (0.05 degrees a side), so a restart does not send every map
|
||||
* pan back to the public mirrors.
|
||||
*
|
||||
* `tile_key` names the cell by its grid indices (`"<lat index>,<lng index>"`),
|
||||
* `payload` is the JSON of the cell's ferratas and whether the answer was cut,
|
||||
* and `fetched_at` is the epoch of that answer in seconds (not ms, so it fits
|
||||
* the 32-bit integer this column is on Postgres). Nothing references the
|
||||
* table and nothing deletes from it: an old cell is served and refreshed, not
|
||||
* dropped.
|
||||
*/
|
||||
export class Migration20200101043200_via_ferrata_tiles_outlive_a_restart extends Migration {
|
||||
override name = 'Migration20200101043200_via_ferrata_tiles_outlive_a_restart';
|
||||
|
||||
override up(): void {
|
||||
this.addSql(`
|
||||
CREATE TABLE IF NOT EXISTS ferrata_tiles (
|
||||
tile_key TEXT PRIMARY KEY NOT NULL,
|
||||
payload TEXT NOT NULL,
|
||||
fetched_at INTEGER NOT NULL
|
||||
)
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { FerrataTiles } from '../entities/FerrataTiles.entity';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
|
||||
export interface FerrataTileRow {
|
||||
tile_key: string;
|
||||
payload: string;
|
||||
/** When Overpass answered for the cell, as an epoch in seconds. */
|
||||
fetched_at: number;
|
||||
}
|
||||
|
||||
export class FerrataTilesRepository extends TrekRepository<FerrataTiles> {
|
||||
/** The stored cells among `keys`, in no particular order; a key never stored is simply absent. */
|
||||
async findByKeys(keys: readonly string[]): Promise<FerrataTileRow[]> {
|
||||
if (keys.length === 0) return [];
|
||||
const rows = await this.find({ tile_key: { $in: [...keys] } }, { fields: ['tile_key', 'payload', 'fetched_at'] });
|
||||
return rows.map((row) => ({ tile_key: row.tile_key, payload: row.payload, fetched_at: row.fetched_at }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes every cell, replacing the payload and fetch time of one already
|
||||
* stored. The caller wraps it in a transaction, so a batch lands whole.
|
||||
*/
|
||||
async upsertTiles(rows: readonly FerrataTileRow[]): Promise<void> {
|
||||
if (rows.length === 0) return;
|
||||
await this.upsertMany(
|
||||
rows.map((row) => ({ ...row })),
|
||||
{ onConflictFields: ['tile_key'], onConflictAction: 'merge' },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,15 @@
|
||||
import { FERRATAS_RESULT_CAP, type Ferrata, type FerrataBbox, type FerrataGrade } from '@trek/shared';
|
||||
import {
|
||||
FERRATAS_RESULT_CAP,
|
||||
ferrataTileBlockKey,
|
||||
type Ferrata,
|
||||
type FerrataBbox,
|
||||
type FerrataGrade,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Pure pieces of the via ferrata lookup: the Overpass query, the cache key, the
|
||||
* Pure pieces of the via ferrata lookup: the Overpass query, the tile grid, the
|
||||
* grade parser and the mapping from Overpass elements to the wire rows. Kept
|
||||
* free of I/O so the service and its tests share them.
|
||||
*/
|
||||
@@ -11,7 +17,7 @@ import { z } from 'zod';
|
||||
/** How long Overpass itself may work on one query, in seconds (`[timeout:N]`). */
|
||||
export const FERRATA_QUERY_TIMEOUT_S = 15;
|
||||
|
||||
/** The grid the box is snapped outwards to, so nearby requests share one cache entry. */
|
||||
/** The grid the box is snapped outwards to; one cell of it is the unit the answers are stored by. */
|
||||
const GRID_DEG = 0.05;
|
||||
|
||||
const GRADE_RE = /^([0-6])([+-])?$/;
|
||||
@@ -37,9 +43,8 @@ const snapUp = (v: number) => Math.ceil(v / GRID_DEG - 1e-9) * GRID_DEG;
|
||||
const fixed = (v: number) => Number(v.toFixed(2));
|
||||
|
||||
/**
|
||||
* The box grown outwards to the 0.05 degree grid and clamped to the globe.
|
||||
* The query asks for the snapped box, so a cache entry always covers every
|
||||
* request that maps onto it.
|
||||
* The box grown outwards to the 0.05 degree grid and clamped to the globe,
|
||||
* so it is made of whole cells.
|
||||
*/
|
||||
export function snapFerrataBbox(bbox: FerrataBbox): FerrataBbox {
|
||||
return {
|
||||
@@ -50,8 +55,104 @@ export function snapFerrataBbox(bbox: FerrataBbox): FerrataBbox {
|
||||
};
|
||||
}
|
||||
|
||||
export function ferrataCacheKey(snapped: FerrataBbox): string {
|
||||
return `${snapped.south},${snapped.west},${snapped.north},${snapped.east}`;
|
||||
/**
|
||||
* The grid cells a box covers, as tile keys (`"<lat index>,<lng index>"`, the
|
||||
* cell's south-west corner divided by the grid), south to north and west to
|
||||
* east. The box is snapped outwards first, so every cell is whole. `null` when
|
||||
* the box covers more than `max` cells: the caller refuses it rather than
|
||||
* walking a grid it never meant to ask for.
|
||||
*/
|
||||
export function ferrataTileKeysFor(bbox: FerrataBbox, max: number): string[] | null {
|
||||
const snapped = snapFerrataBbox(bbox);
|
||||
const lat0 = Math.round(snapped.south / GRID_DEG);
|
||||
const lat1 = Math.round(snapped.north / GRID_DEG);
|
||||
const lng0 = Math.round(snapped.west / GRID_DEG);
|
||||
const lng1 = Math.round(snapped.east / GRID_DEG);
|
||||
if ((lat1 - lat0) * (lng1 - lng0) > max) return null;
|
||||
const keys: string[] = [];
|
||||
for (let lat = lat0; lat < lat1; lat += 1) {
|
||||
for (let lng = lng0; lng < lng1; lng += 1) keys.push(tileKey(lat, lng));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
const tileKey = (lat: number, lng: number) => `${lat},${lng}`;
|
||||
|
||||
function tileIndices(key: string): [number, number] {
|
||||
const parts = key.split(',').map(Number);
|
||||
const lat = parts[0] ?? Number.NaN;
|
||||
const lng = parts[1] ?? Number.NaN;
|
||||
if (parts.length !== 2 || !Number.isInteger(lat) || !Number.isInteger(lng)) {
|
||||
throw new Error(`Not a ferrata tile key: ${key}`);
|
||||
}
|
||||
return [lat, lng];
|
||||
}
|
||||
|
||||
/** The one box around every given cell, which is what a batch asks Overpass for. */
|
||||
export function ferrataTilesBbox(keys: readonly string[]): FerrataBbox {
|
||||
if (keys.length === 0) throw new Error('No ferrata tiles to bound');
|
||||
let south = Infinity;
|
||||
let west = Infinity;
|
||||
let north = -Infinity;
|
||||
let east = -Infinity;
|
||||
for (const key of keys) {
|
||||
const [lat, lng] = tileIndices(key);
|
||||
south = Math.min(south, lat);
|
||||
west = Math.min(west, lng);
|
||||
north = Math.max(north, lat + 1);
|
||||
east = Math.max(east, lng + 1);
|
||||
}
|
||||
return {
|
||||
south: fixed(Math.max(-90, south * GRID_DEG)),
|
||||
west: fixed(Math.max(-180, west * GRID_DEG)),
|
||||
north: fixed(Math.min(90, north * GRID_DEG)),
|
||||
east: fixed(Math.min(180, east * GRID_DEG)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The cells grouped into the grid-aligned blocks of FERRATA_TILES_PER_REQUEST_SIDE
|
||||
* cells a side that one Overpass query covers (the blocks the client asks in),
|
||||
* each block's cells in the order given.
|
||||
*/
|
||||
export function ferrataTileBlocks(keys: readonly string[]): string[][] {
|
||||
const blocks = new Map<string, string[]>();
|
||||
for (const key of keys) {
|
||||
const [lat, lng] = tileIndices(key);
|
||||
const block = ferrataTileBlockKey(lat, lng);
|
||||
const cells = blocks.get(block);
|
||||
if (cells) cells.push(key);
|
||||
else blocks.set(block, [key]);
|
||||
}
|
||||
return [...blocks.values()];
|
||||
}
|
||||
|
||||
/** The grid rows (or columns) a coordinate lies in: two when it sits on the line between them. */
|
||||
function gridIndices(value: number): number[] {
|
||||
const steps = value / GRID_DEG;
|
||||
const nearest = Math.round(steps);
|
||||
return Math.abs(steps - nearest) < 1e-9 ? [nearest - 1, nearest] : [Math.floor(steps)];
|
||||
}
|
||||
|
||||
/**
|
||||
* The ferratas of one Overpass answer, handed to every cell among `keys` that
|
||||
* a point of their line lies in (a point on a cell edge counts for both
|
||||
* cells). A cell no ferrata touches gets an empty list, which is an answer
|
||||
* too: it is stored, so the cell is not asked for again.
|
||||
*/
|
||||
export function splitFerratasByTile(ferratas: readonly Ferrata[], keys: ReadonlySet<string>): Map<string, Ferrata[]> {
|
||||
const byTile = new Map<string, Ferrata[]>();
|
||||
for (const key of keys) byTile.set(key, []);
|
||||
for (const ferrata of ferratas) {
|
||||
const touched = new Set<string>();
|
||||
for (const [lat, lng] of ferrata.geometry) {
|
||||
for (const row of gridIndices(lat)) {
|
||||
for (const column of gridIndices(lng)) touched.add(tileKey(row, column));
|
||||
}
|
||||
}
|
||||
for (const key of touched) byTile.get(key)?.push(ferrata);
|
||||
}
|
||||
return byTile;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
import { FerrataTiles } from '../../db/entities/FerrataTiles.entity';
|
||||
import { PermissionsModule } from '../permissions/permissions.module';
|
||||
import { FerratasOverpassClient } from './ferratas-overpass.client';
|
||||
import { FerratasController } from './ferratas.controller';
|
||||
import { FerratasService } from './ferratas.service';
|
||||
import { MikroOrmModule } from '@mikro-orm/nestjs';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Via ferratas around a trip, read from OpenStreetMap through Overpass.
|
||||
* PermissionsModule brings TripAccessGuard; RuntimeEnvService (the Overpass
|
||||
* endpoints and deadline) comes from the global AppConfigModule.
|
||||
* endpoints and deadline) comes from the global AppConfigModule. The answers
|
||||
* are kept per grid cell in `ferrata_tiles`; UnitOfWork and MikroORM come from
|
||||
* the global OrmModule.
|
||||
*/
|
||||
@Module({
|
||||
imports: [PermissionsModule],
|
||||
imports: [PermissionsModule, MikroOrmModule.forFeature([FerrataTiles])],
|
||||
controllers: [FerratasController],
|
||||
providers: [FerratasService, FerratasOverpassClient],
|
||||
})
|
||||
|
||||
@@ -1,99 +1,309 @@
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { FerrataTiles } from '../../db/entities/FerrataTiles.entity';
|
||||
import { FERRATA_TILE_PAYLOAD, type FerrataTilePayload } from '../../db/json-columns';
|
||||
import type { FerrataTileRow, FerrataTilesRepository } from '../../db/repositories/FerrataTiles.repository';
|
||||
import { decodeJsonResult, encodeJson, logJsonFailure } from '../../utils/json-column';
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import { badRequest, DomainError } from '../common/domain-error';
|
||||
import { withRequestContext } from '../database/request-context';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { FerratasOverpassClient } from './ferratas-overpass.client';
|
||||
import { buildFerrataQuery, ferrataCacheKey, snapFerrataBbox, toFerratas } from './ferratas.helpers';
|
||||
import {
|
||||
buildFerrataQuery,
|
||||
ferrataTileBlocks,
|
||||
ferrataTileKeysFor,
|
||||
ferrataTilesBbox,
|
||||
splitFerratasByTile,
|
||||
toFerratas,
|
||||
} from './ferratas.helpers';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { FERRATAS_ATTRIBUTION, FERRATAS_UNAVAILABLE_CODE, type FerrataBbox, type FerratasResponse } from '@trek/shared';
|
||||
import {
|
||||
FERRATA_BBOX_TOO_LARGE_MESSAGE,
|
||||
FERRATAS_ATTRIBUTION,
|
||||
FERRATAS_RESULT_CAP,
|
||||
FERRATAS_UNAVAILABLE_CODE,
|
||||
type Ferrata,
|
||||
type FerrataBbox,
|
||||
type FerratasResponse,
|
||||
} from '@trek/shared';
|
||||
|
||||
/** Via ferratas change on the scale of seasons; a day keeps Overpass out of every map pan. */
|
||||
export const FERRATA_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
/**
|
||||
* Via ferratas change on the scale of seasons. A cell older than this is still
|
||||
* served at once, and asked for again in the background.
|
||||
*/
|
||||
export const FERRATA_TILE_STALE_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
/** The most boxes held at once; the oldest goes first, so roaming the globe cannot grow the map without bound. */
|
||||
export const FERRATA_CACHE_MAX = 300;
|
||||
/** The most cells held in memory in front of the table; the least recently used goes first. */
|
||||
const FERRATA_TILE_MEMORY_MAX = 2000;
|
||||
|
||||
/**
|
||||
* The most cells one request may cover. The contract's span cap already keeps
|
||||
* a box to 11 by 11 cells; this holds the service to a bound of its own.
|
||||
*/
|
||||
const FERRATA_TILES_MAX_PER_REQUEST = 200;
|
||||
|
||||
/**
|
||||
* The most Overpass queries this process runs at once. A box is queried in
|
||||
* blocks of 0.1 degree, so a wide one turns into several queries; they wait
|
||||
* their turn here rather than all hitting the public mirrors together.
|
||||
*/
|
||||
const FERRATA_QUERIES_AT_ONCE = 2;
|
||||
|
||||
/** `fetched_at` is kept in seconds, so the epoch fits a 32-bit integer column on Postgres too. */
|
||||
const MS_PER_S = 1000;
|
||||
|
||||
/**
|
||||
* `fetched_at` of a cell whose answer was cut at the cap: the oldest possible,
|
||||
* so a restart serves it and asks again rather than keeping a partial cell for
|
||||
* FERRATA_TILE_STALE_MS. This process keeps it for what it is meanwhile.
|
||||
*/
|
||||
const FERRATA_CUT_TILE_FETCHED_AT = 0;
|
||||
|
||||
export const FERRATAS_UNAVAILABLE_MESSAGE = 'Via ferrata data is unavailable right now';
|
||||
|
||||
interface CacheEntry {
|
||||
at: number;
|
||||
value: FerratasResponse;
|
||||
interface StoredTile {
|
||||
fetchedAt: number;
|
||||
payload: FerrataTilePayload;
|
||||
}
|
||||
|
||||
type TileBatch = Promise<ReadonlyMap<string, StoredTile>>;
|
||||
|
||||
/**
|
||||
* The via ferratas inside a box, from OpenStreetMap.
|
||||
*
|
||||
* Read-only and per process: answers are cached by the box snapped to a 0.05
|
||||
* degree grid, and concurrent requests for the same snapped box share one
|
||||
* Overpass call. A failure is never cached, so the next request tries again;
|
||||
* it reaches the client as a 502 `{ error, code }` it can show, never as a
|
||||
* 500 with a stack.
|
||||
* The unit is one cell of the 0.05 degree grid. A box is resolved into its
|
||||
* cells, each served from `ferrata_tiles` (with a small LRU in front), and
|
||||
* Overpass is asked only for the cells never fetched: one query per
|
||||
* grid-aligned block of FERRATA_TILES_PER_REQUEST_SIDE cells a side (0.1
|
||||
* degree), at most FERRATA_QUERIES_AT_ONCE at a time, each answer split back
|
||||
* per cell and stored. A block whose answer was cut at the cap is asked again
|
||||
* cell by cell, and a cell still cut on its own is kept but stored as old, so
|
||||
* it is not served as complete for a month. A cell older than
|
||||
* FERRATA_TILE_STALE_MS is served as it is and refreshed in the background,
|
||||
* in queries of its own, so a refresh can never fail the request that saw
|
||||
* it; nothing is deleted. Concurrent requests for a cell share one call.
|
||||
*
|
||||
* A request fails only when a cell it needs has never been fetched and
|
||||
* Overpass fails too: the client gets a 502 `{ error, code }` it can show,
|
||||
* never a 500 with a stack. A failed background refresh is logged and leaves
|
||||
* the old cell in place.
|
||||
*/
|
||||
@Injectable()
|
||||
export class FerratasService {
|
||||
private readonly cache = new Map<string, CacheEntry>();
|
||||
private readonly pending = new Map<string, Promise<FerratasResponse>>();
|
||||
private readonly memory = new Map<string, StoredTile>();
|
||||
private readonly pending = new Map<string, TileBatch>();
|
||||
/** Overpass queries running now, and the ones waiting for a turn. */
|
||||
private running = 0;
|
||||
private readonly waiting: Array<() => void> = [];
|
||||
|
||||
constructor(private readonly overpass: FerratasOverpassClient) {}
|
||||
constructor(
|
||||
private readonly overpass: FerratasOverpassClient,
|
||||
@InjectRepository(FerrataTiles) private readonly tiles: FerrataTilesRepository,
|
||||
private readonly uow: UnitOfWork,
|
||||
private readonly orm: MikroORM,
|
||||
) {}
|
||||
|
||||
/** The via ferratas that reach into `bbox`; the cache holds the wider snapped box. */
|
||||
/** The via ferratas that reach into `bbox`, gathered from the cells it covers. */
|
||||
async list(bbox: FerrataBbox, now: number = Date.now()): Promise<FerratasResponse> {
|
||||
return withinBbox(await this.listSnapped(bbox, now), bbox);
|
||||
const keys = ferrataTileKeysFor(bbox, FERRATA_TILES_MAX_PER_REQUEST);
|
||||
if (keys === null) throw badRequest(FERRATA_BBOX_TOO_LARGE_MESSAGE);
|
||||
|
||||
const tiles = new Map(await this.stored(keys));
|
||||
const stale = keys.filter((key) => {
|
||||
const tile = tiles.get(key);
|
||||
return tile !== undefined && now - tile.fetchedAt >= FERRATA_TILE_STALE_MS && !this.pending.has(key);
|
||||
});
|
||||
const missing = keys.filter((key) => !tiles.has(key));
|
||||
|
||||
if (stale.length > 0) this.refreshInBackground(stale, now);
|
||||
if (missing.length > 0) {
|
||||
for (const [key, tile] of await this.fetchMissing(missing, now)) tiles.set(key, tile);
|
||||
}
|
||||
return gather(keys, tiles, bbox);
|
||||
}
|
||||
|
||||
private async listSnapped(bbox: FerrataBbox, now: number): Promise<FerratasResponse> {
|
||||
const snapped = snapFerrataBbox(bbox);
|
||||
const key = ferrataCacheKey(snapped);
|
||||
const hit = this.cache.get(key);
|
||||
if (hit && now - hit.at < FERRATA_CACHE_TTL_MS) return hit.value;
|
||||
if (hit) this.cache.delete(key);
|
||||
/** The cells among `keys` that were ever fetched: memory first, the table for the rest. */
|
||||
private async stored(keys: readonly string[]): Promise<Map<string, StoredTile>> {
|
||||
const found = new Map<string, StoredTile>();
|
||||
const unread: string[] = [];
|
||||
for (const key of keys) {
|
||||
const hit = this.memory.get(key);
|
||||
if (hit) {
|
||||
this.remember(key, hit);
|
||||
found.set(key, hit);
|
||||
} else {
|
||||
unread.push(key);
|
||||
}
|
||||
}
|
||||
for (const row of await this.tiles.findByKeys(unread)) {
|
||||
const decoded = decodeJsonResult(FERRATA_TILE_PAYLOAD, row.payload);
|
||||
if (!decoded.ok || decoded.value === null) {
|
||||
logJsonFailure(FERRATA_TILE_PAYLOAD, decoded.ok ? 'schema' : decoded.reason, row.tile_key);
|
||||
continue;
|
||||
}
|
||||
const tile = { fetchedAt: row.fetched_at * MS_PER_S, payload: decoded.value };
|
||||
this.remember(row.tile_key, tile);
|
||||
found.set(row.tile_key, tile);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
const inFlight = this.pending.get(key);
|
||||
if (inFlight !== undefined) return inFlight;
|
||||
// Memoised in-flight load, not a missing await: concurrent callers join the
|
||||
// promise above, and this frame awaits it below.
|
||||
const load = this.load(snapped);
|
||||
this.pending.set(key, load);
|
||||
try {
|
||||
const value = await load;
|
||||
this.remember(key, { at: now, value });
|
||||
return value;
|
||||
} finally {
|
||||
this.pending.delete(key);
|
||||
/**
|
||||
* The cells never fetched, awaited. A cell another request is already
|
||||
* fetching joins that batch; the rest go out one batch per block.
|
||||
*/
|
||||
private async fetchMissing(missing: readonly string[], now: number): Promise<Map<string, StoredTile>> {
|
||||
const tiles = new Map<string, StoredTile>();
|
||||
const batches = new Set<TileBatch>();
|
||||
const own: string[] = [];
|
||||
for (const key of missing) {
|
||||
const hit = this.memory.get(key);
|
||||
const inFlight = this.pending.get(key);
|
||||
if (hit) tiles.set(key, hit);
|
||||
else if (inFlight !== undefined) batches.add(inFlight);
|
||||
else own.push(key);
|
||||
}
|
||||
for (const block of ferrataTileBlocks(own)) batches.add(this.batch(block, now));
|
||||
|
||||
for (const answer of await Promise.all(batches)) {
|
||||
for (const [key, tile] of answer) tiles.set(key, tile);
|
||||
}
|
||||
return tiles;
|
||||
}
|
||||
|
||||
/** Refreshes old cells without holding up the request, a batch per block; a failure only costs the refresh. */
|
||||
private refreshInBackground(keys: readonly string[], now: number): void {
|
||||
for (const block of ferrataTileBlocks(keys)) {
|
||||
this.batch(block, now).catch((err: unknown) => {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
logWarn(`[Ferratas] background refresh of ${block.length} tile(s) failed: ${reason}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async load(snapped: FerrataBbox): Promise<FerratasResponse> {
|
||||
let elements: unknown[];
|
||||
/**
|
||||
* One Overpass query for `keys`, shared with every request that needs one of
|
||||
* them until it settles. It runs in a request context of its own: it can
|
||||
* outlive the request that started it.
|
||||
*/
|
||||
private batch(keys: readonly string[], now: number): TileBatch {
|
||||
const run = withRequestContext(this.orm, () => this.fetchAndStore(keys, now));
|
||||
for (const key of keys) this.pending.set(key, run);
|
||||
const settle = () => {
|
||||
for (const key of keys) {
|
||||
if (this.pending.get(key) === run) this.pending.delete(key);
|
||||
}
|
||||
};
|
||||
void run.then(settle, settle);
|
||||
return run;
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks Overpass for `keys` and stores the answer per cell. An answer cut at
|
||||
* the cap is not stored for a block: its cells are asked again one by one,
|
||||
* which is the most a cut answer can be narrowed. A single cell still cut is
|
||||
* kept in memory as it is, and stored as old, so the next start refreshes it.
|
||||
*/
|
||||
private async fetchAndStore(keys: readonly string[], now: number): Promise<Map<string, StoredTile>> {
|
||||
const elements = await this.queryOverpass(keys);
|
||||
const { ferratas, truncated } = toFerratas(elements);
|
||||
if (truncated && keys.length > 1) {
|
||||
const tiles = new Map<string, StoredTile>();
|
||||
const answers = await Promise.all(keys.map((key) => this.fetchAndStore([key], now)));
|
||||
for (const answer of answers) for (const [key, tile] of answer) tiles.set(key, tile);
|
||||
return tiles;
|
||||
}
|
||||
const byTile = splitFerratasByTile(ferratas, new Set(keys));
|
||||
const storedAt = truncated ? FERRATA_CUT_TILE_FETCHED_AT : Math.floor(now / MS_PER_S);
|
||||
|
||||
const tiles = new Map<string, StoredTile>();
|
||||
const rows: FerrataTileRow[] = [];
|
||||
for (const key of keys) {
|
||||
const payload: FerrataTilePayload = { ferratas: byTile.get(key) ?? [], truncated };
|
||||
tiles.set(key, { fetchedAt: now, payload });
|
||||
rows.push({ tile_key: key, payload: encodeJson(FERRATA_TILE_PAYLOAD, payload), fetched_at: storedAt });
|
||||
}
|
||||
try {
|
||||
elements = await this.overpass.query(buildFerrataQuery(snapped));
|
||||
await this.uow.transactional(() => this.tiles.upsertTiles(rows));
|
||||
} catch (err) {
|
||||
// The answer is still good for this process; the next start asks again.
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
logWarn(`[Ferratas] could not store ${rows.length} tile(s): ${reason}`);
|
||||
}
|
||||
for (const [key, tile] of tiles) this.remember(key, tile);
|
||||
return tiles;
|
||||
}
|
||||
|
||||
/** One Overpass query for the box around `keys`, once a turn is free; a failure becomes the 502 the client reads. */
|
||||
private async queryOverpass(keys: readonly string[]): Promise<unknown[]> {
|
||||
await this.turn();
|
||||
try {
|
||||
return await this.overpass.query(buildFerrataQuery(ferrataTilesBbox(keys)));
|
||||
} catch {
|
||||
// The client already logged why each mirror failed.
|
||||
throw new DomainError(502, FERRATAS_UNAVAILABLE_MESSAGE, {
|
||||
code: FERRATAS_UNAVAILABLE_CODE,
|
||||
details: { code: FERRATAS_UNAVAILABLE_CODE },
|
||||
});
|
||||
} finally {
|
||||
// The turn passes straight to the next query waiting, or is given back.
|
||||
const next = this.waiting.shift();
|
||||
if (next) next();
|
||||
else this.running -= 1;
|
||||
}
|
||||
const { ferratas, truncated } = toFerratas(elements);
|
||||
return { ferratas, truncated, source: 'openstreetmap', attribution: FERRATAS_ATTRIBUTION };
|
||||
}
|
||||
|
||||
private remember(key: string, entry: CacheEntry): void {
|
||||
this.cache.delete(key);
|
||||
while (this.cache.size >= FERRATA_CACHE_MAX) {
|
||||
const oldest = this.cache.keys().next();
|
||||
if (oldest.done) break;
|
||||
this.cache.delete(oldest.value);
|
||||
/** Resolves once the caller holds one of FERRATA_QUERIES_AT_ONCE turns, handed over by a query that finished. */
|
||||
private async turn(): Promise<void> {
|
||||
if (this.running < FERRATA_QUERIES_AT_ONCE) {
|
||||
this.running += 1;
|
||||
return;
|
||||
}
|
||||
this.cache.set(key, entry);
|
||||
await new Promise<void>((resolve) => this.waiting.push(resolve));
|
||||
}
|
||||
|
||||
private remember(key: string, tile: StoredTile): void {
|
||||
this.memory.delete(key);
|
||||
while (this.memory.size >= FERRATA_TILE_MEMORY_MAX) {
|
||||
const oldest = this.memory.keys().next();
|
||||
if (oldest.done) break;
|
||||
this.memory.delete(oldest.value);
|
||||
}
|
||||
this.memory.set(key, tile);
|
||||
}
|
||||
}
|
||||
|
||||
/** A ferrata belongs to the box when any point of its line lies inside it. */
|
||||
function withinBbox(response: FerratasResponse, bbox: FerrataBbox): FerratasResponse {
|
||||
const inside = ([lat, lng]: readonly unknown[]) =>
|
||||
typeof lat === 'number' &&
|
||||
typeof lng === 'number' &&
|
||||
lat >= bbox.south &&
|
||||
lat <= bbox.north &&
|
||||
lng >= bbox.west &&
|
||||
lng <= bbox.east;
|
||||
return { ...response, ferratas: response.ferratas.filter((ferrata) => ferrata.geometry.some(inside)) };
|
||||
function reachesInto(ferrata: Ferrata, bbox: FerrataBbox): boolean {
|
||||
return ferrata.geometry.some(
|
||||
([lat, lng]) => lat >= bbox.south && lat <= bbox.north && lng >= bbox.west && lng <= bbox.east,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The answer out of the cells: every ferrata once, however many cells it
|
||||
* crosses, kept when it reaches into the asked box, cut at the contract's cap.
|
||||
* The answer is marked cut when any cell came from a cut Overpass answer.
|
||||
*/
|
||||
function gather(keys: readonly string[], tiles: ReadonlyMap<string, StoredTile>, bbox: FerrataBbox): FerratasResponse {
|
||||
const seen = new Set<string>();
|
||||
const ferratas: Ferrata[] = [];
|
||||
let truncated = false;
|
||||
for (const key of keys) {
|
||||
const tile = tiles.get(key);
|
||||
if (!tile) continue;
|
||||
truncated ||= tile.payload.truncated;
|
||||
for (const ferrata of tile.payload.ferratas) {
|
||||
if (seen.has(ferrata.id)) continue;
|
||||
seen.add(ferrata.id);
|
||||
if (reachesInto(ferrata, bbox)) ferratas.push(ferrata);
|
||||
}
|
||||
}
|
||||
return {
|
||||
ferratas: ferratas.slice(0, FERRATAS_RESULT_CAP),
|
||||
truncated: truncated || ferratas.length > FERRATAS_RESULT_CAP,
|
||||
source: 'openstreetmap',
|
||||
attribution: FERRATAS_ATTRIBUTION,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3,10 +3,12 @@
|
||||
* TripAccessGuard over a migrated snapshot db. Only Overpass is faked, by stubbing
|
||||
* the global fetch; the response contract is checked on every 200 (NODE_ENV=test).
|
||||
*/
|
||||
import { FerrataTiles } from '../../src/db/entities/FerrataTiles.entity';
|
||||
import { AppConfigModule } from '../../src/nest/app-config/app-config.module';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
import { FerratasModule } from '../../src/nest/ferratas/ferratas.module';
|
||||
import { createSnapshotTestDb } from '../helpers/db-mock';
|
||||
import { countRows } from '../helpers/factories/rows';
|
||||
import { makeTrip } from '../helpers/factories/trips';
|
||||
import { makeUser } from '../helpers/factories/users';
|
||||
import { createTestMikroOrmModule } from '../helpers/test-orm';
|
||||
@@ -159,6 +161,20 @@ describe('Ferratas e2e (real guards + snapshot SQLite)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('200 from the stored cells for the same box, without asking Overpass again', async () => {
|
||||
const fetcher = vi.fn();
|
||||
vi.stubGlobal('fetch', fetcher);
|
||||
const res = await request(server)
|
||||
.get(url())
|
||||
.set('Cookie', sessionCookie(ownerId))
|
||||
.query({ bbox: '46.5,12.05,46.6,12.2' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.ferratas.map((f: { id: string }) => f.id)).toEqual(['way/4242']);
|
||||
expect(fetcher).not.toHaveBeenCalled();
|
||||
// 46.5 to 46.6 by 12.05 to 12.2: two rows of three cells.
|
||||
expect(await countRows(app.get(MikroORM), FerrataTiles)).toBe(6);
|
||||
});
|
||||
|
||||
it('502 { error, code } when no Overpass mirror answers', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.stubGlobal(
|
||||
|
||||
@@ -30,19 +30,20 @@ describe('ALL_ENTITIES', () => {
|
||||
});
|
||||
|
||||
/**
|
||||
* Plan 2 Task 3: every one of the 132 entities the migrated schema
|
||||
* Plan 2 Task 3: every one of the 133 entities the migrated schema
|
||||
* describes (120 rewritten + the 5 document-sync tables added this task,
|
||||
* + push_subscriptions and google_api_usage from legacy steps 245 and 254,
|
||||
* + tour_types, tours and tour_waypoints from the Tours migration,
|
||||
* + scheduler_leases from the cron lease migration
|
||||
* + user_sessions, the session rows behind revocable session tokens)
|
||||
* + user_sessions, the session rows behind revocable session tokens
|
||||
* + ferrata_tiles, the stored via ferrata grid cells)
|
||||
* is registered, and no two share a class name (a silent duplicate would
|
||||
* shadow one of them in `orm.getMetadata()`, exactly the kind of drift
|
||||
* `entity-schema-parity.test.ts` can't see — it iterates the metadata map,
|
||||
* which de-duplicates by class name).
|
||||
*/
|
||||
it('ENT-003: exactly 132 entities, every class name unique', () => {
|
||||
expect(ALL_ENTITIES.length).toBe(132);
|
||||
it('ENT-003: exactly 133 entities, every class name unique', () => {
|
||||
expect(ALL_ENTITIES.length).toBe(133);
|
||||
const names = ALL_ENTITIES.map((s) => s.meta.className);
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
});
|
||||
|
||||
@@ -15,7 +15,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
* `scripts/generate-entities.ts`'s `RULE1d_fixNaNNumericDefaults` and
|
||||
* `tests/unit/db/generate-entities.test.ts`'s `RULE1D-*` fixture tests for
|
||||
* the mechanism). This file pins the OUTPUT of that rule against the real,
|
||||
* committed entities (every one of the 132, not a sample) and reproduces
|
||||
* committed entities (every one of the 133, not a sample) and reproduces
|
||||
* the reviewer's own insert probe so a future regenerate/hand-edit that
|
||||
* reintroduces a `NaN` initialiser fails loudly here, not silently at a
|
||||
* caller's first `em.create()`.
|
||||
@@ -28,7 +28,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 446;
|
||||
|
||||
describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => {
|
||||
it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 132 entities', () => {
|
||||
it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => {
|
||||
const failures: string[] = [];
|
||||
for (const schema of ALL_ENTITIES) {
|
||||
const Ctor = schema.class as new () => Record<string, unknown>;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* `Migration20200101043200_via_ferrata_tiles_outlive_a_restart`: the table
|
||||
* that keeps the via ferrata grid cells exists only from this step on, keyed
|
||||
* by the cell, with every column required.
|
||||
*/
|
||||
import { createMigrationOrm, migrateTo, pendingNames, rawExec, rawQuery } from '../../helpers/migration-step';
|
||||
import type { MikroORM } from '@mikro-orm/sqlite';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const TARGET = 'Migration20200101043200_via_ferrata_tiles_outlive_a_restart';
|
||||
|
||||
async function ormBeforeTarget(): Promise<MikroORM> {
|
||||
const orm = await createMigrationOrm();
|
||||
const names = await pendingNames(orm);
|
||||
const index = names.indexOf(TARGET);
|
||||
expect(index).toBeGreaterThan(0);
|
||||
await migrateTo(orm, names[index - 1]);
|
||||
return orm;
|
||||
}
|
||||
|
||||
const tables = (orm: MikroORM) =>
|
||||
rawQuery<{ name: string }>(orm, "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'ferrata_tiles'");
|
||||
|
||||
describe('Via ferrata tiles migration', () => {
|
||||
it('FERTILEMIG-001: creates ferrata_tiles with a required key, payload and fetch time', async () => {
|
||||
const orm = await ormBeforeTarget();
|
||||
try {
|
||||
expect(await tables(orm)).toEqual([]);
|
||||
|
||||
await migrateTo(orm, TARGET);
|
||||
|
||||
expect(await tables(orm)).toEqual([{ name: 'ferrata_tiles' }]);
|
||||
const columns = await rawQuery<{ name: string; type: string; notnull: number; pk: number }>(
|
||||
orm,
|
||||
'SELECT name, type, "notnull", pk FROM pragma_table_info(\'ferrata_tiles\') ORDER BY cid',
|
||||
);
|
||||
expect(columns).toEqual([
|
||||
{ name: 'tile_key', type: 'TEXT', notnull: 1, pk: 1 },
|
||||
{ name: 'payload', type: 'TEXT', notnull: 1, pk: 0 },
|
||||
{ name: 'fetched_at', type: 'INTEGER', notnull: 1, pk: 0 },
|
||||
]);
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
it('FERTILEMIG-002: holds one row per cell and refuses a row without a payload', async () => {
|
||||
const orm = await ormBeforeTarget();
|
||||
try {
|
||||
await migrateTo(orm, TARGET);
|
||||
|
||||
await rawExec(orm, "INSERT INTO ferrata_tiles (tile_key, payload, fetched_at) VALUES ('942,224', '{}', 1)");
|
||||
await expect(
|
||||
rawExec(orm, "INSERT INTO ferrata_tiles (tile_key, payload, fetched_at) VALUES ('942,224', '{}', 2)"),
|
||||
).rejects.toThrow(/UNIQUE|PRIMARY KEY/);
|
||||
await expect(
|
||||
rawExec(orm, "INSERT INTO ferrata_tiles (tile_key, payload, fetched_at) VALUES ('942,225', NULL, 2)"),
|
||||
).rejects.toThrow(/NOT NULL/);
|
||||
await expect(
|
||||
rawExec(orm, "INSERT INTO ferrata_tiles (tile_key, payload, fetched_at) VALUES (NULL, '{}', 2)"),
|
||||
).rejects.toThrow(/NOT NULL/);
|
||||
} finally {
|
||||
await orm.close(true);
|
||||
}
|
||||
}, 30000);
|
||||
});
|
||||
@@ -14,6 +14,7 @@ describe('known migrations', () => {
|
||||
expect(known.has('Migration20200101042900_tour_planned_total_duration')).toBe(true);
|
||||
expect(known.has('Migration20200101043000_tour_break_additional_duration')).toBe(true);
|
||||
expect(known.has('Migration20200101043100_an_imported_tour_has_no_hiking_difficulty')).toBe(true);
|
||||
expect(known.has('Migration20200101043200_via_ferrata_tiles_outlive_a_restart')).toBe(true);
|
||||
expect(known.size).toBe(fs.readdirSync(path.join(__dirname, '../../../src/db/migrations')).length);
|
||||
});
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ const UNNUMBERED = [
|
||||
'Migration20200101042900_tour_planned_total_duration',
|
||||
'Migration20200101043000_tour_break_additional_duration',
|
||||
'Migration20200101043100_an_imported_tour_has_no_hiking_difficulty',
|
||||
'Migration20200101043200_via_ferrata_tiles_outlive_a_restart',
|
||||
];
|
||||
// The last step the positional runner ever had; planLegacyBaseline refuses a
|
||||
// schema_version past it.
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* FerrataTilesRepository: the cells of the via ferrata grid that
|
||||
* FerratasService keeps between restarts, read by key and written as a batch.
|
||||
*/
|
||||
import { FerrataTiles } from '../../../../src/db/entities/FerrataTiles.entity';
|
||||
import type { FerrataTilesRepository } from '../../../../src/db/repositories/FerrataTiles.repository';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { findRow, findRows, insertRows } from '../../../helpers/factories/rows';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const testDb = createSnapshotTestDb();
|
||||
let t: TestOrm;
|
||||
let tiles: FerrataTilesRepository;
|
||||
|
||||
beforeAll(async () => {
|
||||
t = await createTestOrm(testDb);
|
||||
tiles = t.repo(FerrataTiles);
|
||||
});
|
||||
beforeEach(() => {
|
||||
resetTestDb(testDb);
|
||||
t.clear();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
testDb.close();
|
||||
});
|
||||
|
||||
const empty = '{"ferratas":[],"truncated":false}';
|
||||
|
||||
describe('FerrataTilesRepository.findByKeys', () => {
|
||||
it('FERTILEREPO-001: answers the stored cells among the keys and leaves the others out', async () => {
|
||||
await insertRows(t, FerrataTiles, [
|
||||
{ tile_key: '942,224', payload: empty, fetched_at: 100 },
|
||||
{ tile_key: '942,225', payload: '{"ferratas":[],"truncated":true}', fetched_at: 200 },
|
||||
{ tile_key: '950,230', payload: empty, fetched_at: 300 },
|
||||
]);
|
||||
|
||||
const rows = await tiles.findByKeys(['942,224', '942,225', '943,224']);
|
||||
|
||||
expect(rows.sort((a, b) => a.tile_key.localeCompare(b.tile_key))).toEqual([
|
||||
{ tile_key: '942,224', payload: empty, fetched_at: 100 },
|
||||
{ tile_key: '942,225', payload: '{"ferratas":[],"truncated":true}', fetched_at: 200 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('FERTILEREPO-002: no keys asks for nothing', async () => {
|
||||
await insertRows(t, FerrataTiles, [{ tile_key: '942,224', payload: empty, fetched_at: 100 }]);
|
||||
expect(await tiles.findByKeys([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('FERTILEREPO-003: a cell on the other side of the globe keeps its negative key', async () => {
|
||||
await insertRows(t, FerrataTiles, [{ tile_key: '-1800,-3600', payload: empty, fetched_at: 5 }]);
|
||||
expect(await tiles.findByKeys(['-1800,-3600'])).toEqual([
|
||||
{ tile_key: '-1800,-3600', payload: empty, fetched_at: 5 },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FerrataTilesRepository.upsertTiles', () => {
|
||||
it('FERTILEREPO-004: writes a row per new cell', async () => {
|
||||
await tiles.upsertTiles([
|
||||
{ tile_key: '942,224', payload: empty, fetched_at: 100 },
|
||||
{ tile_key: '942,225', payload: empty, fetched_at: 100 },
|
||||
]);
|
||||
|
||||
const rows = await findRows(t, FerrataTiles, {}, { tile_key: 'asc' });
|
||||
expect(rows).toEqual([
|
||||
{ tile_key: '942,224', payload: empty, fetched_at: 100 },
|
||||
{ tile_key: '942,225', payload: empty, fetched_at: 100 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('FERTILEREPO-005: a cell stored before gets the new payload and fetch time, still as one row', async () => {
|
||||
await insertRows(t, FerrataTiles, [{ tile_key: '942,224', payload: empty, fetched_at: 100 }]);
|
||||
const refreshed = '{"ferratas":[],"truncated":true}';
|
||||
|
||||
await tiles.upsertTiles([
|
||||
{ tile_key: '942,224', payload: refreshed, fetched_at: 900 },
|
||||
{ tile_key: '942,225', payload: empty, fetched_at: 900 },
|
||||
]);
|
||||
|
||||
expect(await findRows(t, FerrataTiles, { tile_key: '942,224' })).toHaveLength(1);
|
||||
expect(await findRow(t, FerrataTiles, { tile_key: '942,224' })).toEqual({
|
||||
tile_key: '942,224',
|
||||
payload: refreshed,
|
||||
fetched_at: 900,
|
||||
});
|
||||
expect(await findRow(t, FerrataTiles, { tile_key: '942,225' })).toMatchObject({ fetched_at: 900 });
|
||||
});
|
||||
|
||||
it('FERTILEREPO-006: an empty batch writes nothing', async () => {
|
||||
await tiles.upsertTiles([]);
|
||||
expect(await findRows(t, FerrataTiles)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,16 @@
|
||||
import {
|
||||
FERRATA_QUERY_TIMEOUT_S,
|
||||
buildFerrataQuery,
|
||||
ferrataCacheKey,
|
||||
ferrataTileBlocks,
|
||||
ferrataTileKeysFor,
|
||||
ferrataTilesBbox,
|
||||
overpassFerrataAnswerSchema,
|
||||
parseFerrataGrade,
|
||||
snapFerrataBbox,
|
||||
splitFerratasByTile,
|
||||
toFerratas,
|
||||
} from '../../../src/nest/ferratas/ferratas.helpers';
|
||||
import { FERRATAS_RESULT_CAP } from '@trek/shared';
|
||||
import { FERRATAS_RESULT_CAP, type Ferrata } from '@trek/shared';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
@@ -48,7 +51,7 @@ describe('parseFerrataGrade', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapFerrataBbox and ferrataCacheKey', () => {
|
||||
describe('snapFerrataBbox', () => {
|
||||
it('grows the box outwards to the 0.05 degree grid', () => {
|
||||
expect(snapFerrataBbox({ south: 47.12, west: 11.21, north: 47.38, east: 11.57 })).toEqual({
|
||||
south: 47.1,
|
||||
@@ -71,12 +74,119 @@ describe('snapFerrataBbox and ferrataCacheKey', () => {
|
||||
east: -179.9,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('gives two nearby requests the same key', () => {
|
||||
const a = ferrataCacheKey(snapFerrataBbox({ south: 47.11, west: 11.21, north: 47.39, east: 11.59 }));
|
||||
const b = ferrataCacheKey(snapFerrataBbox({ south: 47.12, west: 11.22, north: 47.38, east: 11.58 }));
|
||||
expect(a).toBe(b);
|
||||
expect(a).toBe('47.1,11.2,47.4,11.6');
|
||||
describe('ferrataTileKeysFor', () => {
|
||||
it('names every whole cell the snapped box covers, south to north and west to east', () => {
|
||||
expect(ferrataTileKeysFor({ south: 47.12, west: 11.21, north: 47.18, east: 11.29 }, 200)).toEqual([
|
||||
'942,224',
|
||||
'942,225',
|
||||
'943,224',
|
||||
'943,225',
|
||||
]);
|
||||
});
|
||||
|
||||
it('gives two nearby requests the same cells', () => {
|
||||
const a = ferrataTileKeysFor({ south: 47.11, west: 11.21, north: 47.39, east: 11.59 }, 200);
|
||||
const b = ferrataTileKeysFor({ south: 47.12, west: 11.22, north: 47.38, east: 11.58 }, 200);
|
||||
expect(a).toEqual(b);
|
||||
expect(a).toHaveLength(6 * 8);
|
||||
});
|
||||
|
||||
it('a box on the grid covers exactly its own cells', () => {
|
||||
expect(ferrataTileKeysFor({ south: 47.1, west: 11.2, north: 47.15, east: 11.25 }, 200)).toEqual(['942,224']);
|
||||
});
|
||||
|
||||
it('keys cells south of the equator and west of Greenwich by negative indices', () => {
|
||||
expect(ferrataTileKeysFor({ south: -0.04, west: -0.04, north: 0.04, east: 0.04 }, 200)).toEqual([
|
||||
'-1,-1',
|
||||
'-1,0',
|
||||
'0,-1',
|
||||
'0,0',
|
||||
]);
|
||||
});
|
||||
|
||||
it('answers null for a box over the cell cap instead of walking it', () => {
|
||||
expect(ferrataTileKeysFor({ south: 47, west: 11, north: 48, east: 12 }, 200)).toBeNull();
|
||||
expect(ferrataTileKeysFor({ south: 47, west: 11, north: 47.5, east: 11.5 }, 100)).toHaveLength(100);
|
||||
expect(ferrataTileKeysFor({ south: 47, west: 11, north: 47.5, east: 11.5 }, 99)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ferrataTileBlocks', () => {
|
||||
it('groups cells into the grid-aligned blocks of two by two that one query covers', () => {
|
||||
expect(ferrataTileBlocks(['942,224', '942,225', '943,224', '944,224', '942,226', '-1,-1', '-2,-1'])).toEqual([
|
||||
['942,224', '942,225', '943,224'],
|
||||
['944,224'],
|
||||
['942,226'],
|
||||
['-1,-1', '-2,-1'],
|
||||
]);
|
||||
expect(ferrataTileBlocks([])).toEqual([]);
|
||||
});
|
||||
|
||||
it('refuses a key that is not a cell', () => {
|
||||
expect(() => ferrataTileBlocks(['nope'])).toThrow(/tile key/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ferrataTilesBbox', () => {
|
||||
it('bounds one cell by its own edges', () => {
|
||||
expect(ferrataTilesBbox(['942,224'])).toEqual({ south: 47.1, west: 11.2, north: 47.15, east: 11.25 });
|
||||
});
|
||||
|
||||
it('bounds scattered cells by one box around all of them', () => {
|
||||
expect(ferrataTilesBbox(['942,230', '945,224', '-1,-1'])).toEqual({
|
||||
south: -0.05,
|
||||
west: -0.05,
|
||||
north: 47.3,
|
||||
east: 11.55,
|
||||
});
|
||||
});
|
||||
|
||||
it('refuses no cells and a key that is not a cell', () => {
|
||||
expect(() => ferrataTilesBbox([])).toThrow();
|
||||
expect(() => ferrataTilesBbox(['947.1,11'])).toThrow(/tile key/);
|
||||
expect(() => ferrataTilesBbox(['1,2,3'])).toThrow(/tile key/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitFerratasByTile', () => {
|
||||
const ferrata = (id: string, geometry: [number, number][]): Ferrata => ({
|
||||
id,
|
||||
name: null,
|
||||
grade: { raw: null, level: null, plus: null },
|
||||
routeName: null,
|
||||
geometry,
|
||||
osmUrl: `https://www.openstreetmap.org/${id}`,
|
||||
});
|
||||
|
||||
it('hands a ferrata to every asked cell a point of its line lies in', () => {
|
||||
const crossing = ferrata('way/1', [
|
||||
[47.12, 11.21],
|
||||
[47.17, 11.21],
|
||||
]);
|
||||
const byTile = splitFerratasByTile([crossing], new Set(['942,224', '943,224', '942,225']));
|
||||
expect(byTile.get('942,224')).toEqual([crossing]);
|
||||
expect(byTile.get('943,224')).toEqual([crossing]);
|
||||
expect(byTile.get('942,225')).toEqual([]);
|
||||
});
|
||||
|
||||
it('counts a point on a cell edge for the cells on both sides', () => {
|
||||
const onCorner = ferrata('way/2', [
|
||||
[47.15, 11.25],
|
||||
[47.15, 11.25],
|
||||
]);
|
||||
const byTile = splitFerratasByTile([onCorner], new Set(['942,224', '942,225', '943,224', '943,225']));
|
||||
expect([...byTile.values()].every((list) => list.length === 1)).toBe(true);
|
||||
});
|
||||
|
||||
it('gives every asked cell a list, empty where nothing lies, and leaves cells it was not asked about out', () => {
|
||||
const elsewhere = ferrata('way/3', [
|
||||
[46.51, 12.06],
|
||||
[46.52, 12.07],
|
||||
]);
|
||||
const byTile = splitFerratasByTile([elsewhere], new Set(['942,224']));
|
||||
expect([...byTile.entries()]).toEqual([['942,224', []]]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,117 +1,454 @@
|
||||
/**
|
||||
* FerratasService over a real snapshot db: the box resolved into grid cells,
|
||||
* the cells served from `ferrata_tiles` (and the memory in front of it), and
|
||||
* Overpass, faked, asked once per 0.1 degree block for the cells never fetched.
|
||||
*/
|
||||
import { FerrataTiles } from '../../../src/db/entities/FerrataTiles.entity';
|
||||
import { FerrataTilesRepository } from '../../../src/db/repositories/FerrataTiles.repository';
|
||||
import { DomainError } from '../../../src/nest/common/domain-error';
|
||||
import { FerratasOverpassClient } from '../../../src/nest/ferratas/ferratas-overpass.client';
|
||||
import { FerratasModule } from '../../../src/nest/ferratas/ferratas.module';
|
||||
import {
|
||||
FERRATA_CACHE_MAX,
|
||||
FERRATA_CACHE_TTL_MS,
|
||||
FERRATA_TILE_STALE_MS,
|
||||
FERRATAS_UNAVAILABLE_MESSAGE,
|
||||
FerratasService,
|
||||
} from '../../../src/nest/ferratas/ferratas.service';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { FERRATAS_ATTRIBUTION, FERRATAS_UNAVAILABLE_CODE, ferratasResponseSchema } from '@trek/shared';
|
||||
import { createSnapshotTestDb } from '../../helpers/db-mock';
|
||||
import { countRows, deleteRows, findRow, findRows, insertRows } from '../../helpers/factories/rows';
|
||||
import { resetTestDb } from '../../helpers/test-db';
|
||||
import { createTestModule, type TestModule } from '../../helpers/test-module';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import {
|
||||
FERRATA_BBOX_TOO_LARGE_MESSAGE,
|
||||
FERRATAS_ATTRIBUTION,
|
||||
FERRATAS_RESULT_CAP,
|
||||
FERRATAS_UNAVAILABLE_CODE,
|
||||
ferratasResponseSchema,
|
||||
type Ferrata,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const bbox = { south: 47.12, west: 11.21, north: 47.38, east: 11.57 };
|
||||
const db = createSnapshotTestDb();
|
||||
const NOW = 1_800_000_000_000;
|
||||
/** What `fetched_at` holds for NOW: the table keeps seconds. */
|
||||
const NOW_S = NOW / 1000;
|
||||
const seconds = (ms: number) => Math.floor(ms / 1000);
|
||||
|
||||
/** 47.1 to 47.2 north, 11.2 to 11.3 east once snapped: 2 by 2 cells, one block of one query. */
|
||||
const bbox = { south: 47.12, west: 11.21, north: 47.18, east: 11.29 };
|
||||
/** 47.1 to 47.4 north, 11.2 to 11.6 east once snapped: 6 by 8 cells, 3 by 4 blocks. */
|
||||
const wide = { south: 47.12, west: 11.21, north: 47.38, east: 11.57 };
|
||||
const element = {
|
||||
type: 'way',
|
||||
id: 42,
|
||||
tags: { highway: 'via_ferrata', name: 'Klettersteig', via_ferrata_scale: '2+' },
|
||||
geometry: [
|
||||
{ lat: 47.2, lon: 11.3 },
|
||||
{ lat: 47.21, lon: 11.31 },
|
||||
{ lat: 47.15, lon: 11.25 },
|
||||
{ lat: 47.16, lon: 11.26 },
|
||||
],
|
||||
};
|
||||
|
||||
const stored = (id: string, geometry: [number, number][]): Ferrata => ({
|
||||
id,
|
||||
name: 'Stored',
|
||||
grade: { raw: null, level: null, plus: null },
|
||||
routeName: null,
|
||||
geometry,
|
||||
osmUrl: `https://www.openstreetmap.org/${id}`,
|
||||
});
|
||||
const payload = (ferratas: Ferrata[], truncated = false) => JSON.stringify({ ferratas, truncated });
|
||||
|
||||
const query = vi.fn<(q: string) => Promise<unknown[]>>();
|
||||
let t: TestModule | undefined;
|
||||
let service: FerratasService;
|
||||
const orm = () => {
|
||||
if (!t) throw new Error('No container booted');
|
||||
return t.get(MikroORM);
|
||||
};
|
||||
|
||||
/** A fresh container on the same db: an empty memory, what a restart looks like. */
|
||||
async function boot(): Promise<void> {
|
||||
await t?.close();
|
||||
t = await createTestModule({
|
||||
db,
|
||||
imports: [FerratasModule],
|
||||
overrides: [{ provide: FerratasOverpassClient, useValue: { query } }],
|
||||
});
|
||||
service = t.get(FerratasService);
|
||||
}
|
||||
|
||||
function deferred(): { promise: Promise<unknown[]>; resolve: (v: unknown[]) => void; reject: (e: Error) => void } {
|
||||
let resolve: (v: unknown[]) => void = () => {};
|
||||
let reject: (e: Error) => void = () => {};
|
||||
const promise = new Promise<unknown[]>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
resetTestDb(db);
|
||||
query.mockReset();
|
||||
query.mockResolvedValue([element]);
|
||||
const moduleRef = await Test.createTestingModule({
|
||||
providers: [FerratasService, { provide: FerratasOverpassClient, useValue: { query } }],
|
||||
}).compile();
|
||||
service = moduleRef.get(FerratasService);
|
||||
await boot();
|
||||
});
|
||||
|
||||
describe('FerratasService.list', () => {
|
||||
it('answers the ferratas with source and attribution, inside the response contract', async () => {
|
||||
const res = await service.list(bbox);
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await t?.close();
|
||||
db.close();
|
||||
});
|
||||
|
||||
describe('FerratasService.list: cells fetched for the first time', () => {
|
||||
it('FERSVC-001: answers the ferratas with source and attribution, inside the response contract', async () => {
|
||||
const res = await service.list(bbox, NOW);
|
||||
expect(res).toMatchObject({ truncated: false, source: 'openstreetmap', attribution: FERRATAS_ATTRIBUTION });
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(ferratasResponseSchema.safeParse(res).success).toBe(true);
|
||||
});
|
||||
|
||||
it('answers only the ferratas that reach into the asked box, not the whole snapped box', async () => {
|
||||
it('FERSVC-002: asks Overpass once for a box of one block', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(query.mock.calls[0]?.[0]).toContain('(47.1,11.2,47.2,11.3)');
|
||||
});
|
||||
|
||||
it('FERSVC-023: asks a wider box block by block, never more than 0.1 degree a side, at most two at once', async () => {
|
||||
const gates = Array.from({ length: 12 }, () => deferred());
|
||||
let next = 0;
|
||||
query.mockImplementation(() => (gates[next++] ?? deferred()).promise);
|
||||
|
||||
const answer = service.list(wide, NOW);
|
||||
await vi.waitFor(() => expect(query).toHaveBeenCalledTimes(2));
|
||||
gates[0]?.resolve([element]);
|
||||
await vi.waitFor(() => expect(query).toHaveBeenCalledTimes(3));
|
||||
for (const gate of gates) gate.resolve([element]);
|
||||
const res = await answer;
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(12);
|
||||
for (const [text] of query.mock.calls) {
|
||||
const [south, west, north, east] = (/\(([\d.,-]+)\)/.exec(text)?.[1] ?? '').split(',').map(Number);
|
||||
expect((north ?? 0) - (south ?? 0)).toBeLessThanOrEqual(0.1 + 1e-9);
|
||||
expect((east ?? 0) - (west ?? 0)).toBeLessThanOrEqual(0.1 + 1e-9);
|
||||
}
|
||||
expect(await countRows(orm(), FerrataTiles)).toBe(6 * 8);
|
||||
});
|
||||
|
||||
it('FERSVC-003: stores every cell, each with the ferratas whose line touches it, at the time in seconds', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
|
||||
expect(await countRows(orm(), FerrataTiles)).toBe(2 * 2);
|
||||
// 47.15 and 11.25 lie on cell edges, so way 42 belongs to the four cells around that corner.
|
||||
const holding = (await findRows(orm(), FerrataTiles, {}, { tile_key: 'asc' }))
|
||||
.filter((row) => row.payload.includes('way/42'))
|
||||
.map((row) => row.tile_key);
|
||||
expect(holding).toEqual(['942,224', '942,225', '943,224', '943,225']);
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toMatchObject({ fetched_at: NOW_S });
|
||||
|
||||
// The next block north holds none of it: its cells are stored empty.
|
||||
await service.list({ south: 47.21, west: 11.21, north: 47.29, east: 11.29 }, NOW);
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '944,224' })).toEqual({
|
||||
tile_key: '944,224',
|
||||
payload: payload([]),
|
||||
fetched_at: NOW_S,
|
||||
});
|
||||
});
|
||||
|
||||
it('FERSVC-004: answers a ferrata crossing several cells once', async () => {
|
||||
const res = await service.list(bbox, NOW);
|
||||
expect(res.ferratas).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('FERSVC-005: answers only the ferratas that reach into the asked box, not the whole cells', async () => {
|
||||
const outside = {
|
||||
...element,
|
||||
id: 43,
|
||||
geometry: [
|
||||
{ lat: 47.1, lon: 11.3 },
|
||||
{ lat: 47.11, lon: 11.31 },
|
||||
{ lat: 47.1, lon: 11.22 },
|
||||
{ lat: 47.11, lon: 11.23 },
|
||||
],
|
||||
};
|
||||
query.mockResolvedValue([element, outside]);
|
||||
|
||||
const res = await service.list(bbox);
|
||||
const res = await service.list(bbox, NOW);
|
||||
|
||||
expect(res.ferratas.map((ferrata) => ferrata.id)).toEqual(['way/42']);
|
||||
// The cell still keeps it, for the next box that does reach that far south.
|
||||
expect((await findRow(orm(), FerrataTiles, { tile_key: '942,224' }))?.payload).toContain('way/43');
|
||||
});
|
||||
|
||||
it('asks Overpass for the box snapped to the grid', async () => {
|
||||
await service.list(bbox);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(query.mock.calls[0]?.[0]).toContain('(47.1,11.2,47.4,11.6)');
|
||||
});
|
||||
|
||||
it('serves a nearby request from the cache inside the TTL', async () => {
|
||||
await service.list(bbox, 1_000);
|
||||
await service.list({ south: 47.11, west: 11.22, north: 47.39, east: 11.59 }, 1_000 + FERRATA_CACHE_TTL_MS - 1);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('asks again once the entry is older than the TTL', async () => {
|
||||
await service.list(bbox, 1_000);
|
||||
await service.list(bbox, 1_000 + FERRATA_CACHE_TTL_MS);
|
||||
expect(query).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('shares one Overpass call between concurrent identical requests', async () => {
|
||||
let release: (v: unknown[]) => void = () => {};
|
||||
query.mockReturnValueOnce(new Promise((resolve) => (release = resolve)));
|
||||
const both = Promise.all([service.list(bbox), service.list(bbox)]);
|
||||
release([element]);
|
||||
it('FERSVC-006: shares one Overpass call between concurrent identical requests', async () => {
|
||||
const pending = deferred();
|
||||
query.mockReturnValueOnce(pending.promise);
|
||||
const both = Promise.all([service.list(bbox, NOW), service.list(bbox, NOW)]);
|
||||
await vi.waitFor(() => expect(query).toHaveBeenCalledTimes(1));
|
||||
pending.resolve([element]);
|
||||
const [a, b] = await both;
|
||||
expect(a).toBe(b);
|
||||
expect(a).toEqual(b);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('turns an Overpass failure into a 502 with a code, and does not cache it', async () => {
|
||||
it('FERSVC-007: asks a cut block again cell by cell, and stores a cell still cut as old so a restart asks again', async () => {
|
||||
const many = Array.from({ length: FERRATAS_RESULT_CAP + 1 }, (_, i) => ({ ...element, id: 1000 + i }));
|
||||
query.mockResolvedValue(many);
|
||||
|
||||
const res = await service.list(bbox, NOW);
|
||||
|
||||
expect(res.truncated).toBe(true);
|
||||
expect(res.ferratas).toHaveLength(FERRATAS_RESULT_CAP);
|
||||
expect(ferratasResponseSchema.safeParse(res).success).toBe(true);
|
||||
// The block, then each of its four cells on its own.
|
||||
expect(query).toHaveBeenCalledTimes(1 + 4);
|
||||
expect(query.mock.calls[1]?.[0]).toContain('(47.1,11.2,47.15,11.25)');
|
||||
const row = await findRow(orm(), FerrataTiles, { tile_key: '942,224' });
|
||||
expect(row?.fetched_at).toBe(0);
|
||||
expect(row?.payload).toContain('"truncated":true');
|
||||
|
||||
// This process keeps the cut cells; a restart serves them and asks again in the background.
|
||||
expect((await service.list(bbox, NOW + 1)).truncated).toBe(true);
|
||||
expect(query).toHaveBeenCalledTimes(5);
|
||||
await boot();
|
||||
query.mockReset();
|
||||
query.mockResolvedValue([element]);
|
||||
expect((await service.list(bbox, NOW)).truncated).toBe(true);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(async () =>
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toMatchObject({ fetched_at: NOW_S }),
|
||||
);
|
||||
expect((await service.list(bbox, NOW + 1)).truncated).toBe(false);
|
||||
});
|
||||
|
||||
it('FERSVC-024: a block cut at the cap whose cells each fit is stored whole, and not marked cut', async () => {
|
||||
const many = Array.from({ length: FERRATAS_RESULT_CAP + 1 }, (_, i) => ({ ...element, id: 1000 + i }));
|
||||
query.mockResolvedValueOnce(many).mockResolvedValue([element]);
|
||||
|
||||
const res = await service.list(bbox, NOW);
|
||||
|
||||
expect(res.truncated).toBe(false);
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(5);
|
||||
const rows = await findRows(orm(), FerrataTiles, {}, { tile_key: 'asc' });
|
||||
expect(rows.map((r) => [r.tile_key, r.fetched_at, r.payload.includes('"truncated":false')])).toEqual([
|
||||
['942,224', NOW_S, true],
|
||||
['942,225', NOW_S, true],
|
||||
['943,224', NOW_S, true],
|
||||
['943,225', NOW_S, true],
|
||||
]);
|
||||
});
|
||||
|
||||
it('FERSVC-008: refuses a box over the cell cap before asking anyone', async () => {
|
||||
const err = await service.list({ south: 47, west: 11, north: 48, east: 12 }, NOW).catch((e: unknown) => e);
|
||||
expect(err).toBeInstanceOf(DomainError);
|
||||
expect((err as DomainError).getStatus()).toBe(400);
|
||||
expect((err as DomainError).getResponse()).toEqual({ error: FERRATA_BBOX_TOO_LARGE_MESSAGE });
|
||||
expect(query).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('FerratasService.list: cells already stored', () => {
|
||||
it('FERSVC-009: serves a nearby request from the cells, without Overpass', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
const res = await service.list({ south: 47.11, west: 11.22, north: 47.19, east: 11.28 }, NOW + 1);
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('FERSVC-010: serves the cells from the table after a restart', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
await boot();
|
||||
|
||||
const res = await service.list(bbox, NOW + 1);
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('FERSVC-011: serves the cells from memory without reading the table again', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
await deleteRows(orm(), FerrataTiles);
|
||||
|
||||
const res = await service.list(bbox, NOW + 1);
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('FERSVC-012: asks Overpass only for the cells not stored yet', async () => {
|
||||
await service.list({ south: 47.1, west: 11.2, north: 47.2, east: 11.3 }, NOW);
|
||||
query.mockClear();
|
||||
|
||||
await service.list({ south: 47.1, west: 11.2, north: 47.3, east: 11.3 }, NOW);
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(query.mock.calls[0]?.[0]).toContain('(47.2,11.2,47.3,11.3)');
|
||||
expect(await countRows(orm(), FerrataTiles)).toBe(4 * 2);
|
||||
});
|
||||
|
||||
it('FERSVC-013: reads a stored cell that does not decode as never fetched, and fetches it again', async () => {
|
||||
await insertRows(orm(), FerrataTiles, [{ tile_key: '942,224', payload: '{"ferratas":"nope"}', fetched_at: NOW_S }]);
|
||||
query.mockResolvedValue([]);
|
||||
|
||||
await service.list({ south: 47.1, west: 11.2, north: 47.15, east: 11.25 }, NOW + 1000);
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toEqual({
|
||||
tile_key: '942,224',
|
||||
payload: payload([]),
|
||||
fetched_at: NOW_S + 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('FerratasService.list: old cells', () => {
|
||||
const cell = { south: 47.1, west: 11.2, north: 47.15, east: 11.25 };
|
||||
const old = stored('way/7', [
|
||||
[47.11, 11.21],
|
||||
[47.12, 11.22],
|
||||
]);
|
||||
const seedOld = () =>
|
||||
insertRows(orm(), FerrataTiles, [
|
||||
{ tile_key: '942,224', payload: payload([old]), fetched_at: seconds(NOW - FERRATA_TILE_STALE_MS) },
|
||||
]);
|
||||
|
||||
it('FERSVC-014: serves an old cell at once and refreshes it in the background', async () => {
|
||||
await seedOld();
|
||||
const pending = deferred();
|
||||
query.mockReturnValueOnce(pending.promise);
|
||||
|
||||
const res = await service.list(cell, NOW);
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/7']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
expect(query.mock.calls[0]?.[0]).toContain('(47.1,11.2,47.15,11.25)');
|
||||
|
||||
pending.resolve([]);
|
||||
await vi.waitFor(async () =>
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toEqual({
|
||||
tile_key: '942,224',
|
||||
payload: payload([]),
|
||||
fetched_at: NOW_S,
|
||||
}),
|
||||
);
|
||||
expect((await service.list(cell, NOW + 1)).ferratas).toEqual([]);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('FERSVC-015: a cell just under the age limit is not refreshed', async () => {
|
||||
await insertRows(orm(), FerrataTiles, [
|
||||
{ tile_key: '942,224', payload: payload([old]), fetched_at: seconds(NOW - FERRATA_TILE_STALE_MS) + 1 },
|
||||
]);
|
||||
await service.list(cell, NOW);
|
||||
expect(query).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('FERSVC-016: starts one refresh for an old cell, however many requests see it', async () => {
|
||||
await seedOld();
|
||||
const pending = deferred();
|
||||
query.mockReturnValueOnce(pending.promise);
|
||||
|
||||
await service.list(cell, NOW);
|
||||
await service.list(cell, NOW);
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
pending.resolve([]);
|
||||
await vi.waitFor(async () =>
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toMatchObject({ fetched_at: NOW_S }),
|
||||
);
|
||||
});
|
||||
|
||||
it('FERSVC-017: a failed refresh keeps the old cell, never reaches the client, and is tried again later', async () => {
|
||||
await seedOld();
|
||||
query.mockRejectedValueOnce(new Error('Could not reach any Overpass endpoint'));
|
||||
const err = await service.list(bbox).catch((e: unknown) => e);
|
||||
|
||||
const res = await service.list(cell, NOW);
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/7']);
|
||||
// Let the rejected refresh settle; an unhandled rejection would fail the run.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toMatchObject({
|
||||
fetched_at: seconds(NOW - FERRATA_TILE_STALE_MS),
|
||||
});
|
||||
|
||||
query.mockResolvedValueOnce([]);
|
||||
expect((await service.list(cell, NOW + 1000)).ferratas.map((f) => f.id)).toEqual(['way/7']);
|
||||
expect(query).toHaveBeenCalledTimes(2);
|
||||
await vi.waitFor(async () =>
|
||||
expect(await findRow(orm(), FerrataTiles, { tile_key: '942,224' })).toMatchObject({ fetched_at: NOW_S + 1 }),
|
||||
);
|
||||
});
|
||||
|
||||
it('FERSVC-018: an old cell is refreshed in a query of its own, so its failure cannot fail the new cells', async () => {
|
||||
await seedOld();
|
||||
query.mockImplementation((text) =>
|
||||
text.includes('(47.1,11.2,47.15,11.25)') ? Promise.reject(new Error('timeout')) : Promise.resolve([]),
|
||||
);
|
||||
|
||||
const res = await service.list({ south: 47.1, west: 11.2, north: 47.2, east: 11.25 }, NOW);
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(2);
|
||||
expect(query.mock.calls.map(([text]) => text.match(/\(([\d.,]+)\)/)?.[1]).sort()).toEqual([
|
||||
'47.1,11.2,47.15,11.25',
|
||||
'47.15,11.2,47.2,11.25',
|
||||
]);
|
||||
// The old cell is served as it was; the new one came back on its own.
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/7']);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(await findRows(orm(), FerrataTiles, {}, { tile_key: 'asc' })).toEqual([
|
||||
{ tile_key: '942,224', payload: payload([old]), fetched_at: seconds(NOW - FERRATA_TILE_STALE_MS) },
|
||||
{ tile_key: '943,224', payload: payload([]), fetched_at: NOW_S },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('FerratasService.list: Overpass failing', () => {
|
||||
it('FERSVC-019: turns a failure for a cell never fetched into a 502 with a code, and stores nothing', async () => {
|
||||
query.mockRejectedValueOnce(new Error('Could not reach any Overpass endpoint'));
|
||||
const err = await service.list(bbox, NOW).catch((e: unknown) => e);
|
||||
expect(err).toBeInstanceOf(DomainError);
|
||||
const domain = err as DomainError;
|
||||
expect(domain.getStatus()).toBe(502);
|
||||
expect(domain.code).toBe(FERRATAS_UNAVAILABLE_CODE);
|
||||
expect(domain.getResponse()).toEqual({ error: FERRATAS_UNAVAILABLE_MESSAGE, code: FERRATAS_UNAVAILABLE_CODE });
|
||||
expect(await countRows(orm(), FerrataTiles)).toBe(0);
|
||||
|
||||
const retry = await service.list(bbox);
|
||||
const retry = await service.list(bbox, NOW);
|
||||
expect(retry.ferratas).toHaveLength(1);
|
||||
expect(query).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('evicts the oldest box once the cache is full', async () => {
|
||||
const boxAt = (i: number) => ({ south: 10 + i * 0.1, west: 10, north: 10.04 + i * 0.1, east: 10.04 });
|
||||
// One after another, so the insertion order is the call order.
|
||||
await Array.from({ length: FERRATA_CACHE_MAX + 1 }, (_, i) => i).reduce<Promise<unknown>>(async (previous, i) => {
|
||||
await previous;
|
||||
return service.list(boxAt(i), 1_000);
|
||||
}, Promise.resolve());
|
||||
expect(query).toHaveBeenCalledTimes(FERRATA_CACHE_MAX + 1);
|
||||
it('FERSVC-020: fails every request that joined the failed batch', async () => {
|
||||
const pending = deferred();
|
||||
query.mockReturnValueOnce(pending.promise);
|
||||
const both = Promise.allSettled([service.list(bbox, NOW), service.list(bbox, NOW)]);
|
||||
await vi.waitFor(() => expect(query).toHaveBeenCalledTimes(1));
|
||||
pending.reject(new Error('down'));
|
||||
const results = await both;
|
||||
expect(results.map((r) => r.status)).toEqual(['rejected', 'rejected']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
await service.list(boxAt(FERRATA_CACHE_MAX), 1_000);
|
||||
expect(query).toHaveBeenCalledTimes(FERRATA_CACHE_MAX + 1);
|
||||
await service.list(boxAt(0), 1_000);
|
||||
expect(query).toHaveBeenCalledTimes(FERRATA_CACHE_MAX + 2);
|
||||
it('FERSVC-021: needs no Overpass at all while every cell is stored and fresh', async () => {
|
||||
await service.list(bbox, NOW);
|
||||
query.mockRejectedValue(new Error('down'));
|
||||
const res = await service.list(bbox, NOW + 1);
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('FERSVC-022: an answer the table refuses to store is still served, and kept in memory', async () => {
|
||||
vi.spyOn(FerrataTilesRepository.prototype, 'upsertTiles').mockRejectedValueOnce(new Error('disk full'));
|
||||
|
||||
const res = await service.list(bbox, NOW);
|
||||
|
||||
expect(res.ferratas.map((f) => f.id)).toEqual(['way/42']);
|
||||
expect(await countRows(orm(), FerrataTiles)).toBe(0);
|
||||
await service.list(bbox, NOW + 1);
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user