diff --git a/server/src/db/json-columns.ts b/server/src/db/json-columns.ts index bd7ef469c..fed3dbb6e 100644 --- a/server/src/db/json-columns.ts +++ b/server/src/db/json-columns.ts @@ -2,11 +2,15 @@ import type { JsonColumn } from '../utils/json-column'; import { FERRATAS_RESULT_CAP, ferrataSchema, + paddleGeometrySchema, paddleLockSchema, + paddleRegionBoundarySchema, tourFixedLegSchema, trackMarksSchema, type Ferrata, + type PaddleGeometry, type PaddleLock, + type PaddleRegionBoundary, type TourFixedLeg, type TrackMarks, } from '@trek/shared'; @@ -166,3 +170,24 @@ export const TOUR_TRACK_MARKS: JsonColumn = { schema: trackMarksSchema.nullable(), fallback: () => null, }; + +/** A paddle object's own geometry (PAD-05A). A broken row has none, and the object is left out rather than drawn anywhere. */ +export const PADDLE_FEATURE_GEOMETRY: JsonColumn = { + column: 'paddle_features.geometry', + schema: paddleGeometrySchema.nullable(), + fallback: () => null, +}; + +/** The OSM tags a paddle object was judged by. A broken row shows none. */ +export const PADDLE_FEATURE_TAGS: JsonColumn> = { + column: 'paddle_features.tags', + schema: z.record(z.string(), z.string()), + fallback: () => ({}), +}; + +/** The boundary of a region TREK reads paddle data for. A broken row has none, and the region counts by its box. */ +export const PADDLE_REGION_BOUNDARY: JsonColumn = { + column: 'paddle_regions.boundary', + schema: paddleRegionBoundarySchema.nullable(), + fallback: () => null, +}; diff --git a/server/src/db/migrations/Migration20200101044100_paddle_obstacles_from_openstreetmap.ts b/server/src/db/migrations/Migration20200101044100_paddle_obstacles_from_openstreetmap.ts index c1aeef6be..14cbff956 100644 --- a/server/src/db/migrations/Migration20200101044100_paddle_obstacles_from_openstreetmap.ts +++ b/server/src/db/migrations/Migration20200101044100_paddle_obstacles_from_openstreetmap.ts @@ -58,6 +58,8 @@ export class Migration20200101044100_paddle_obstacles_from_openstreetmap extends this.addSql( 'CREATE UNIQUE INDEX IF NOT EXISTS idx_paddle_features_region_ref ON paddle_features(region_id, osm_ref)', ); - this.addSql('CREATE INDEX IF NOT EXISTS idx_paddle_features_box ON paddle_features(min_lat, max_lat, min_lng, max_lng)'); + this.addSql( + 'CREATE INDEX IF NOT EXISTS idx_paddle_features_box ON paddle_features(min_lat, max_lat, min_lng, max_lng)', + ); } } diff --git a/server/src/db/repositories/PaddleFeatures.repository.ts b/server/src/db/repositories/PaddleFeatures.repository.ts index c237a79e3..5c483fb1b 100644 --- a/server/src/db/repositories/PaddleFeatures.repository.ts +++ b/server/src/db/repositories/PaddleFeatures.repository.ts @@ -69,12 +69,17 @@ export class PaddleFeaturesRepository extends TrekRepository { /** Drops a region's objects and writes the new read in their place; the caller holds the transaction. */ async replaceRegion(regionId: string, rows: readonly PaddleFeatureRow[]): Promise { await this.db().deleteFrom('paddle_features').where('region_id', '=', regionId).execute(); - for (let start = 0; start < rows.length; start += INSERT_CHUNK) { - await this.db() - .insertInto('paddle_features') - .values(rows.slice(start, start + INSERT_CHUNK).map((row) => ({ ...row, region_id: regionId }))) - .onConflict((oc) => oc.columns(['region_id', 'osm_ref']).doNothing()) - .execute(); - } + await this.insertFrom(regionId, rows, 0); + } + + /** The rows from `start` on, a chunk per statement, one after another on the transaction's connection. */ + private async insertFrom(regionId: string, rows: readonly PaddleFeatureRow[], start: number): Promise { + if (start >= rows.length) return; + await this.db() + .insertInto('paddle_features') + .values(rows.slice(start, start + INSERT_CHUNK).map((row) => ({ ...row, region_id: regionId }))) + .onConflict((oc) => oc.columns(['region_id', 'osm_ref']).doNothing()) + .execute(); + await this.insertFrom(regionId, rows, start + INSERT_CHUNK); } } diff --git a/server/src/db/repositories/PaddleRegions.repository.ts b/server/src/db/repositories/PaddleRegions.repository.ts index a80610f72..677a75fdb 100644 --- a/server/src/db/repositories/PaddleRegions.repository.ts +++ b/server/src/db/repositories/PaddleRegions.repository.ts @@ -1,7 +1,7 @@ import type { PaddleRegions } from '../entities/PaddleRegions.entity'; import type { DB } from '../kysely/db'; -import { TrekRepository } from './_shared/trek-repository'; import type { Box } from './PaddleFeatures.repository'; +import { TrekRepository } from './_shared/trek-repository'; export type PaddleRegionStatus = 'queued' | 'importing' | 'ready' | 'failed'; @@ -55,16 +55,21 @@ export class PaddleRegionsRepository extends TrekRepository { return await this.db().selectFrom('paddle_regions').select(COLUMNS).where('id', '=', id).executeTakeFirst(); } - /** The regions with data whose box reaches into `box`. */ - async readyInBox(box: Box): Promise { + /** + * The regions holding data whose box reaches into `box`, by name. A region + * read again, or whose last read failed, still holds the data of the read + * before, so it counts until it is removed. + */ + async withDataInBox(box: Box): Promise { return await this.db() .selectFrom('paddle_regions') .select(COLUMNS) - .where('status', '=', 'ready') + .where('imported_at', 'is not', null) .where('min_lat', '<=', box.north) .where('max_lat', '>=', box.south) .where('min_lng', '<=', box.east) .where('max_lng', '>=', box.west) + .orderBy('name', 'asc') .execute(); } @@ -78,7 +83,9 @@ export class PaddleRegionsRepository extends TrekRepository { .executeTakeFirst(); } - async insertRegion(row: Pick): Promise { + async insertRegion( + row: Pick, + ): Promise { await this.db() .insertInto('paddle_regions') .values({ ...row, status: 'queued', feature_count: 0 }) @@ -88,16 +95,29 @@ export class PaddleRegionsRepository extends TrekRepository { async updateRegion( id: string, - patch: Partial>, + patch: Partial< + Pick< + PaddleRegionRow, + | 'status' + | 'source_timestamp' + | 'imported_at' + | 'feature_count' + | 'error' + | 'min_lat' + | 'min_lng' + | 'max_lat' + | 'max_lng' + > + >, ): Promise { await this.db().updateTable('paddle_regions').set(patch).where('id', '=', id).execute(); } - /** Queues every region that has data again, for the monthly refresh; returns how many. */ + /** Queues every region that has data or a failed read again, for the monthly refresh; returns how many. */ async queueReady(): Promise { const result = await this.db() .updateTable('paddle_regions') - .set({ status: 'queued' }) + .set({ status: 'queued', error: null }) .where('status', 'in', ['ready', 'failed']) .executeTakeFirst(); return Number(result.numUpdatedRows ?? 0); diff --git a/server/src/nest/app.module.ts b/server/src/nest/app.module.ts index e60fec620..e54520052 100644 --- a/server/src/nest/app.module.ts +++ b/server/src/nest/app.module.ts @@ -43,7 +43,6 @@ import { DaysModule } from './days/days.module'; import { DocSyncModule } from './doc-sync/doc-sync.module'; import { FeedsModule } from './feeds/feeds.module'; import { FerratasModule } from './ferratas/ferratas.module'; -import { PaddleObstaclesModule } from './paddle-obstacles/paddle-obstacles.module'; import { FilesModule } from './files/files.module'; import { GeoModule } from './geo/geo.module'; import { GoogleQuotaModule } from './google-quota/google-quota.module'; @@ -62,6 +61,7 @@ import { NotificationsModule } from './notifications/notifications.module'; import { OauthModule } from './oauth/oauth.module'; import { OidcModule } from './oidc/oidc.module'; import { PackingModule } from './packing/packing.module'; +import { PaddleObstaclesModule } from './paddle-obstacles/paddle-obstacles.module'; import { PermissionsModule } from './permissions/permissions.module'; import { PhotosModule } from './photos/photos.module'; import { PlaceEnrichmentModule } from './place-enrichment/place-enrichment.module'; diff --git a/server/src/nest/paddle-obstacles/geofabrik.client.ts b/server/src/nest/paddle-obstacles/geofabrik.client.ts index 4396566df..6e12236c4 100644 --- a/server/src/nest/paddle-obstacles/geofabrik.client.ts +++ b/server/src/nest/paddle-obstacles/geofabrik.client.ts @@ -1,18 +1,20 @@ import { readCappedJson } from '../../utils/cappedFetch'; -import { regionBoundarySchema, type RegionBoundary } from './paddle-geo'; +import type { RegionBoundary } from './paddle-geo'; import { Injectable } from '@nestjs/common'; -import { z } from 'zod'; +import { paddleRegionBoundarySchema } from '@trek/shared'; import { createWriteStream } from 'node:fs'; import { unlink } from 'node:fs/promises'; import { Readable, Transform } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as WebReadableStream } from 'node:stream/web'; +import { z } from 'zod'; /** * The outbound half of PAD-05A: Geofabrik's list of regional extracts and the * download of one. Only Geofabrik's own host is ever asked, at addresses its - * list names, so an admin choosing a region can point TREK nowhere else. + * list names, so an admin choosing a region can point TREK nowhere else. A + * redirect is followed only to another address of Geofabrik's. */ export const GEOFABRIK_HOST = 'https://download.geofabrik.de/'; @@ -24,6 +26,7 @@ const INDEX_FRESH_MS = 24 * 60 * 60 * 1000; /** A whole country is a few gigabytes; a regional extract far less. More than this is refused. */ export const MAX_EXTRACT_BYTES = 4 * 1024 * 1024 * 1024; const DOWNLOAD_TIMEOUT_MS = 60 * 60 * 1000; +const MAX_REDIRECTS = 3; const indexSchema = z.object({ features: z.array( @@ -47,6 +50,32 @@ export interface GeofabrikRegion { boundary: RegionBoundary | null; } +/** An address on Geofabrik's own servers, over https. */ +function isGeofabrikUrl(url: URL): boolean { + return url.protocol === 'https:' && (url.hostname === 'geofabrik.de' || url.hostname.endsWith('.geofabrik.de')); +} + +/** An extract the list names: Geofabrik's host, a `-latest.osm.pbf` file. */ +export function isExtractUrl(url: string): boolean { + return url.startsWith(GEOFABRIK_HOST) && url.endsWith('-latest.osm.pbf'); +} + +/** + * `fetch` that follows a redirect only while it stays on Geofabrik's servers. + * A redirect anywhere else is refused, so the list's addresses are the only + * ones TREK ever reads an extract from. + */ +async function fetchFromGeofabrik(url: string, signal: AbortSignal, hops = 0): Promise { + const response = await fetch(url, { signal, redirect: 'manual' }); + if (response.status < 300 || response.status >= 400) return response; + await response.body?.cancel().catch(() => undefined); + const location = response.headers.get('location'); + const next = location ? new URL(location, url) : null; + if (!next || !isGeofabrikUrl(next)) throw new Error('Geofabrik redirected away from its own servers'); + if (hops >= MAX_REDIRECTS) throw new Error('Geofabrik redirected too often'); + return await fetchFromGeofabrik(next.href, signal, hops + 1); +} + @Injectable() export class GeofabrikClient { private index: { regions: Map; fetchedAt: number } | null = null; @@ -54,18 +83,25 @@ export class GeofabrikClient { /** Every region the list names that has an extract on Geofabrik's host, by id. */ async regions(now: number = Date.now()): Promise> { if (this.index && now - this.index.fetchedAt < INDEX_FRESH_MS) return this.index.regions; - const response = await fetch(INDEX_URL, { signal: AbortSignal.timeout(INDEX_TIMEOUT_MS) }); + const response = await fetchFromGeofabrik(INDEX_URL, AbortSignal.timeout(INDEX_TIMEOUT_MS)); if (!response.ok) { await response.body?.cancel().catch(() => undefined); throw new Error(`Geofabrik index ${response.status}`); } - const parsed = indexSchema.parse(await readCappedJson(response, MAX_INDEX_BYTES)); + const parsed = indexSchema.safeParse(await readCappedJson(response, MAX_INDEX_BYTES)); + if (!parsed.success) throw new Error('Geofabrik index is not the list of extracts'); const regions = new Map(); - for (const feature of parsed.features) { + for (const feature of parsed.data.features) { const { id, name, parent, urls } = feature.properties; - if (!urls.pbf?.startsWith(GEOFABRIK_HOST) || !urls.pbf.endsWith('-latest.osm.pbf')) continue; - const boundary = regionBoundarySchema.safeParse(feature.geometry); - regions.set(id, { id, name, parent: parent ?? null, pbfUrl: urls.pbf, boundary: boundary.success ? boundary.data : null }); + if (!urls.pbf || !isExtractUrl(urls.pbf)) continue; + const boundary = paddleRegionBoundarySchema.safeParse(feature.geometry); + regions.set(id, { + id, + name, + parent: parent ?? null, + pbfUrl: urls.pbf, + boundary: boundary.success ? boundary.data : null, + }); } this.index = { regions, fetchedAt: now }; return regions; @@ -73,26 +109,49 @@ export class GeofabrikClient { /** * One region's extract written to `target`. The address must be one the - * list named; the body is counted as it streams and cut at - * MAX_EXTRACT_BYTES, and the file goes when the download fails. + * list named; the body is counted as it streams and cut at `maxBytes` + * (MAX_EXTRACT_BYTES unless the caller allows less), and the file goes when + * the download fails or `signal` calls it off. `onProgress` hears the share + * received so far when the server said how much there is. */ - async download(url: string, target: string): Promise { - if (!url.startsWith(GEOFABRIK_HOST) || !url.endsWith('-latest.osm.pbf')) throw new Error('Not a Geofabrik extract'); - const response = await fetch(url, { signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS) }); + async download( + url: string, + target: string, + options: { signal?: AbortSignal; onProgress?: (fraction: number) => void; maxBytes?: number } = {}, + ): Promise { + const maxBytes = Math.min(options.maxBytes ?? MAX_EXTRACT_BYTES, MAX_EXTRACT_BYTES); + if (!isExtractUrl(url)) throw new Error('Not a Geofabrik extract'); + const deadline = AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS); + const signal = options.signal ? AbortSignal.any([options.signal, deadline]) : deadline; + const response = await fetchFromGeofabrik(url, signal); if (!response.ok || !response.body) { await response.body?.cancel().catch(() => undefined); throw new Error(`Geofabrik download ${response.status}`); } + const declared = Number(response.headers.get('content-length') ?? 0); + if (declared > maxBytes) { + await response.body.cancel().catch(() => undefined); + throw new Error('Extract larger than allowed'); + } let received = 0; const counter = new Transform({ transform(chunk: Buffer, _encoding, done) { received += chunk.length; - if (received > MAX_EXTRACT_BYTES) done(new Error('Extract larger than allowed')); - else done(null, chunk); + if (received > maxBytes) { + done(new Error('Extract larger than allowed')); + return; + } + if (declared > 0) options.onProgress?.(Math.min(1, received / declared)); + done(null, chunk); }, }); try { - await pipeline(Readable.fromWeb(response.body as unknown as WebReadableStream), counter, createWriteStream(target)); + await pipeline( + Readable.fromWeb(response.body as unknown as WebReadableStream), + counter, + createWriteStream(target), + { signal }, + ); } catch (err) { await unlink(target).catch(() => undefined); throw err; diff --git a/server/src/nest/paddle-obstacles/osm-pbf.reader.ts b/server/src/nest/paddle-obstacles/osm-pbf.reader.ts index f44a2615e..3028355ef 100644 --- a/server/src/nest/paddle-obstacles/osm-pbf.reader.ts +++ b/server/src/nest/paddle-obstacles/osm-pbf.reader.ts @@ -1,4 +1,4 @@ -import { open } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; import { inflateSync } from 'node:zlib'; /** @@ -13,9 +13,15 @@ import { inflateSync } from 'node:zlib'; * megabytes). The format: https://wiki.openstreetmap.org/wiki/PBF_Format */ +/** A varint is at most ten bytes; more means the data is not protobuf. */ +const MAX_VARINT_BYTES = 10; +const TWO_POW_63 = 2 ** 63; +const TWO_POW_64 = 2 ** 64; /** A blob's size may not exceed this, by the format's own rule. */ const MAX_BLOB_BYTES = 32 * 1024 * 1024; const MAX_HEADER_BYTES = 64 * 1024; +/** Read a quarter megabyte at a time: a block or two, so no chunk holds the event loop for long. */ +const READ_CHUNK_BYTES = 256 * 1024; export interface OsmNode { id: number; @@ -56,13 +62,20 @@ class Proto { varint(): number { let result = 0; let scale = 1; - for (;;) { + for (let read = 0; read < MAX_VARINT_BYTES; read++) { if (this.pos >= this.end) throw new Error('Truncated varint'); const byte = this.buf[this.pos++] as number; result += (byte & 0x7f) * scale; if (byte < 0x80) return result; scale *= 128; } + throw new Error('Malformed varint'); + } + + /** A plain (not zigzag) signed 64-bit varint: a negative value is written as its two's complement. */ + int64(): number { + const raw = this.varint(); + return raw >= TWO_POW_63 ? raw - TWO_POW_64 : raw; } /** A zigzag-encoded signed varint. */ @@ -94,12 +107,17 @@ class Proto { skip(wire: number): void { if (wire === 0) this.varint(); - else if (wire === 1) this.pos += 8; + else if (wire === 1) this.advance(8); else if (wire === 2) this.span(); - else if (wire === 5) this.pos += 4; + else if (wire === 5) this.advance(4); else throw new Error(`Unsupported wire type ${wire}`); } + private advance(bytes: number): void { + this.pos += bytes; + if (this.pos > this.end) throw new Error('Truncated field'); + } + /** A packed field of varints (or zigzag varints), decoded whole. */ packed(signed: boolean): number[] { const [start, stop] = this.span(); @@ -158,7 +176,11 @@ interface BlockContext { lonOffset: number; } -function tagsOf(keys: readonly number[], values: readonly number[], strings: readonly string[]): Record { +function tagsOf( + keys: readonly number[], + values: readonly number[], + strings: readonly string[], +): Record { const tags: Record = {}; keys.forEach((key, index) => { const value = values[index]; @@ -260,8 +282,8 @@ export function readDataBlock(block: Buffer, visitor: OsmBlockVisitor): void { } } else if (key.field === 2 && key.wire === 2) groups.push(p.bytes()); else if (key.field === 17 && key.wire === 0) ctx.granularity = p.varint(); - else if (key.field === 19 && key.wire === 0) ctx.latOffset = p.svarint(); - else if (key.field === 20 && key.wire === 0) ctx.lonOffset = p.svarint(); + else if (key.field === 19 && key.wire === 0) ctx.latOffset = p.int64(); + else if (key.field === 20 && key.wire === 0) ctx.lonOffset = p.int64(); else p.skip(key.wire); } for (const group of groups) { @@ -276,42 +298,86 @@ export function readDataBlock(block: Buffer, visitor: OsmBlockVisitor): void { } /** - * Every block of a PBF file in order, read from disk one at a time. The - * header block is answered with its timestamp; data blocks go to the visitor. - * `onBlock` is told the bytes read so far, for progress and to give way to - * other work between blocks. + * The bytes read from a file and not yet taken as a blob, kept as the chunks + * they arrived in and joined only when a blob reaches across chunks. + */ +class ByteQueue { + private chunks: Buffer[] = []; + length = 0; + + push(chunk: Buffer): void { + this.chunks.push(chunk); + this.length += chunk.length; + } + + /** The first `n` bytes, left in the queue; null while fewer have arrived. */ + peek(n: number): Buffer | null { + if (this.length < n) return null; + const first = this.chunks[0]; + if (first && first.length >= n) return first.subarray(0, n); + const joined = Buffer.concat(this.chunks); + this.chunks = [joined]; + return joined.subarray(0, n); + } + + /** Takes the first `n` bytes off the queue; the caller has peeked at least that many. */ + drop(n: number): void { + let left = n; + while (left > 0) { + const first = this.chunks[0] as Buffer; + if (first.length <= left) { + this.chunks.shift(); + left -= first.length; + } else { + this.chunks[0] = first.subarray(left); + left = 0; + } + } + this.length -= n; + } +} + +/** The next whole blob at the head of the queue, taken off it; null while it has not all arrived. */ +function nextBlob(queue: ByteQueue): { type: string; data: Buffer } | null { + const prefix = queue.peek(4); + if (!prefix) return null; + const headerLength = prefix.readUInt32BE(0); + if (headerLength > MAX_HEADER_BYTES) throw new Error('Blob header too large'); + const head = queue.peek(4 + headerLength); + if (!head) return null; + const header = readBlobHeader(head.subarray(4)); + if (header.size > MAX_BLOB_BYTES) throw new Error('Blob too large'); + const whole = queue.peek(4 + headerLength + header.size); + if (!whole) return null; + queue.drop(whole.length); + return { type: header.type, data: whole.subarray(4 + headerLength) }; +} + +/** + * Every block of a PBF file in order, streamed from disk. The header block is + * answered with its timestamp; data blocks go to the visitor. The blocks of + * one chunk are read at once, and the event loop gets its turn between + * chunks. `onProgress` is told the bytes read so far. */ export async function readPbfFile( path: string, visitor: OsmBlockVisitor, - onBlock?: (bytesRead: number) => Promise | void, + onProgress?: (bytesRead: number) => void, ): Promise<{ timestamp: number | null }> { - const file = await open(path, 'r'); + const queue = new ByteQueue(); let timestamp: number | null = null; - try { - let offset = 0; - const lengthBuf = Buffer.alloc(4); - for (;;) { - const { bytesRead } = await file.read(lengthBuf, 0, 4, offset); - if (bytesRead === 0) break; - if (bytesRead < 4) throw new Error('Truncated blob length'); - const headerLength = lengthBuf.readUInt32BE(0); - if (headerLength > MAX_HEADER_BYTES) throw new Error('Blob header too large'); - const headerBuf = Buffer.alloc(headerLength); - await file.read(headerBuf, 0, headerLength, offset + 4); - const header = readBlobHeader(headerBuf); - if (header.size > MAX_BLOB_BYTES) throw new Error('Blob too large'); - const blobBuf = Buffer.alloc(header.size); - const read = await file.read(blobBuf, 0, header.size, offset + 4 + headerLength); - if (read.bytesRead < header.size) throw new Error('Truncated blob'); - const block = readBlob(blobBuf); - if (header.type === 'OSMHeader') timestamp = readHeaderTimestamp(block); - else if (header.type === 'OSMData') readDataBlock(block, visitor); - offset += 4 + headerLength + header.size; - await onBlock?.(offset); + let bytesRead = 0; + for await (const chunk of createReadStream(path, { highWaterMark: READ_CHUNK_BYTES })) { + const bytes = chunk as Buffer; + queue.push(bytes); + bytesRead += bytes.length; + for (let blob = nextBlob(queue); blob; blob = nextBlob(queue)) { + const block = readBlob(blob.data); + if (blob.type === 'OSMHeader') timestamp = readHeaderTimestamp(block); + else if (blob.type === 'OSMData') readDataBlock(block, visitor); } - } finally { - await file.close(); + onProgress?.(bytesRead); } + if (queue.length > 0) throw new Error('Truncated blob'); return { timestamp }; } diff --git a/server/src/nest/paddle-obstacles/paddle-extract.ts b/server/src/nest/paddle-obstacles/paddle-extract.ts index 1d6088bab..42dacadbc 100644 --- a/server/src/nest/paddle-obstacles/paddle-extract.ts +++ b/server/src/nest/paddle-obstacles/paddle-extract.ts @@ -3,6 +3,7 @@ import { keptTags, paddleKindOf, paddleNameOf } from './paddle-tags'; import type { PaddleGeometry, PaddleObstacleKind } from '@trek/shared'; import { createHash } from 'node:crypto'; +import { stat } from 'node:fs/promises'; /** * PAD-05A: the paddle objects of one regional extract, read in two passes. @@ -42,10 +43,12 @@ function hashOf(tags: Record, geometry: PaddleGeometry): string const rounded = JSON.stringify(geometry, (_key, value: unknown) => typeof value === 'number' ? Math.round(value * 1e6) / 1e6 : value, ); - return createHash('sha1').update(JSON.stringify(sorted)).update(rounded).digest('hex').slice(0, 16); + return createHash('sha256').update(JSON.stringify(sorted)).update(rounded).digest('hex').slice(0, 16); } -function boxOf(points: ReadonlyArray): Pick { +function boxOf( + points: ReadonlyArray, +): Pick { let minLat = 90; let minLng = 180; let maxLat = -90; @@ -88,7 +91,10 @@ function build( }; } -function wayFeature(pending: PendingWay, coords: ReadonlyMap): ExtractedFeature | null { +function wayFeature( + pending: PendingWay, + coords: ReadonlyMap, +): ExtractedFeature | null { const points = pending.way.refs.flatMap((ref) => { const at = coords.get(ref); return at ? [at] : []; @@ -98,7 +104,9 @@ function wayFeature(pending: PendingWay, coords: ReadonlyMap= 4 && first[0] === last[0] && first[1] === last[1]; const line = points.map(([lng, lat]) => [lng, lat] as [number, number]); - const geometry: PaddleGeometry = closed ? { type: 'Polygon', coordinates: [line] } : { type: 'LineString', coordinates: line }; + const geometry: PaddleGeometry = closed + ? { type: 'Polygon', coordinates: [line] } + : { type: 'LineString', coordinates: line }; return build(`way/${pending.way.id}`, pending.kind, pending.way.tags, geometry, points); } @@ -144,11 +152,18 @@ export function groupLockParts(features: ExtractedFeature[]): void { } } -/** Every paddle object of the extract at `path`, and the date its data is current to. */ +/** + * Every paddle object of the extract at `path`, and the date its data is + * current to. `onProgress` hears the share of the work done: the first pass + * is the first half, the lookup of the ways' nodes the second. + */ export async function extractPaddleFeatures( path: string, - onProgress?: (bytesRead: number) => Promise | void, + onProgress?: (fraction: number) => void, ): Promise<{ timestamp: number | null; features: ExtractedFeature[] }> { + const { size } = await stat(path); + const progressOf = (pass: number) => (bytesRead: number) => + onProgress?.(size > 0 ? (pass + Math.min(1, bytesRead / size)) / 2 : 1); const features: ExtractedFeature[] = []; const pending: PendingWay[] = []; const needed = new Set(); @@ -170,20 +185,25 @@ export async function extractPaddleFeatures( for (const ref of refs) needed.add(ref); }, }, - onProgress, + progressOf(0), ); if (pending.length > 0) { const coords = new Map(); - await readPbfFile(path, { - node: (node) => { - if (needed.has(node.id)) coords.set(node.id, [node.lon, node.lat]); + await readPbfFile( + path, + { + node: (node) => { + if (needed.has(node.id)) coords.set(node.id, [node.lon, node.lat]); + }, }, - }); + progressOf(1), + ); for (const way of pending) { const feature = wayFeature(way, coords); if (feature) features.push(feature); } } + onProgress?.(1); groupLockParts(features); return { timestamp, features }; } diff --git a/server/src/nest/paddle-obstacles/paddle-geo.ts b/server/src/nest/paddle-obstacles/paddle-geo.ts index 946969bf4..ff7a2af8a 100644 --- a/server/src/nest/paddle-obstacles/paddle-geo.ts +++ b/server/src/nest/paddle-obstacles/paddle-geo.ts @@ -1,14 +1,11 @@ import type { Box } from '../../db/repositories/PaddleFeatures.repository'; -import { PADDLE_OBSTACLES_MAX_SPAN_DEG, type PaddleCoverageState } from '@trek/shared'; -import { z } from 'zod'; +import { + PADDLE_OBSTACLES_MAX_SPAN_DEG, + type PaddleCoverageState, + type PaddleRegionBoundary as RegionBoundary, +} from '@trek/shared'; -/** Geofabrik's region boundaries: a polygon or several, rings of [lng, lat]. */ -const ring = z.array(z.tuple([z.number(), z.number()])); -export const regionBoundarySchema = z.discriminatedUnion('type', [ - z.object({ type: z.literal('Polygon'), coordinates: z.array(ring) }), - z.object({ type: z.literal('MultiPolygon'), coordinates: z.array(z.array(ring)) }), -]); -export type RegionBoundary = z.infer; +export type { RegionBoundary }; /** `south,west,north,east` as a box, or null when it is not one or wider than a request may ask. */ export function parseBox(text: string): Box | null { diff --git a/server/src/nest/paddle-obstacles/paddle-obstacles.service.ts b/server/src/nest/paddle-obstacles/paddle-obstacles.service.ts index 0f1baaaf1..755edaa26 100644 --- a/server/src/nest/paddle-obstacles/paddle-obstacles.service.ts +++ b/server/src/nest/paddle-obstacles/paddle-obstacles.service.ts @@ -1,15 +1,18 @@ import { PaddleFeatures } from '../../db/entities/PaddleFeatures.entity'; import { PaddleRegions } from '../../db/entities/PaddleRegions.entity'; +import { PADDLE_FEATURE_GEOMETRY, PADDLE_FEATURE_TAGS, PADDLE_REGION_BOUNDARY } from '../../db/json-columns'; import type { Box, PaddleFeatureRow, PaddleFeaturesRepository } from '../../db/repositories/PaddleFeatures.repository'; import type { PaddleRegionsRepository } from '../../db/repositories/PaddleRegions.repository'; +import { utcSuffix } from '../../db/types'; +import { decodeJson } from '../../utils/json-column'; import { logWarn } from '../audit/audit-log.logger'; import { badRequest } from '../common/domain-error'; -import { coverageOf, parseBox, regionBoundarySchema, type RegionBoundary } from './paddle-geo'; +import { coverageOf, parseBox } from './paddle-geo'; import { InjectRepository } from '@mikro-orm/nestjs'; import { Injectable } from '@nestjs/common'; import { PADDLE_OBSTACLES_CAP, - paddleFeatureSchema, + paddleObstacleKindSchema, type PaddleFeature, type PaddleObstaclesResponse, } from '@trek/shared'; @@ -17,29 +20,31 @@ import { export const PADDLE_BOX_INVALID_MESSAGE = 'bbox must be south,west,north,east and at most one degree a side'; function featureOf(row: PaddleFeatureRow): PaddleFeature | null { - try { - const parsed = paddleFeatureSchema.safeParse({ - ref: row.osm_ref, - kind: row.kind, - group_key: row.group_key, - name: row.name, - geometry: JSON.parse(row.geometry), - tags: JSON.parse(row.tags), - tag_hash: row.tag_hash, - region_id: row.region_id, - }); - if (parsed.success) return parsed.data; - } catch { - // Unreadable JSON falls through to the log line below. + const context = `${row.region_id}/${row.osm_ref}`; + const kind = paddleObstacleKindSchema.safeParse(row.kind); + const geometry = decodeJson(PADDLE_FEATURE_GEOMETRY, row.geometry, context); + if (!kind.success || !geometry) { + logWarn(`[Paddle] stored object ${context} does not read; left out`); + return null; } - logWarn(`[Paddle] stored object ${row.region_id}/${row.osm_ref} does not read; left out`); - return null; + return { + ref: row.osm_ref, + kind: kind.data, + group_key: row.group_key, + name: row.name, + geometry, + tags: decodeJson(PADDLE_FEATURE_TAGS, row.tags, context), + tag_hash: row.tag_hash, + region_id: row.region_id, + }; } /** * PAD-05A, the reading side: the paddle objects inside a map area and how far * TREK's data covers it. An area outside every region answers `none`, not an - * empty list read as "no obstacles"; one cut at the cap says so. + * empty list read as "no obstacles"; one cut at the cap says so. Where two + * regions overlap (a country and one of its states), an object both hold is + * answered once. */ @Injectable() export class PaddleObstaclesService { @@ -58,29 +63,29 @@ export class PaddleObstaclesService { async read(box: Box): Promise { const [rows, regions] = await Promise.all([ this.features.findInBox(box, PADDLE_OBSTACLES_CAP + 1), - this.regions.readyInBox(box), + this.regions.withDataInBox(box), ]); - const boundaries = regions.map((region): RegionBoundary | null => { - if (!region.boundary) return null; - try { - const parsed = regionBoundarySchema.safeParse(JSON.parse(region.boundary)); - return parsed.success ? parsed.data : null; - } catch { - return null; - } + const seen = new Set(); + const unique = rows.filter((row) => { + if (seen.has(row.osm_ref)) return false; + seen.add(row.osm_ref); + return true; }); return { - features: rows.slice(0, PADDLE_OBSTACLES_CAP).flatMap((row) => featureOf(row) ?? []), + features: unique.slice(0, PADDLE_OBSTACLES_CAP).flatMap((row) => featureOf(row) ?? []), coverage: { - state: coverageOf(box, boundaries), + state: coverageOf( + box, + regions.map((region) => decodeJson(PADDLE_REGION_BOUNDARY, region.boundary, region.id)), + ), regions: regions.map((region) => ({ id: region.id, name: region.name, source_timestamp: region.source_timestamp, - imported_at: region.imported_at === null ? null : `${region.imported_at.replace(' ', 'T')}Z`, + imported_at: utcSuffix(region.imported_at), })), }, - truncated: rows.length > PADDLE_OBSTACLES_CAP, + truncated: unique.length > PADDLE_OBSTACLES_CAP, }; } } diff --git a/server/src/nest/paddle-obstacles/paddle-regions.job.ts b/server/src/nest/paddle-obstacles/paddle-regions.job.ts index 6b0462d26..c78ad4609 100644 --- a/server/src/nest/paddle-obstacles/paddle-regions.job.ts +++ b/server/src/nest/paddle-obstacles/paddle-regions.job.ts @@ -16,12 +16,22 @@ export class PaddleRegionsJob implements OnApplicationBootstrap { private readonly registrar: CronRegistrarService, ) {} + /** @txIndependent the boot resume and the monthly tick are two separate runs, never one sequence of writes */ onApplicationBootstrap(): void { if (!this.registrar.isEnabled()) return; - void this.registrar.runOnBoot('paddle-regions-resume', () => this.regions.resumeAfterRestart()); + void this.registrar.runOnBoot('paddle-regions-resume', () => this.resume()); this.registrar.register('paddle-regions-refresh', '20 4 3 * *', () => this.tick()); } + /** The boot's half: a failure is logged, never thrown at the boot. */ + async resume(): Promise { + try { + await this.regions.resumeAfterRestart(); + } catch (err) { + logError(`[Paddle] resuming the reads after a restart: ${err instanceof Error ? err.message : String(err)}`); + } + } + async tick(): Promise { try { const queued = await this.regions.queueRefresh(); diff --git a/server/src/nest/paddle-obstacles/paddle-regions.service.ts b/server/src/nest/paddle-obstacles/paddle-regions.service.ts index b554f99da..acc79656c 100644 --- a/server/src/nest/paddle-obstacles/paddle-regions.service.ts +++ b/server/src/nest/paddle-obstacles/paddle-regions.service.ts @@ -1,38 +1,47 @@ import { PaddleFeatures } from '../../db/entities/PaddleFeatures.entity'; import { PaddleRegions } from '../../db/entities/PaddleRegions.entity'; +import { PADDLE_FEATURE_GEOMETRY, PADDLE_FEATURE_TAGS, PADDLE_REGION_BOUNDARY } from '../../db/json-columns'; import type { PaddleFeatureRow, PaddleFeaturesRepository } from '../../db/repositories/PaddleFeatures.repository'; import type { PaddleRegionRow, PaddleRegionsRepository } from '../../db/repositories/PaddleRegions.repository'; -import { dbNow } from '../../db/types'; +import { dbNow, utcSuffix } from '../../db/types'; +import { encodeJson } from '../../utils/json-column'; import { DataPathsService } from '../app-config/data-paths.service'; import { logError, logInfo } from '../audit/audit-log.logger'; import { DomainError } from '../common/domain-error'; import { withRequestContext } from '../database/request-context'; import { UnitOfWork } from '../database/unit-of-work'; -import { GeofabrikClient } from './geofabrik.client'; +import { GeofabrikClient, type GeofabrikRegion } from './geofabrik.client'; import { extractPaddleFeatures, type ExtractedFeature } from './paddle-extract'; import { boxOfBoundary } from './paddle-geo'; import { MikroORM } from '@mikro-orm/core'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { Injectable } from '@nestjs/common'; +import { Injectable, type OnApplicationShutdown } from '@nestjs/common'; import type { PaddleRegion, PaddleRegionCatalogEntry } from '@trek/shared'; -import { mkdir, unlink } from 'node:fs/promises'; +import { mkdir, rm, unlink } from 'node:fs/promises'; import { join } from 'node:path'; const CATALOG_LIMIT = 25; export const REGION_UNKNOWN_MESSAGE = 'Geofabrik has no extract by that id'; export const CATALOG_UNAVAILABLE_MESSAGE = 'The list of regions cannot be fetched right now'; +export const REGION_NOT_FOUND_MESSAGE = 'Region not found'; +const REMOVED_WHILE_READ = 'removed while it was read; nothing kept'; +const STOPPED_AT_SHUTDOWN = 'stopped by the shutdown; read again on the next start'; -function toRegion(row: PaddleRegionRow): PaddleRegion { +/** What the region being read is doing, for the admin list; held in memory, as one process reads. */ +type ReadProgress = NonNullable; + +function toRegion(row: PaddleRegionRow, progress: ReadProgress | null = null): PaddleRegion { return { id: row.id, name: row.name, status: row.status, source_timestamp: row.source_timestamp, - imported_at: row.imported_at === null ? null : `${row.imported_at.replace(' ', 'T')}Z`, + imported_at: utcSuffix(row.imported_at), feature_count: row.feature_count, error: row.error, + progress: row.status === 'importing' ? progress : null, }; } @@ -43,8 +52,8 @@ function toRow(feature: ExtractedFeature): PaddleFeatureRow { kind: feature.kind, group_key: feature.group_key, name: feature.name, - geometry: JSON.stringify(feature.geometry), - tags: JSON.stringify(feature.tags), + geometry: encodeJson(PADDLE_FEATURE_GEOMETRY, feature.geometry), + tags: encodeJson(PADDLE_FEATURE_TAGS, feature.tags), tag_hash: feature.tag_hash, min_lat: feature.min_lat, min_lng: feature.min_lng, @@ -53,17 +62,41 @@ function toRow(feature: ExtractedFeature): PaddleFeatureRow { }; } +/** The box around a read's objects, for a region whose boundary Geofabrik does not give. */ +function extentOf( + features: readonly ExtractedFeature[], +): Pick { + if (features.length === 0) return { min_lat: null, min_lng: null, max_lat: null, max_lng: null }; + const box = { min_lat: 90, min_lng: 180, max_lat: -90, max_lng: -180 }; + for (const feature of features) { + box.min_lat = Math.min(box.min_lat, feature.min_lat); + box.min_lng = Math.min(box.min_lng, feature.min_lng); + box.max_lat = Math.max(box.max_lat, feature.max_lat); + box.max_lng = Math.max(box.max_lng, feature.max_lng); + } + return box; +} + +function messageOf(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + /** * PAD-05A: the regions TREK holds paddle data for. An admin picks a region * from Geofabrik's list; TREK downloads its extract into the data folder's * temporary space, reads the paddle objects out of it, replaces the region's * rows in one transaction and deletes the file. Regions are read one at a * time, in the background, never on a request's path; a failed read keeps the - * region's previous data and says why. + * region's previous data and says why. Removing a region while it is read + * calls the read off. */ @Injectable() -export class PaddleRegionsService { +export class PaddleRegionsService implements OnApplicationShutdown { private running: Promise | null = null; + /** A region was queued after the running reads last looked at the queue. */ + private again = false; + private current: { id: string; abort: AbortController; progress: ReadProgress } | null = null; + private stopping = false; constructor( private readonly geofabrik: GeofabrikClient, @@ -75,18 +108,12 @@ export class PaddleRegionsService { ) {} async list(): Promise { - return (await this.regions.list()).map(toRegion); + return (await this.regions.list()).map((row) => toRegion(row, this.progressOf(row.id))); } /** Regions of Geofabrik's list whose name or id matches, with the regions they lie in. */ async catalog(query: string): Promise { - let all: Awaited>; - try { - all = await this.geofabrik.regions(); - } catch (err) { - logError(`[Paddle] Geofabrik index: ${err instanceof Error ? err.message : String(err)}`); - throw new DomainError(502, CATALOG_UNAVAILABLE_MESSAGE, { code: 'catalog_unavailable', details: { code: 'catalog_unavailable' } }); - } + const all = await this.catalogue(); const needle = query.toLowerCase(); const path = (id: string | null): string[] => { const names: string[] = []; @@ -95,42 +122,61 @@ export class PaddleRegionsService { }; return [...all.values()] .filter((region) => region.name.toLowerCase().includes(needle) || region.id.includes(needle)) - .sort((a, b) => Number(!a.name.toLowerCase().startsWith(needle)) - Number(!b.name.toLowerCase().startsWith(needle))) + .sort( + (a, b) => Number(!a.name.toLowerCase().startsWith(needle)) - Number(!b.name.toLowerCase().startsWith(needle)), + ) .slice(0, CATALOG_LIMIT) .map((region) => ({ id: region.id, name: region.name, path: path(region.parent) })); } - /** Adds a region and queues its first read; a region already there is queued again. */ + /** Adds a region and queues its first read; a region already there is read again, as `refresh` does. */ async add(id: string): Promise { - const region = (await this.geofabrik.regions().catch(() => new Map())).get(id); - if (!region) throw new DomainError(400, REGION_UNKNOWN_MESSAGE, { code: 'unknown_region', details: { code: 'unknown_region' } }); + if (await this.regions.findById(id)) return await this.refresh(id); + const region = (await this.catalogue()).get(id); + if (!region) + throw new DomainError(400, REGION_UNKNOWN_MESSAGE, { + code: 'unknown_region', + details: { code: 'unknown_region' }, + }); const box = region.boundary ? boxOfBoundary(region.boundary) : null; - await this.regions.insertRegion({ - id, - name: region.name, - boundary: region.boundary ? JSON.stringify(region.boundary) : null, - min_lat: box?.south ?? null, - min_lng: box?.west ?? null, - max_lat: box?.north ?? null, - max_lng: box?.east ?? null, + await this.uow.transactional(async () => { + await this.regions.insertRegion({ + id, + name: region.name, + boundary: region.boundary ? encodeJson(PADDLE_REGION_BOUNDARY, region.boundary) : null, + min_lat: box?.south ?? null, + min_lng: box?.west ?? null, + max_lat: box?.north ?? null, + max_lng: box?.east ?? null, + }); + await this.regions.updateRegion(id, { status: 'queued', error: null }); }); - await this.regions.updateRegion(id, { status: 'queued', error: null }); this.kick(); const row = await this.regions.findById(id); if (!row) throw new Error('Added region could not be read back'); return toRegion(row); } + /** Queues a region's read again; one already waiting or being read is left as it is. */ async refresh(id: string): Promise { const row = await this.regions.findById(id); - if (!row) throw new DomainError(404, 'Region not found'); + if (!row) throw new DomainError(404, REGION_NOT_FOUND_MESSAGE); + if (row.status === 'queued' || row.status === 'importing') return toRegion(row, this.progressOf(id)); await this.regions.updateRegion(id, { status: 'queued', error: null }); this.kick(); return toRegion({ ...row, status: 'queued', error: null }); } + /** Removes a region and its objects; a read of it under way is called off. */ async remove(id: string): Promise { - if (!(await this.regions.deleteRegion(id))) throw new DomainError(404, 'Region not found'); + if (!(await this.regions.deleteRegion(id))) throw new DomainError(404, REGION_NOT_FOUND_MESSAGE); + if (this.current?.id === id) this.current.abort.abort(REMOVED_WHILE_READ); + } + + /** A read under way stops with the process; the boot after it queues the region again. */ + onApplicationShutdown(): void { + this.stopping = true; + this.current?.abort.abort(STOPPED_AT_SHUTDOWN); } /** Every region with data queued again, for the monthly refresh; the reads follow in the background. */ @@ -140,41 +186,103 @@ export class PaddleRegionsService { return queued; } - /** A restart left a read unfinished: it goes back to the queue, and the queue starts. */ + /** + * A restart left a read unfinished: it goes back to the queue, the file it + * was reading is cleared away, and the queue starts. + */ async resumeAfterRestart(): Promise { await this.regions.requeueInterrupted(); + await rm(this.downloadDir(), { recursive: true, force: true }); this.kick(); } - /** Starts the background reads unless they already run; the promise is for tests to wait on. */ - kick(): Promise { - if (!this.running) { - this.running = withRequestContext(this.orm, () => this.drain()).finally(() => { - this.running = null; - }); - } - return this.running; + /** + * Starts the background reads unless they already run. A region queued + * while they run is still picked up: the run looks at the queue again + * before it ends. + * + * @txStandalone the reads run detached, later, each region's rows in a transaction of their own; no caller's transaction could hold them + */ + kick(): void { + this.again = true; + if (this.running !== null || this.stopping) return; + this.running = this.run().finally(() => { + this.running = null; + if (this.again) this.kick(); + }); } - private async drain(): Promise { - for (let next = await this.regions.nextQueued(); next; next = await this.regions.nextQueued()) { - await this.importRegion(next); + /** Resolves once the queue is empty and nothing reads; for tests and an orderly shutdown. */ + async idle(): Promise { + if (this.running === null) return; + await this.running; + await this.idle(); + } + + /** Where extracts are downloaded to while they are read; nothing in it outlives its read. */ + private downloadDir(): string { + return join(this.paths.tmpDir, 'paddle'); + } + + private progressOf(id: string): ReadProgress | null { + return this.current?.id === id ? { ...this.current.progress } : null; + } + + private async catalogue(): Promise> { + try { + return await this.geofabrik.regions(); + } catch (err) { + logError(`[Paddle] Geofabrik index: ${messageOf(err)}`); + throw new DomainError(502, CATALOG_UNAVAILABLE_MESSAGE, { + code: 'catalog_unavailable', + details: { code: 'catalog_unavailable' }, + }); } } + private async run(): Promise { + this.again = false; + try { + await withRequestContext(this.orm, () => this.drain()); + } catch (err) { + logError(`[Paddle] the read queue stopped: ${messageOf(err)}`); + } + } + + /** The queued regions, one after another, oldest first. */ + private async drain(): Promise { + if (this.stopping) return; + const next = await this.regions.nextQueued(); + if (!next) return; + await this.importRegion(next); + await this.drain(); + } + /** @txIndependent the download and the read sit between the status writes; only the swap of rows is one transaction. */ private async importRegion(region: PaddleRegionRow): Promise { await this.regions.updateRegion(region.id, { status: 'importing', error: null }); - const dir = join(this.paths.tmpDir, 'paddle'); - const file = join(dir, `${region.id}-${Date.now()}.osm.pbf`); + const abort = new AbortController(); + const current = { id: region.id, abort, progress: { phase: 'download', fraction: null } as ReadProgress }; + this.current = current; + const dir = this.downloadDir(); + // Some of Geofabrik's ids are nested (`us/texas`); the file stays in the one folder. + const file = join(dir, `${region.id.replaceAll('/', '_')}-${Date.now()}.osm.pbf`); try { const known = (await this.geofabrik.regions()).get(region.id); if (!known) throw new Error(REGION_UNKNOWN_MESSAGE); await mkdir(dir, { recursive: true }); const started = Date.now(); - await this.geofabrik.download(known.pbfUrl, file); - const { timestamp, features } = await extractPaddleFeatures(file); - await this.uow.transactional(async () => { + await this.geofabrik.download(known.pbfUrl, file, { + signal: abort.signal, + onProgress: (fraction) => (current.progress = { phase: 'download', fraction }), + }); + current.progress = { phase: 'read', fraction: 0 }; + const { timestamp, features } = await extractPaddleFeatures(file, (fraction) => { + current.progress = { phase: 'read', fraction }; + }); + abort.signal.throwIfAborted(); + const kept = await this.uow.transactional(async () => { + if (!(await this.regions.findById(region.id))) return false; await this.features.replaceRegion(region.id, features.map(toRow)); await this.regions.updateRegion(region.id, { status: 'ready', @@ -182,14 +290,26 @@ export class PaddleRegionsService { source_timestamp: timestamp === null ? null : new Date(timestamp * 1000).toISOString(), imported_at: dbNow(), error: null, + // Without a boundary the region covers what its objects span, so the map can still tell it is covered. + ...(known.boundary ? {} : extentOf(features)), }); + return true; }); - logInfo(`[Paddle] ${region.id}: ${features.length} objects read in ${Math.round((Date.now() - started) / 1000)} s`); + if (!kept) throw new Error(REMOVED_WHILE_READ); + logInfo( + `[Paddle] ${region.id}: ${features.length} objects read in ${Math.round((Date.now() - started) / 1000)} s`, + ); } catch (err) { - const message = err instanceof Error ? err.message : String(err); - logError(`[Paddle] ${region.id}: ${message}`); - await this.regions.updateRegion(region.id, { status: 'failed', error: message.slice(0, 500) }); + if (abort.signal.aborted) { + logInfo(`[Paddle] ${region.id}: ${String(abort.signal.reason)}`); + } else if (messageOf(err) === REMOVED_WHILE_READ) { + logInfo(`[Paddle] ${region.id}: ${REMOVED_WHILE_READ}`); + } else { + logError(`[Paddle] ${region.id}: ${messageOf(err)}`); + await this.regions.updateRegion(region.id, { status: 'failed', error: messageOf(err).slice(0, 500) }); + } } finally { + if (this.current === current) this.current = null; await unlink(file).catch(() => undefined); } } diff --git a/server/src/nest/tours/tours.service.ts b/server/src/nest/tours/tours.service.ts index dc23bcc20..e734155de 100644 --- a/server/src/nest/tours/tours.service.ts +++ b/server/src/nest/tours/tours.service.ts @@ -2,6 +2,7 @@ import { Places } from '../../db/entities/Places.entity'; import { TourTypes } from '../../db/entities/TourTypes.entity'; import { TourWaypoints } from '../../db/entities/TourWaypoints.entity'; import { Tours } from '../../db/entities/Tours.entity'; +import { TOUR_TRACK_MARKS } from '../../db/json-columns'; import type { PlacesRepository } from '../../db/repositories/Places.repository'; import type { TourTypesRepository } from '../../db/repositories/TourTypes.repository'; import type { TourWaypointsRepository } from '../../db/repositories/TourWaypoints.repository'; @@ -12,7 +13,6 @@ import type { ToursRepository, TourUpdate, } from '../../db/repositories/Tours.repository'; -import { TOUR_TRACK_MARKS } from '../../db/json-columns'; import { decodeJson } from '../../utils/json-column'; import { DomainError } from '../common/domain-error'; import { toRowId } from '../common/row-id'; diff --git a/server/src/nest/weather/weather.controller.ts b/server/src/nest/weather/weather.controller.ts index 4e130ed08..a69a68476 100644 --- a/server/src/nest/weather/weather.controller.ts +++ b/server/src/nest/weather/weather.controller.ts @@ -1,9 +1,9 @@ import { JwtAuthGuard } from '../auth-core/jwt-auth.guard'; +import { ResponseContract } from '../common/response-contract'; +import { WindQueryDto } from './dto/wind-query.dto'; import { ApiError } from './weather.impl'; import { WeatherService } from './weather.service'; -import { WindQueryDto } from './dto/wind-query.dto'; import { WindService } from './wind.service'; -import { ResponseContract } from '../common/response-contract'; import { Controller, Get, HttpException, Query, UseGuards } from '@nestjs/common'; import { windForecastResponseSchema, type WeatherResult, type WindForecastResponse } from '@trek/shared'; diff --git a/server/src/nest/weather/wind.service.ts b/server/src/nest/weather/wind.service.ts index dabb59bb9..f59b37286 100644 --- a/server/src/nest/weather/wind.service.ts +++ b/server/src/nest/weather/wind.service.ts @@ -3,6 +3,7 @@ import { logWarn } from '../audit/audit-log.logger'; import { DomainError } from '../common/domain-error'; import { Injectable } from '@nestjs/common'; import { WIND_FORECAST_DAYS, type WindForecastResponse, type WindForecastStatus, type WindQuery } from '@trek/shared'; + import { z } from 'zod'; /** Open-Meteo on the request path: a deadline and a size cap, as the weather calls have. */ @@ -56,7 +57,7 @@ export class WindService { const hit = this.cache.get(key); if (hit && hit.expiresAt > now) return hit.value; const running = this.pending.get(key); - if (running) return running; + if (running !== undefined) return running; const request = this.fetchDay(query, now) .then((value) => { this.remember(key, value, now); diff --git a/server/tests/e2e/paddle-obstacles.e2e.test.ts b/server/tests/e2e/paddle-obstacles.e2e.test.ts new file mode 100644 index 000000000..c7f5933a1 --- /dev/null +++ b/server/tests/e2e/paddle-obstacles.e2e.test.ts @@ -0,0 +1,291 @@ +/** + * Paddle obstacles e2e (PAD-05A): `GET /api/paddle/obstacles` and the admin + * routes `/api/admin/paddle-regions*` through the real AddonGuard, + * JwtAuthGuard and AdminGuard, the Zod pipe and the response contracts, over + * the real services and repositories on a migrated temp SQLite db. + * + * Replaced: `GeofabrikClient`, so no test reaches Geofabrik; its download + * writes an extract the test builds (tests/helpers/osm-pbf-fixture.ts), which + * the real reader then reads. + */ +import { AppConfigModule } from '../../src/nest/app-config/app-config.module'; +import { DataPathsService } from '../../src/nest/app-config/data-paths.service'; +import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; +import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { GeofabrikClient, type GeofabrikRegion } from '../../src/nest/paddle-obstacles/geofabrik.client'; +import { PaddleObstaclesModule } from '../../src/nest/paddle-obstacles/paddle-obstacles.module'; +import { PADDLE_BOX_INVALID_MESSAGE } from '../../src/nest/paddle-obstacles/paddle-obstacles.service'; +import { + CATALOG_UNAVAILABLE_MESSAGE, + PaddleRegionsService, + REGION_NOT_FOUND_MESSAGE, + REGION_UNKNOWN_MESSAGE, +} from '../../src/nest/paddle-obstacles/paddle-regions.service'; +import { createSnapshotTestDb } from '../helpers/db-mock'; +import { setAddonEnabled } from '../helpers/factories/settings'; +import { makeAdmin, makeUser } from '../helpers/factories/users'; +import { pbfFile } from '../helpers/osm-pbf-fixture'; +import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm'; +import { TestUnitOfWorkModule } from '../helpers/test-uow'; +import { sessionCookie } from './harness'; +import { Test } from '@nestjs/testing'; + +import cookieParser from 'cookie-parser'; +import type { Server } from 'http'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/nest/audit/audit-log.logger', () => ({ + LOG_LEVEL: 'error', + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +/** A migrated temp database handed to the code under test, rather than a mocked database module. */ +const db = createSnapshotTestDb(); + +const USER = 1; +const ADMIN = 2; +const REGION: GeofabrikRegion = { + id: 'oberpfalz', + name: 'Oberpfalz', + parent: 'bayern', + pbfUrl: 'https://download.geofabrik.de/europe/germany/bayern/oberpfalz-latest.osm.pbf', + boundary: { + type: 'Polygon', + coordinates: [ + [ + [11.5, 48.9], + [13, 48.9], + [13, 50], + [11.5, 50], + [11.5, 48.9], + ], + ], + }, +}; +const BAYERN: GeofabrikRegion = { ...REGION, id: 'bayern', name: 'Bayern', parent: null, boundary: null }; +/** The weir at Lindenmühle, Regenstauf, as a line across the Regen. */ +const EXTRACT = pbfFile( + [ + { + dense: [ + { id: 1, lat: 49.1231, lon: 12.1301 }, + { id: 2, lat: 49.124, lon: 12.1315 }, + ], + }, + { ways: [{ id: 29256405, refs: [1, 2], tags: { waterway: 'weir', name: 'Wehr Lindenmühle' } }] }, + ], + { timestamp: Date.UTC(2026, 9, 1) / 1000 }, +); +const BOX = '49.1,12.1,49.2,12.2'; + +const geofabrik = { + regions: vi.fn<() => Promise>>(), + download: vi.fn<(url: string, target: string) => Promise>(), +}; + +let orm: TestOrm; +let server: Server; +let dir: string; +let app: Awaited>; + +async function build() { + const moduleRef = await Test.createTestingModule({ + imports: [ + await TestUnitOfWorkModule.forRoot(db), + await createTestMikroOrmModule(db), + AppConfigModule, + PaddleObstaclesModule, + ], + }) + .overrideProvider(GeofabrikClient) + .useValue(geofabrik) + .overrideProvider(DataPathsService) + .useValue({ tmpDir: dir }) + .compile(); + const nest = moduleRef.createNestApplication(); + nest.use(cookieParser()); + nest.useGlobalPipes(new ZodValidationPipe()); + nest.useGlobalFilters(new TrekExceptionFilter()); + await nest.init(); + return nest; +} + +const as = (userId: number) => ({ Cookie: sessionCookie(userId) }); +const obstacles = (bbox: string, userId?: number) => { + const req = request(server).get('/api/paddle/obstacles').query({ bbox }); + return userId === undefined ? req : req.set(as(userId)); +}; +const admin = { + list: (userId = ADMIN) => request(server).get('/api/admin/paddle-regions').set(as(userId)), + catalog: (q: string, userId = ADMIN) => + request(server).get('/api/admin/paddle-regions/catalog').query({ q }).set(as(userId)), + add: (id: string, userId = ADMIN) => request(server).post('/api/admin/paddle-regions').set(as(userId)).send({ id }), + refresh: (id: string, userId = ADMIN) => + request(server).post('/api/admin/paddle-regions/refresh').set(as(userId)).send({ id }), + remove: (id: string, userId = ADMIN) => + request(server).post('/api/admin/paddle-regions/remove').set(as(userId)).send({ id }), +}; +const settle = () => app.get(PaddleRegionsService).idle(); + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-paddle-e2e-')); + orm = await createTestOrm(db); + await makeUser(orm, { id: USER, email: 'paddler@example.test' }); + await makeAdmin(orm, { id: ADMIN, email: 'admin@example.test' }); + app = await build(); + server = app.getHttpServer(); +}); + +beforeEach(async () => { + vi.clearAllMocks(); + geofabrik.regions.mockResolvedValue( + new Map([ + [REGION.id, REGION], + [BAYERN.id, BAYERN], + ]), + ); + geofabrik.download.mockImplementation((_url, target) => writeFile(target, EXTRACT)); + await setAddonEnabled(orm, 'tours', true); + await admin.remove(REGION.id); + await admin.remove(BAYERN.id); + await settle(); +}); + +afterAll(async () => { + await app.close(); + await orm.close(); + db.close(); + await rm(dir, { recursive: true, force: true }); +}); + +describe('GET /api/paddle/obstacles', () => { + it('PADE2E-001: 404 while Tours is off, ahead of the session check; 401 without a cookie once on', async () => { + await setAddonEnabled(orm, 'tours', false); + for (const res of [await obstacles(BOX), await obstacles(BOX, USER)]) { + expect(res.status).toBe(404); + expect(res.body).toEqual({ error: 'Tours addon is not enabled' }); + } + await setAddonEnabled(orm, 'tours', true); + const anonymous = await obstacles(BOX); + expect(anonymous.status).toBe(401); + expect(anonymous.body).toEqual({ error: 'Access token required', code: 'AUTH_REQUIRED' }); + }); + + it('PADE2E-002: 400 for a box that is not one, and for one wider than a degree', async () => { + expect((await obstacles('1,2,3', USER)).status).toBe(400); + expect((await request(server).get('/api/paddle/obstacles').set(as(USER))).status).toBe(400); + const wide = await obstacles('48,11,50,13', USER); + expect(wide.status).toBe(400); + expect(wide.body).toEqual({ error: PADDLE_BOX_INVALID_MESSAGE }); + }); + + it('PADE2E-003: an area without data answers none, never an empty all-clear', async () => { + const res = await obstacles(BOX, USER); + expect(res.status).toBe(200); + expect(res.body).toEqual({ features: [], coverage: { state: 'none', regions: [] }, truncated: false }); + }); +}); + +describe('/api/admin/paddle-regions', () => { + it('PADE2E-004: 401 without a cookie and 403 for a user who is no admin, on every route', async () => { + expect((await request(server).get('/api/admin/paddle-regions')).status).toBe(401); + for (const res of [ + await admin.list(USER), + await admin.catalog('ober', USER), + await admin.add(REGION.id, USER), + await admin.refresh(REGION.id, USER), + await admin.remove(REGION.id, USER), + ]) { + expect(res.status).toBe(403); + expect(res.body).toEqual({ error: 'Admin access required' }); + } + expect(geofabrik.regions).not.toHaveBeenCalled(); + }); + + it('PADE2E-005: an admin adds a region, its read lands, and the map reads the weir as a line with the date of the data', async () => { + const added = await admin.add(REGION.id); + expect(added.status).toBe(201); + expect(added.body).toMatchObject({ id: 'oberpfalz', name: 'Oberpfalz', feature_count: 0 }); + await settle(); + + const list = await admin.list(); + expect(list.status).toBe(200); + expect(list.body.regions).toEqual([ + expect.objectContaining({ + id: 'oberpfalz', + status: 'ready', + feature_count: 1, + source_timestamp: '2026-10-01T00:00:00.000Z', + error: null, + progress: null, + }), + ]); + + const res = await obstacles(BOX, USER); + expect(res.status).toBe(200); + expect(res.body.coverage).toMatchObject({ state: 'covered', regions: [{ id: 'oberpfalz', name: 'Oberpfalz' }] }); + expect(res.body.features).toEqual([ + expect.objectContaining({ + ref: 'way/29256405', + kind: 'weir', + name: 'Wehr Lindenmühle', + geometry: { type: 'LineString', coordinates: expect.any(Array) }, + }), + ]); + }); + + it('PADE2E-006: refresh answers 200 with the region queued, remove 200 with success; both 404 for a region not held', async () => { + await admin.add(REGION.id); + await settle(); + + const refreshed = await admin.refresh(REGION.id); + expect(refreshed.status).toBe(200); + expect(refreshed.body).toMatchObject({ id: 'oberpfalz', status: 'queued' }); + await settle(); + + const removed = await admin.remove(REGION.id); + expect(removed.status).toBe(200); + expect(removed.body).toEqual({ success: true }); + expect((await obstacles(BOX, USER)).body.coverage.state).toBe('none'); + + for (const res of [await admin.refresh(REGION.id), await admin.remove(REGION.id)]) { + expect(res.status).toBe(404); + expect(res.body).toEqual({ error: REGION_NOT_FOUND_MESSAGE }); + } + }); + + it('PADE2E-007: an id Geofabrik does not list is 400 with the reason, a malformed one 400 from the pipe', async () => { + const unknown = await admin.add('atlantis'); + expect(unknown.status).toBe(400); + expect(unknown.body).toEqual({ error: REGION_UNKNOWN_MESSAGE, code: 'unknown_region' }); + expect((await admin.add('Not An Id!')).status).toBe(400); + expect((await admin.add('')).status).toBe(400); + }); + + it('PADE2E-008: the catalogue finds regions with their path; a short query is 400, an unreachable Geofabrik 502', async () => { + const found = await admin.catalog('ober'); + expect(found.status).toBe(200); + expect(found.body).toEqual({ regions: [{ id: 'oberpfalz', name: 'Oberpfalz', path: ['Bayern'] }] }); + + expect((await admin.catalog('o')).status).toBe(400); + + geofabrik.regions.mockRejectedValue(new Error('Geofabrik index 503')); + const down = await admin.catalog('ober'); + expect(down.status).toBe(502); + expect(down.body).toEqual({ error: CATALOG_UNAVAILABLE_MESSAGE, code: 'catalog_unavailable' }); + }); + + it('PADE2E-009: the admin keeps the regions while Tours is off, so data can be prepared before it is switched on', async () => { + await setAddonEnabled(orm, 'tours', false); + expect((await admin.list()).status).toBe(200); + expect((await admin.add(REGION.id)).status).toBe(201); + await settle(); + }); +}); diff --git a/server/tests/e2e/tours.e2e.test.ts b/server/tests/e2e/tours.e2e.test.ts index 1ca4f6cf9..29fcf01bc 100644 --- a/server/tests/e2e/tours.e2e.test.ts +++ b/server/tests/e2e/tours.e2e.test.ts @@ -109,6 +109,8 @@ const FLAT_GPX = ` // Large enough that a guard rejecting before multer drained the body would // surface as a connection reset instead of a JSON response. const BIG_GPX = TRACK_GPX.replace('', ``); +/** A paddle plan without marks: enough for the gates, which answer before the body is read. */ +const NO_MARKS = { paddle_speed_kmh: null, portage_speed_kmh: null, track_marks: { portages: [], locks: [] } }; describe('Tours e2e (real guards + temp SQLite)', () => { let server: Server; @@ -221,6 +223,7 @@ describe('Tours e2e (real guards + temp SQLite)', () => { describe('addon and access gates', () => { it('401 without a cookie', async () => { expect((await request(server).get('/api/trips/5/tours')).status).toBe(401); + expect((await request(server).put('/api/trips/5/tours/1/paddle-plan').send(NO_MARKS)).status).toBe(401); }); it('404 on every route while the addon is disabled', async () => { @@ -238,6 +241,7 @@ describe('Tours e2e (real guards + temp SQLite)', () => { .send(tourBody({ name: 'Renamed' })), await importGpx(OWNER, BIG_GPX), await previewGpx(OWNER, BIG_GPX), + await request(server).put(`/api/trips/5/tours/${placeId}/paddle-plan`).set('Cookie', cookie).send(NO_MARKS), ]; for (const res of responses) { expect(res.status).toBe(404); @@ -257,6 +261,7 @@ describe('Tours e2e (real guards + temp SQLite)', () => { await request(server).put(`/api/trips/5/tours/${placeId}`).set('Cookie', cookie).send(tourBody()), await importGpx(OUTSIDER, BIG_GPX), await previewGpx(OUTSIDER, BIG_GPX), + await request(server).put(`/api/trips/5/tours/${placeId}/paddle-plan`).set('Cookie', cookie).send(NO_MARKS), await request(server).get('/api/trips/999/tours').set('Cookie', sessionCookie(OWNER)), ]; for (const res of responses) { @@ -283,6 +288,7 @@ describe('Tours e2e (real guards + temp SQLite)', () => { .send(tourBody({ name: 'Hijacked' })), await importGpx(VIEWER, BIG_GPX), await previewGpx(VIEWER, BIG_GPX), + await request(server).put(`/api/trips/5/tours/${placeId}/paddle-plan`).set('Cookie', cookie).send(NO_MARKS), ]; for (const res of writes) { expect(res.status).toBe(403); @@ -738,14 +744,15 @@ describe('Tours e2e (real guards + temp SQLite)', () => { portage_speed_kmh: null, track_marks: marks, }); - const results = await Promise.all([ - paddlePlan(hike, plan(none)), - paddlePlan(planned, plan(none)), - paddlePlan( + // One after another: supertest binds the unstarted server per request, and requests at once reset each other. + const results = [ + await paddlePlan(hike, plan(none)), + await paddlePlan(planned, plan(none)), + await paddlePlan( track.place_id, plan({ portages: [], locks: [{ at_m: 999999, wait_minutes: 1, transit_minutes: 1 }] }), ), - paddlePlan( + await paddlePlan( track.place_id, plan({ portages: [ @@ -755,7 +762,7 @@ describe('Tours e2e (real guards + temp SQLite)', () => { locks: [], }), ), - ]); + ]; expect(results.map((res) => res.status)).toEqual([400, 400, 400, 400]); expect(results[0].body.code).toBe('not_a_paddle_tour'); expect(results[1].body.code).toBe('planned_tour'); diff --git a/server/tests/helpers/osm-pbf-fixture.ts b/server/tests/helpers/osm-pbf-fixture.ts new file mode 100644 index 000000000..d00bca9ba --- /dev/null +++ b/server/tests/helpers/osm-pbf-fixture.ts @@ -0,0 +1,156 @@ +/** + * A small OpenStreetMap PBF writer for the paddle tests: just enough of the + * format (blob framing, raw and zlib blobs, a header block, string tables, + * plain and dense nodes, ways) to build an extract in the test itself, so + * no binary fixture is checked in. The field numbers are the ones of + * https://wiki.openstreetmap.org/wiki/PBF_Format. + */ +import { deflateSync } from 'node:zlib'; + +export interface FixtureNode { + id: number; + lat: number; + lon: number; + tags?: Record; +} + +export interface FixtureWay { + id: number; + refs: number[]; + tags?: Record; +} + +export function varint(value: number): Buffer { + const bytes: number[] = []; + let rest = value; + while (rest >= 0x80) { + bytes.push((rest % 128) | 0x80); + rest = Math.floor(rest / 128); + } + bytes.push(rest); + return Buffer.from(bytes); +} + +const zigzag = (value: number): number => (value >= 0 ? value * 2 : -value * 2 - 1); + +export function key(field: number, wire: number): Buffer { + return varint(field * 8 + wire); +} + +export function bytesField(field: number, payload: Buffer): Buffer { + return Buffer.concat([key(field, 2), varint(payload.length), payload]); +} + +export function varintField(field: number, value: number): Buffer { + return Buffer.concat([key(field, 0), varint(value)]); +} + +function packed(field: number, values: number[], signed: boolean): Buffer { + return bytesField(field, Buffer.concat(values.map((value) => varint(signed ? zigzag(value) : value)))); +} + +/** Deltas of a run of numbers, as the dense and way encodings store them. */ +function deltas(values: number[]): number[] { + return values.map((value, index) => value - (values[index - 1] ?? 0)); +} + +/** One data block: a string table and one group of dense nodes, plain nodes and ways. */ +export function dataBlock(content: { dense?: FixtureNode[]; nodes?: FixtureNode[]; ways?: FixtureWay[] }): Buffer { + const strings = ['']; + const index = (text: string): number => { + const at = strings.indexOf(text); + if (at >= 0) return at; + strings.push(text); + return strings.length - 1; + }; + const toUnits = (deg: number) => Math.round(deg * 1e7); + const groups: Buffer[] = []; + + if (content.dense?.length) { + const nodes = content.dense; + const keysVals = nodes.flatMap((node) => [ + ...Object.entries(node.tags ?? {}).flatMap(([k, v]) => [index(k), index(v)]), + 0, + ]); + const dense = Buffer.concat([ + packed(1, deltas(nodes.map((node) => node.id)), true), + packed(8, deltas(nodes.map((node) => toUnits(node.lat))), true), + packed(9, deltas(nodes.map((node) => toUnits(node.lon))), true), + packed(10, keysVals, false), + ]); + groups.push(bytesField(2, dense)); + } + for (const node of content.nodes ?? []) { + const tags = Object.entries(node.tags ?? {}); + const body = Buffer.concat([ + Buffer.concat([key(1, 0), varint(zigzag(node.id))]), + packed( + 2, + tags.map(([k]) => index(k)), + false, + ), + packed( + 3, + tags.map(([, v]) => index(v)), + false, + ), + Buffer.concat([key(8, 0), varint(zigzag(toUnits(node.lat)))]), + Buffer.concat([key(9, 0), varint(zigzag(toUnits(node.lon)))]), + ]); + groups.push(bytesField(1, body)); + } + for (const way of content.ways ?? []) { + const tags = Object.entries(way.tags ?? {}); + const body = Buffer.concat([ + varintField(1, way.id), + packed( + 2, + tags.map(([k]) => index(k)), + false, + ), + packed( + 3, + tags.map(([, v]) => index(v)), + false, + ), + packed(8, deltas(way.refs), true), + ]); + groups.push(bytesField(3, body)); + } + + const table = Buffer.concat(strings.map((text) => bytesField(1, Buffer.from(text, 'utf8')))); + return Buffer.concat([bytesField(1, table), ...groups.map((group) => bytesField(2, group))]); +} + +/** A header block, with the replication timestamp when one is given. */ +export function headerBlock(timestamp?: number): Buffer { + const required = bytesField(4, Buffer.from('OsmSchema-V0.6')); + return timestamp === undefined ? required : Buffer.concat([required, varintField(32, timestamp)]); +} + +/** A blob of `type` around `block`, framed as the file holds it: length, header, blob. */ +export function framedBlob( + type: 'OSMHeader' | 'OSMData' | string, + block: Buffer, + compression: 'zlib' | 'raw' = 'zlib', +): Buffer { + const blob = + compression === 'zlib' + ? Buffer.concat([varintField(2, block.length), bytesField(3, deflateSync(block))]) + : bytesField(1, block); + const header = Buffer.concat([bytesField(1, Buffer.from(type, 'utf8')), varintField(3, blob.length)]); + const length = Buffer.alloc(4); + length.writeUInt32BE(header.length, 0); + return Buffer.concat([length, header, blob]); +} + +/** A whole extract: the header block, then one data block per entry. */ +export function pbfFile( + blocks: Array<{ dense?: FixtureNode[]; nodes?: FixtureNode[]; ways?: FixtureWay[] }>, + options: { timestamp?: number; compression?: 'zlib' | 'raw' } = {}, +): Buffer { + return Buffer.concat([ + framedBlob('OSMHeader', headerBlock(options.timestamp), options.compression), + ...blocks.map((block) => framedBlob('OSMData', dataBlock(block), options.compression)), + ]); +} diff --git a/server/tests/unit/db/repositories/PaddleRegions.repository.test.ts b/server/tests/unit/db/repositories/PaddleRegions.repository.test.ts new file mode 100644 index 000000000..0c4002510 --- /dev/null +++ b/server/tests/unit/db/repositories/PaddleRegions.repository.test.ts @@ -0,0 +1,176 @@ +/** + * The two repositories of PAD-05A: `paddle_regions` (the regions an admin + * chose, their queue and their state) and `paddle_features` (a region's + * objects, found by box and replaced whole). + */ +import { PaddleFeatures } from '../../../../src/db/entities/PaddleFeatures.entity'; +import { PaddleRegions } from '../../../../src/db/entities/PaddleRegions.entity'; +import type { + PaddleFeatureRow, + PaddleFeaturesRepository, +} from '../../../../src/db/repositories/PaddleFeatures.repository'; +import type { PaddleRegionsRepository } from '../../../../src/db/repositories/PaddleRegions.repository'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { countRows, findRow, findRows, updateRows } from '../../../helpers/factories/rows'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const testDb = createSnapshotTestDb(); +let t: TestOrm; +let regions: PaddleRegionsRepository; +let features: PaddleFeaturesRepository; + +beforeAll(async () => { + t = await createTestOrm(testDb); + regions = t.repo(PaddleRegions); + features = t.repo(PaddleFeatures); +}); +beforeEach(() => { + resetTestDb(testDb); + t.clear(); +}); +afterAll(async () => { + await t.close(); + testDb.close(); +}); + +const box = { min_lat: 49, min_lng: 12, max_lat: 50, max_lng: 13 }; +const add = (id: string, name = id) => regions.insertRegion({ id, name, boundary: null, ...box }); + +const feature = (ref: string, lat: number, lng: number): PaddleFeatureRow => ({ + region_id: 'ignored', + osm_ref: ref, + kind: 'weir', + group_key: null, + name: null, + geometry: JSON.stringify({ type: 'Point', coordinates: [lng, lat] }), + tags: '{}', + tag_hash: ref, + min_lat: lat, + min_lng: lng, + max_lat: lat, + max_lng: lng, +}); + +describe('PaddleRegionsRepository', () => { + it('PADREGREPO-001: a region is added queued and empty; adding it again changes nothing', async () => { + await add('oberpfalz', 'Oberpfalz'); + await regions.updateRegion('oberpfalz', { status: 'ready', feature_count: 3 }); + await add('oberpfalz', 'Renamed'); + + expect(await regions.findById('oberpfalz')).toMatchObject({ + id: 'oberpfalz', + name: 'Oberpfalz', + status: 'ready', + feature_count: 3, + ...box, + }); + expect(await regions.findById('nowhere')).toBeUndefined(); + }); + + it('PADREGREPO-002: lists by name; the queue answers the oldest waiting region first', async () => { + await add('b', 'Bayern'); + await add('a', 'Allgäu'); + await updateRows(t, PaddleRegions, { id: 'b' }, { created_at: '2026-01-01 00:00:00' }); + await updateRows(t, PaddleRegions, { id: 'a' }, { created_at: '2026-02-01 00:00:00' }); + + expect((await regions.list()).map((row) => row.name)).toEqual(['Allgäu', 'Bayern']); + expect((await regions.nextQueued())?.id).toBe('b'); + await regions.updateRegion('b', { status: 'importing' }); + expect((await regions.nextQueued())?.id).toBe('a'); + await regions.updateRegion('a', { status: 'ready' }); + expect(await regions.nextQueued()).toBeUndefined(); + }); + + it('PADREGREPO-003: only regions holding data whose box reaches in count for a box', async () => { + await add('read'); + await add('waiting'); + await add('elsewhere'); + await regions.updateRegion('read', { status: 'importing', imported_at: '2026-10-02 04:20:00' }); + await regions.updateRegion('elsewhere', { + imported_at: '2026-10-02 04:20:00', + min_lat: 53, + max_lat: 54, + min_lng: 12, + max_lng: 13, + }); + + const inBox = await regions.withDataInBox({ south: 49.5, west: 12.5, north: 49.6, east: 12.6 }); + expect(inBox.map((row) => row.id)).toEqual(['read']); + }); + + it('PADREGREPO-004: the monthly refresh queues ready and failed regions and clears their error', async () => { + await add('ready'); + await add('failed'); + await add('queued'); + await regions.updateRegion('ready', { status: 'ready' }); + await regions.updateRegion('failed', { status: 'failed', error: 'Geofabrik download 500' }); + + expect(await regions.queueReady()).toBe(2); + expect(await findRows(t, PaddleRegions, {}, { id: 'asc' })).toMatchObject([ + { id: 'failed', status: 'queued', error: null }, + { id: 'queued', status: 'queued' }, + { id: 'ready', status: 'queued' }, + ]); + }); + + it('PADREGREPO-005: a cut read goes back to the queue; removing a region takes its objects', async () => { + await add('oberpfalz'); + await regions.updateRegion('oberpfalz', { status: 'importing' }); + await regions.requeueInterrupted(); + expect((await regions.findById('oberpfalz'))?.status).toBe('queued'); + + await features.replaceRegion('oberpfalz', [feature('node/1', 49.5, 12.5)]); + expect(await regions.deleteRegion('oberpfalz')).toBe(true); + expect(await regions.deleteRegion('oberpfalz')).toBe(false); + expect(await countRows(t, PaddleFeatures)).toBe(0); + }); +}); + +describe('PaddleFeaturesRepository', () => { + it('PADFEATREPO-001: replaces a region whole, chunk by chunk, and leaves other regions alone', async () => { + await add('oberpfalz'); + await add('bayern'); + await features.replaceRegion('bayern', [feature('node/9', 49.5, 12.5)]); + const many = Array.from({ length: 450 }, (_, i) => feature(`node/${i + 1}`, 49 + i / 1000, 12.5)); + await features.replaceRegion('oberpfalz', many); + expect(await countRows(t, PaddleFeatures, { region: 'oberpfalz' })).toBe(450); + + await features.replaceRegion('oberpfalz', [feature('node/1', 49.5, 12.5), feature('node/1', 49.5, 12.5)]); + expect(await countRows(t, PaddleFeatures, { region: 'oberpfalz' })).toBe(1); + expect(await countRows(t, PaddleFeatures, { region: 'bayern' })).toBe(1); + expect(await findRow(t, PaddleFeatures, { region: 'oberpfalz' })).toMatchObject({ osm_ref: 'node/1' }); + + await features.replaceRegion('oberpfalz', []); + expect(await countRows(t, PaddleFeatures, { region: 'oberpfalz' })).toBe(0); + }); + + it('PADFEATREPO-002: finds the objects whose box reaches into the box, in a stable order, at most the limit', async () => { + await add('oberpfalz'); + await features.replaceRegion('oberpfalz', [ + feature('node/1', 49.5, 12.5), + feature('node/2', 49.55, 12.55), + feature('node/3', 49.9, 12.9), + { + ...feature('way/4', 49.4, 12.4), + geometry: JSON.stringify({ + type: 'LineString', + coordinates: [ + [12.4, 49.4], + [12.6, 49.6], + ], + }), + max_lat: 49.6, + max_lng: 12.6, + }, + ]); + + const inside = await features.findInBox({ south: 49.45, west: 12.45, north: 49.6, east: 12.6 }, 10); + expect(inside.map((row) => row.osm_ref)).toEqual(['node/1', 'node/2', 'way/4']); + expect(inside[0]).toMatchObject({ region_id: 'oberpfalz', kind: 'weir', tag_hash: 'node/1' }); + const capped = await features.findInBox({ south: 49, west: 12, north: 50, east: 13 }, 2); + expect(capped.map((row) => row.osm_ref)).toEqual(['node/1', 'node/2']); + }); +}); diff --git a/server/tests/unit/db/repositories/Tours.repository.test.ts b/server/tests/unit/db/repositories/Tours.repository.test.ts index e85e3d276..11722b4db 100644 --- a/server/tests/unit/db/repositories/Tours.repository.test.ts +++ b/server/tests/unit/db/repositories/Tours.repository.test.ts @@ -212,7 +212,13 @@ describe('ToursRepository writes', () => { }); // A route edit always writes the type's own fields; a hike's are empty. - const HIKE_UPDATE = { ...METRICS, boat_type: null, water_context: null, paddle_speed_kmh: null, portage_speed_kmh: null }; + const HIKE_UPDATE = { + ...METRICS, + boat_type: null, + water_context: null, + paddle_speed_kmh: null, + portage_speed_kmh: null, + }; it('TOURREPO-007: updateInTrip replaces the metrics of a tour of that trip and refuses one of another trip', async () => { const { trip, other } = fixture(); diff --git a/server/tests/unit/nest/paddle-obstacles/geofabrik.client.test.ts b/server/tests/unit/nest/paddle-obstacles/geofabrik.client.test.ts new file mode 100644 index 000000000..90a4671d9 --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/geofabrik.client.test.ts @@ -0,0 +1,232 @@ +/** + * The outbound half of PAD-05A against a faked fetch: Geofabrik's list read + * and cached, the extracts only from Geofabrik's host, redirects only within + * it, the download counted against its cap and its file gone on failure. + */ +import { + GEOFABRIK_HOST, + GeofabrikClient, + isExtractUrl, + MAX_EXTRACT_BYTES, +} from '../../../../src/nest/paddle-obstacles/geofabrik.client'; + +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const EXTRACT = `${GEOFABRIK_HOST}europe/germany/bayern/oberpfalz-latest.osm.pbf`; +const square = { + type: 'Polygon', + coordinates: [ + [ + [11.5, 48.9], + [13, 48.9], + [13, 50], + [11.5, 50], + [11.5, 48.9], + ], + ], +}; +const index = { + features: [ + { + properties: { + id: 'bayern', + parent: 'germany', + name: 'Bayern', + urls: { pbf: `${GEOFABRIK_HOST}europe/germany/bayern-latest.osm.pbf` }, + }, + geometry: null, + }, + { properties: { id: 'oberpfalz', parent: 'bayern', name: 'Oberpfalz', urls: { pbf: EXTRACT } }, geometry: square }, + { + properties: { id: 'elsewhere', name: 'Elsewhere', urls: { pbf: 'https://evil.example/x-latest.osm.pbf' } }, + geometry: null, + }, + { + properties: { id: 'dated', name: 'Dated', urls: { pbf: `${GEOFABRIK_HOST}dated-230101.osm.pbf` } }, + geometry: null, + }, + { properties: { id: 'nofile', name: 'No file', urls: {} }, geometry: null }, + ], +}; + +const fetchMock = vi.fn<(url: string, init?: RequestInit) => Promise>(); +let client: GeofabrikClient; +let dir: string; +let n = 0; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-geofabrik-')); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal('fetch', fetchMock); + client = new GeofabrikClient(); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { status }); +const redirect = (location: string) => new Response(null, { status: 302, headers: { location } }); +const target = () => join(dir, `download-${n++}.osm.pbf`); +const exists = (path: string) => + stat(path).then( + () => true, + () => false, + ); + +describe('isExtractUrl', () => { + it('GEOF-001: takes the latest extract on Geofabrik and nothing else', () => { + expect(isExtractUrl(EXTRACT)).toBe(true); + expect(isExtractUrl('https://evil.example/oberpfalz-latest.osm.pbf')).toBe(false); + expect(isExtractUrl(`${GEOFABRIK_HOST}oberpfalz-230101.osm.pbf`)).toBe(false); + expect(isExtractUrl('http://download.geofabrik.de/oberpfalz-latest.osm.pbf')).toBe(false); + }); +}); + +describe('GeofabrikClient.regions', () => { + it('GEOF-002: reads the list with a deadline, keeping the regions with a latest extract on its host', async () => { + fetchMock.mockResolvedValueOnce(json(index)); + + const regions = await client.regions(1000); + + expect([...regions.keys()]).toEqual(['bayern', 'oberpfalz']); + expect(regions.get('oberpfalz')).toEqual({ + id: 'oberpfalz', + name: 'Oberpfalz', + parent: 'bayern', + pbfUrl: EXTRACT, + boundary: square, + }); + expect(regions.get('bayern')?.boundary).toBeNull(); + const [url, init] = fetchMock.mock.calls[0] ?? []; + expect(url).toBe(`${GEOFABRIK_HOST}index-v1.json`); + expect(init?.signal).toBeInstanceOf(AbortSignal); + expect(init?.redirect).toBe('manual'); + }); + + it('GEOF-003: keeps the list for a day, then asks again', async () => { + fetchMock.mockImplementation(() => Promise.resolve(json(index))); + await client.regions(0); + await client.regions(23 * 60 * 60 * 1000); + expect(fetchMock).toHaveBeenCalledTimes(1); + await client.regions(25 * 60 * 60 * 1000); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('GEOF-004: refuses an answer that is not the list, or not a success', async () => { + fetchMock.mockResolvedValueOnce(json({ type: 'FeatureCollection' })); + await expect(client.regions()).rejects.toThrow('Geofabrik index is not the list of extracts'); + fetchMock.mockResolvedValueOnce(new Response('', { status: 200 })); + await expect(client.regions()).rejects.toThrow('Geofabrik index is not the list of extracts'); + fetchMock.mockResolvedValueOnce(json({}, 503)); + await expect(client.regions()).rejects.toThrow('Geofabrik index 503'); + }); + + it('GEOF-005: follows a redirect within Geofabrik, refuses one away from it or too many', async () => { + fetchMock + .mockResolvedValueOnce(redirect('https://mirror.geofabrik.de/index-v1.json')) + .mockResolvedValueOnce(json(index)); + expect((await client.regions(0)).size).toBe(2); + expect(fetchMock.mock.calls[1]?.[0]).toBe('https://mirror.geofabrik.de/index-v1.json'); + + fetchMock.mockReset(); + fetchMock.mockResolvedValueOnce(redirect('https://evil.example/index-v1.json')); + await expect(new GeofabrikClient().regions()).rejects.toThrow('Geofabrik redirected away from its own servers'); + + fetchMock.mockReset(); + fetchMock.mockResolvedValueOnce(redirect('http://download.geofabrik.de/index-v1.json')); + await expect(new GeofabrikClient().regions()).rejects.toThrow('Geofabrik redirected away from its own servers'); + + fetchMock.mockReset(); + fetchMock.mockResolvedValueOnce(new Response(null, { status: 301 })); + await expect(new GeofabrikClient().regions()).rejects.toThrow('Geofabrik redirected away from its own servers'); + + fetchMock.mockReset(); + fetchMock.mockImplementation(() => Promise.resolve(redirect('/again'))); + await expect(new GeofabrikClient().regions()).rejects.toThrow('Geofabrik redirected too often'); + // Three redirects are followed; the fourth is one too many. + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it('GEOF-006: a timeout reaches the caller as an error', async () => { + fetchMock.mockRejectedValueOnce(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + await expect(client.regions()).rejects.toThrow('timeout'); + }); +}); + +describe('GeofabrikClient.download', () => { + it('GEOF-007: writes the extract to the target and tells the share received', async () => { + const body = Buffer.alloc(3000, 7); + fetchMock.mockResolvedValueOnce(new Response(body, { status: 200, headers: { 'content-length': '3000' } })); + const progress: number[] = []; + const path = target(); + + await client.download(EXTRACT, path, { onProgress: (fraction) => progress.push(fraction) }); + + expect(await readFile(path)).toEqual(body); + expect(progress.at(-1)).toBe(1); + expect(fetchMock.mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal); + }); + + it('GEOF-008: refuses an address the list could not have named, without asking', async () => { + await expect(client.download('https://evil.example/x-latest.osm.pbf', target())).rejects.toThrow( + 'Not a Geofabrik extract', + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('GEOF-009: refuses an extract that says it is larger than allowed, before reading it', async () => { + const response = new Response('x', { status: 200, headers: { 'content-length': String(MAX_EXTRACT_BYTES + 1) } }); + fetchMock.mockResolvedValueOnce(response); + const path = target(); + await expect(client.download(EXTRACT, path)).rejects.toThrow('Extract larger than allowed'); + expect(await exists(path)).toBe(false); + }); + + it('GEOF-010: cuts an extract that streams past the cap and removes the file', async () => { + fetchMock.mockResolvedValueOnce(new Response(Buffer.alloc(5000, 1), { status: 200 })); + const path = target(); + await expect(client.download(EXTRACT, path, { maxBytes: 1000 })).rejects.toThrow('Extract larger than allowed'); + expect(await exists(path)).toBe(false); + }); + + it('GEOF-011: a failed answer or a call-off ends the download without a file', async () => { + fetchMock.mockResolvedValueOnce(new Response('gone', { status: 404 })); + const missing = target(); + await expect(client.download(EXTRACT, missing)).rejects.toThrow('Geofabrik download 404'); + expect(await exists(missing)).toBe(false); + + const abort = new AbortController(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(100)); + }, + }); + fetchMock.mockResolvedValueOnce(new Response(stream, { status: 200 })); + const path = target(); + const download = client.download(EXTRACT, path, { signal: abort.signal }); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalled()); + abort.abort(); + await expect(download).rejects.toThrow(); + expect(await exists(path)).toBe(false); + }); + + it('GEOF-012: follows a redirect to another Geofabrik server for the extract', async () => { + fetchMock + .mockResolvedValueOnce(redirect('https://mirror.geofabrik.de/europe/oberpfalz-latest.osm.pbf')) + .mockResolvedValueOnce(new Response(Buffer.from('pbf'), { status: 200 })); + const path = target(); + await client.download(EXTRACT, path); + expect((await readFile(path)).toString()).toBe('pbf'); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/osm-pbf.reader.test.ts b/server/tests/unit/nest/paddle-obstacles/osm-pbf.reader.test.ts new file mode 100644 index 000000000..29194f3ab --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/osm-pbf.reader.test.ts @@ -0,0 +1,206 @@ +/** + * The PBF reader over extracts the test writes itself (tests/helpers/osm-pbf-fixture.ts): + * blob framing across read chunks, raw and zlib blobs, the header timestamp, + * plain and dense nodes, ways, and the ways a broken file is refused. + */ +import { + readDataBlock, + readHeaderTimestamp, + readPbfFile, + type OsmNode, + type OsmWay, +} from '../../../../src/nest/paddle-obstacles/osm-pbf.reader'; +import { + bytesField, + dataBlock, + framedBlob, + headerBlock, + key, + pbfFile, + varint, + varintField, +} from '../../../helpers/osm-pbf-fixture'; + +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +let dir: string; +let n = 0; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-pbf-')); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function fileOf(bytes: Buffer): Promise { + const path = join(dir, `extract-${n++}.osm.pbf`); + await writeFile(path, bytes); + return path; +} + +async function readAll(path: string, onProgress?: (bytes: number) => void) { + const nodes: OsmNode[] = []; + const ways: OsmWay[] = []; + const result = await readPbfFile( + path, + { node: (node) => nodes.push(node), way: (way) => ways.push(way) }, + onProgress, + ); + return { ...result, nodes, ways }; +} + +describe('readPbfFile', () => { + it('PBF-001: reads the header timestamp, dense nodes with their tags, plain nodes and ways', async () => { + const path = await fileOf( + pbfFile( + [ + { + dense: [ + { id: 10, lat: 49.1, lon: 12.1, tags: { waterway: 'weir', name: 'Wehr' } }, + { id: 11, lat: 49.2, lon: 12.2 }, + { id: 15, lat: 49.15, lon: 12.05, tags: { canoe: 'put_in' } }, + ], + nodes: [{ id: 20, lat: -33.9, lon: -70.6, tags: { waterway: 'dam' } }], + ways: [{ id: 29256405, refs: [10, 11, 15], tags: { waterway: 'weir', whitewater: 'hazard' } }], + }, + ], + { timestamp: 1_760_000_000 }, + ), + ); + + const { timestamp, nodes, ways } = await readAll(path); + + expect(timestamp).toBe(1_760_000_000); + expect(nodes.map((node) => node.id)).toEqual([10, 11, 15, 20]); + expect(nodes[0]).toMatchObject({ tags: { waterway: 'weir', name: 'Wehr' } }); + expect(nodes[0]?.lat).toBeCloseTo(49.1, 7); + expect(nodes[0]?.lon).toBeCloseTo(12.1, 7); + expect(nodes[1]?.tags).toEqual({}); + expect(nodes[2]).toMatchObject({ tags: { canoe: 'put_in' } }); + expect(nodes[3]?.lat).toBeCloseTo(-33.9, 7); + expect(nodes[3]?.lon).toBeCloseTo(-70.6, 7); + expect(ways).toEqual([{ id: 29256405, refs: [10, 11, 15], tags: { waterway: 'weir', whitewater: 'hazard' } }]); + }); + + it('PBF-002: reads raw blobs as it reads zlib ones', async () => { + const path = await fileOf( + pbfFile([{ dense: [{ id: 1, lat: 1, lon: 2, tags: { lock: 'yes' } }] }], { compression: 'raw' }), + ); + const { nodes, timestamp } = await readAll(path); + expect(timestamp).toBeNull(); + expect(nodes).toHaveLength(1); + expect(nodes[0]?.tags).toEqual({ lock: 'yes' }); + }); + + it('PBF-003: reads a blob that reaches across read chunks, and ids past 2^31', async () => { + // A string over a megabyte makes the block span several of the reader's quarter-megabyte chunks. + const long = 'x'.repeat(1_200_000); + const path = await fileOf( + pbfFile( + [ + { dense: [{ id: 3_000_000_000, lat: 50, lon: 8, tags: { note: long, waterway: 'weir' } }] }, + { ways: [{ id: 4_000_000_000, refs: [3_000_000_000, 3_000_000_001] }] }, + ], + { compression: 'raw' }, + ), + ); + const progress: number[] = []; + + const { nodes, ways } = await readAll(path, (bytes) => progress.push(bytes)); + + expect(nodes[0]?.id).toBe(3_000_000_000); + expect(nodes[0]?.tags.note).toHaveLength(1_200_000); + expect(ways[0]).toMatchObject({ id: 4_000_000_000, refs: [3_000_000_000, 3_000_000_001] }); + expect(progress.length).toBeGreaterThan(2); + expect([...progress].sort((a, b) => a - b)).toEqual(progress); + }); + + it('PBF-004: skips blocks of a type it does not know', async () => { + const path = await fileOf( + Buffer.concat([ + framedBlob('OSMHeader', headerBlock(5)), + framedBlob('Unknown', Buffer.from('anything')), + framedBlob('OSMData', dataBlock({ dense: [{ id: 7, lat: 0, lon: 0 }] })), + ]), + ); + const { nodes, timestamp } = await readAll(path); + expect(timestamp).toBe(5); + expect(nodes.map((node) => node.id)).toEqual([7]); + }); + + it('PBF-005: refuses a file that ends inside a blob', async () => { + const whole = pbfFile([{ dense: [{ id: 1, lat: 1, lon: 1 }] }]); + await expect(readAll(await fileOf(whole.subarray(0, whole.length - 3)))).rejects.toThrow('Truncated blob'); + await expect(readAll(await fileOf(whole.subarray(0, 2)))).rejects.toThrow('Truncated blob'); + }); + + it('PBF-006: refuses a blob header or a blob larger than the format allows', async () => { + const huge = Buffer.alloc(4); + huge.writeUInt32BE(70_000, 0); + await expect(readAll(await fileOf(huge))).rejects.toThrow('Blob header too large'); + + const header = Buffer.concat([bytesField(1, Buffer.from('OSMData')), varintField(3, 40 * 1024 * 1024)]); + const length = Buffer.alloc(4); + length.writeUInt32BE(header.length, 0); + await expect(readAll(await fileOf(Buffer.concat([length, header])))).rejects.toThrow('Blob too large'); + }); + + it('PBF-007: refuses a blob in a compression it does not read', async () => { + const lzma = bytesField(4, Buffer.from('not zlib')); + const header = Buffer.concat([bytesField(1, Buffer.from('OSMData')), varintField(3, lzma.length)]); + const length = Buffer.alloc(4); + length.writeUInt32BE(header.length, 0); + await expect(readAll(await fileOf(Buffer.concat([length, header, lzma])))).rejects.toThrow( + 'Unsupported blob compression', + ); + }); + + it('PBF-008: an empty file reads as no blocks', async () => { + expect(await readAll(await fileOf(Buffer.alloc(0)))).toEqual({ timestamp: null, nodes: [], ways: [] }); + }); +}); + +describe('readDataBlock and readHeaderTimestamp', () => { + it('PBF-009: applies the block granularity and offsets', () => { + // granularity 1000 and offsets of one and two degrees (plain int64 in nanodegrees). + const block = Buffer.concat([ + dataBlock({ nodes: [{ id: 1, lat: 0.0001, lon: 0.0002 }] }), + varintField(17, 1000), + varintField(19, 1_000_000_000), + varintField(20, 2_000_000_000), + ]); + const nodes: OsmNode[] = []; + readDataBlock(block, { node: (node) => nodes.push(node) }); + // lat = (offset 1e9 + granularity 1000 * stored 1000) * 1e-9 = 1.001. + expect(nodes[0]?.lat).toBeCloseTo(1.001, 9); + expect(nodes[0]?.lon).toBeCloseTo(2.002, 9); + }); + + it('PBF-010: leaves out what the visitor does not ask for', () => { + const block = dataBlock({ dense: [{ id: 1, lat: 0, lon: 0 }], ways: [{ id: 2, refs: [1] }] }); + const ways: OsmWay[] = []; + readDataBlock(block, { way: (way) => ways.push(way) }); + expect(ways.map((way) => way.id)).toEqual([2]); + }); + + it('PBF-011: a header block without a timestamp answers null; fields it does not know are skipped', () => { + expect(readHeaderTimestamp(headerBlock())).toBeNull(); + const withFixed = Buffer.concat([key(30, 1), Buffer.alloc(8), key(31, 5), Buffer.alloc(4), varintField(32, 99)]); + expect(readHeaderTimestamp(withFixed)).toBe(99); + }); + + it('PBF-012: refuses a field of a wire type the format never uses, a cut or endless varint and a cut field', () => { + expect(() => readHeaderTimestamp(key(3, 3))).toThrow('Unsupported wire type 3'); + expect(() => readHeaderTimestamp(Buffer.from([0x80]))).toThrow('Truncated varint'); + expect(() => readHeaderTimestamp(Buffer.alloc(11, 0x80))).toThrow('Malformed varint'); + expect(() => readHeaderTimestamp(Buffer.concat([key(30, 1), Buffer.alloc(3)]))).toThrow('Truncated field'); + expect(() => readHeaderTimestamp(Buffer.concat([key(4, 2), varint(10), Buffer.from('ab')]))).toThrow( + 'Truncated field', + ); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-extract.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-extract.test.ts new file mode 100644 index 000000000..95160bacd --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-extract.test.ts @@ -0,0 +1,188 @@ +/** + * The two-pass extraction of PAD-05A over extracts written in the test: the + * objects kept with their own geometry (a weir across a river stays a line), + * their tag hash, and the parts of one lock grouped while separate locks + * stay apart. + */ +import { + extractPaddleFeatures, + groupLockParts, + type ExtractedFeature, +} from '../../../../src/nest/paddle-obstacles/paddle-extract'; +import { pbfFile, type FixtureNode } from '../../../helpers/osm-pbf-fixture'; + +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +let dir: string; +let n = 0; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-paddle-extract-')); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function extract(blocks: Parameters[0], timestamp?: number) { + const path = join(dir, `region-${n++}.osm.pbf`); + await writeFile(path, pbfFile(blocks, { timestamp })); + const progress: number[] = []; + const result = await extractPaddleFeatures(path, (fraction) => progress.push(fraction)); + return { ...result, progress }; +} + +const byRef = (features: ExtractedFeature[], ref: string) => features.find((feature) => feature.osm_ref === ref); + +/** The Regen at Lindenmühle, Regenstauf: the weir way 29256405 drawn across the river. */ +const weirNodes: FixtureNode[] = [ + { id: 1, lat: 49.1231, lon: 12.1301 }, + { id: 2, lat: 49.1236, lon: 12.1309 }, + { id: 3, lat: 49.124, lon: 12.1315 }, +]; + +describe('extractPaddleFeatures', () => { + it('PEXT-001: keeps a weir way as a line with its own geometry, box and kept tags', async () => { + const { features, timestamp, progress } = await extract( + [ + { dense: weirNodes }, + { + ways: [ + { + id: 29256405, + refs: [1, 2, 3], + tags: { waterway: 'weir', name: 'Wehr Lindenmühle', whitewater: 'hazard', source: 'bing' }, + }, + ], + }, + ], + 1_760_000_000, + ); + + expect(timestamp).toBe(1_760_000_000); + const weir = byRef(features, 'way/29256405'); + expect(weir).toMatchObject({ + kind: 'weir', + name: 'Wehr Lindenmühle', + tags: { waterway: 'weir', name: 'Wehr Lindenmühle', whitewater: 'hazard' }, + group_key: null, + }); + expect(weir?.geometry.type).toBe('LineString'); + expect(weir?.geometry.coordinates).toHaveLength(3); + expect(weir?.min_lat).toBeCloseTo(49.1231, 6); + expect(weir?.max_lat).toBeCloseTo(49.124, 6); + expect(weir?.min_lng).toBeCloseTo(12.1301, 6); + expect(weir?.max_lng).toBeCloseTo(12.1315, 6); + expect(weir?.tag_hash).toMatch(/^[0-9a-f]{16}$/); + expect(progress.at(-1)).toBe(1); + expect([...progress].sort((a, b) => a - b)).toEqual(progress); + }); + + it('PEXT-002: keeps tagged nodes that are paddle objects as points, and leaves the rest', async () => { + const { features, progress } = await extract([ + { + dense: [ + { id: 10, lat: 53.27, lon: 12.81, tags: { canoe: 'put_in', name: 'Einsetzstelle Mirow' } }, + { id: 11, lat: 53.28, lon: 12.82, tags: { amenity: 'bench' } }, + { id: 12, lat: 53.29, lon: 12.83 }, + ], + }, + ]); + expect(features.map((feature) => feature.osm_ref)).toEqual(['node/10']); + expect(features[0]).toMatchObject({ kind: 'put_in', geometry: { type: 'Point' } }); + expect(features[0]?.min_lat).toBe(features[0]?.max_lat); + // No way to look up: the second pass is skipped and the progress jumps to the end. + expect(progress.at(-1)).toBe(1); + }); + + it('PEXT-003: a closed way is an area, a way with fewer than two known nodes is dropped', async () => { + const { features } = await extract([ + { + dense: [ + { id: 1, lat: 0, lon: 0 }, + { id: 2, lat: 0, lon: 0.001 }, + { id: 3, lat: 0.001, lon: 0.001 }, + ], + }, + { + ways: [ + { id: 100, refs: [1, 2, 3, 1], tags: { lock: 'yes', waterway: 'canal' } }, + { id: 101, refs: [1, 99], tags: { waterway: 'dam' } }, + ], + }, + ]); + expect(byRef(features, 'way/100')?.geometry.type).toBe('Polygon'); + expect(byRef(features, 'way/101')).toBeUndefined(); + }); + + it('PEXT-004: thins a very long way to every n-th node and keeps both of its ends', async () => { + const nodes = Array.from({ length: 4501 }, (_, i) => ({ id: i + 1, lat: 50 + i * 1e-5, lon: 8 })); + const { features } = await extract([ + { dense: nodes }, + { ways: [{ id: 7, refs: nodes.map((node) => node.id), tags: { waterway: 'river', canoe: 'yes' } }] }, + ]); + const line = byRef(features, 'way/7'); + expect(line?.kind).toBe('access'); + const coordinates = line?.geometry.type === 'LineString' ? line.geometry.coordinates : []; + expect(coordinates.length).toBeLessThanOrEqual(2001); + expect(coordinates[0]?.[1]).toBeCloseTo(50, 6); + expect(coordinates.at(-1)?.[1]).toBeCloseTo(50 + 4500 * 1e-5, 6); + }); + + it('PEXT-005: the hash changes with a kept tag and with the geometry, not with a dropped tag', async () => { + const weir = (tags: Record, lat = 49.1) => + extract([{ dense: [{ id: 5, lat, lon: 12, tags: { waterway: 'weir', ...tags } }] }]).then( + (result) => result.features[0]?.tag_hash, + ); + const base = await weir({ name: 'A' }); + expect(await weir({ name: 'A' })).toBe(base); + expect(await weir({ name: 'A', source: 'survey' })).toBe(base); + expect(await weir({ name: 'B' })).not.toBe(base); + expect(await weir({ name: 'A', canoe: 'no' })).not.toBe(base); + expect(await weir({ name: 'A' }, 49.2)).not.toBe(base); + }); +}); + +describe('groupLockParts', () => { + const part = (ref: string, kind: ExtractedFeature['kind'], lat: number, lng: number): ExtractedFeature => ({ + osm_ref: ref, + kind, + name: null, + geometry: { type: 'Point', coordinates: [lng, lat] }, + tags: {}, + tag_hash: ref, + group_key: null, + min_lat: lat, + max_lat: lat, + min_lng: lng, + max_lng: lng, + }); + + it('PEXT-006: the chamber and the gates of one lock share a key; a lock 2 km on stays apart', () => { + const chamber = part('way/30', 'lock', 53.3, 13.0); + const upper = part('node/31', 'lock_gate', 53.3, 13.0012); + const lower = part('node/32', 'lock_gate', 53.3, 12.9988); + const other = part('way/40', 'lock', 53.3, 13.03); + const weir = part('way/50', 'weir', 53.3, 13.0001); + const features = [chamber, upper, lower, other, weir]; + + groupLockParts(features); + + expect(chamber.group_key).toBe('lock:node/31'); + expect(upper.group_key).toBe('lock:node/31'); + expect(lower.group_key).toBe('lock:node/31'); + expect(other.group_key).toBeNull(); + expect(weir.group_key).toBeNull(); + }); + + it('PEXT-007: parts chained within reach of each other form one lock', () => { + const a = part('node/1', 'lock_gate', 0, 0); + const b = part('node/2', 'lock_gate', 0, 0.002); + const c = part('node/3', 'lock_gate', 0, 0.004); + groupLockParts([c, a, b]); + expect(new Set([a.group_key, b.group_key, c.group_key])).toEqual(new Set(['lock:node/1'])); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-geo.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-geo.test.ts new file mode 100644 index 000000000..c2912010c --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-geo.test.ts @@ -0,0 +1,156 @@ +/** + * The geometry of PAD-05A's reading side: the box a request asks for, whether + * a point lies in a region's boundary, and how far the data covers a box. + */ +import { + boundaryContains, + boxOfBoundary, + coverageOf, + parseBox, + type RegionBoundary, +} from '../../../../src/nest/paddle-obstacles/paddle-geo'; + +import { describe, expect, it } from 'vitest'; + +/** A square region 49 to 50 north and 11 to 13 east, with a hole 49.4 to 49.6 north, 11.8 to 12.2 east. */ +const holed: RegionBoundary = { + type: 'Polygon', + coordinates: [ + [ + [11, 49], + [13, 49], + [13, 50], + [11, 50], + [11, 49], + ], + [ + [11.8, 49.4], + [12.2, 49.4], + [12.2, 49.6], + [11.8, 49.6], + [11.8, 49.4], + ], + ], +}; + +const twoIslands: RegionBoundary = { + type: 'MultiPolygon', + coordinates: [ + [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ], + [ + [ + [5, 5], + [6, 5], + [6, 6], + [5, 6], + [5, 5], + ], + ], + ], +}; + +describe('parseBox', () => { + it('PGEO-001: reads south,west,north,east', () => { + expect(parseBox('49.1,12.0,49.3,12.4')).toEqual({ south: 49.1, west: 12, north: 49.3, east: 12.4 }); + expect(parseBox('-34,-71,-33.5,-70.5')).toEqual({ south: -34, west: -71, north: -33.5, east: -70.5 }); + }); + + it.each([ + ['49.1,12.0,49.3', 'a corner missing'], + ['49.1,12.0,49.3,abc', 'a part that is no number'], + ['49.3,12.0,49.1,12.4', 'south above north'], + ['49.1,12.4,49.3,12.0', 'west east of east'], + ['89.5,0,91,0.5', 'a latitude past the pole'], + ['0,179.5,0.5,181', 'a longitude past the date line'], + ['49,12,50.5,12.5', 'more than one degree north to south'], + ['49,12,49.5,13.5', 'more than one degree west to east'], + ])('PGEO-002: refuses %s (%s)', (text) => { + expect(parseBox(text)).toBeNull(); + }); + + it('PGEO-003: takes a box of exactly one degree a side', () => { + expect(parseBox('49,12,50,13')).toEqual({ south: 49, west: 12, north: 50, east: 13 }); + }); +}); + +describe('boundaryContains', () => { + it('PGEO-004: a point in the outer ring and in no hole is inside', () => { + expect(boundaryContains(holed, 49.2, 11.5)).toBe(true); + expect(boundaryContains(holed, 49.5, 12)).toBe(false); + expect(boundaryContains(holed, 48.9, 12)).toBe(false); + }); + + it('PGEO-005: a point in any polygon of a multipolygon is inside', () => { + expect(boundaryContains(twoIslands, 0.5, 0.5)).toBe(true); + expect(boundaryContains(twoIslands, 5.5, 5.5)).toBe(true); + expect(boundaryContains(twoIslands, 3, 3)).toBe(false); + }); + + it('PGEO-006: a polygon without rings contains nothing', () => { + expect(boundaryContains({ type: 'Polygon', coordinates: [] }, 0, 0)).toBe(false); + }); +}); + +describe('coverageOf', () => { + it('PGEO-007: without a region the area is not covered at all', () => { + expect(coverageOf({ south: 49, west: 11, north: 49.1, east: 11.1 }, [])).toBe('none'); + }); + + it('PGEO-008: a box wholly inside a boundary is covered', () => { + expect(coverageOf({ south: 49.1, west: 11.1, north: 49.2, east: 11.3 }, [holed])).toBe('covered'); + }); + + it('PGEO-009: a box reaching past the edge, or into a hole, is partly covered', () => { + expect(coverageOf({ south: 49.9, west: 12.9, north: 50.1, east: 13.1 }, [holed])).toBe('partial'); + expect(coverageOf({ south: 49.45, west: 11.9, north: 49.55, east: 12.1 }, [holed])).toBe('partial'); + }); + + it('PGEO-010: two regions together can cover a box neither covers alone', () => { + const west: RegionBoundary = { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 1], + [0, 0], + ], + ], + }; + const east: RegionBoundary = { + type: 'Polygon', + coordinates: [ + [ + [0.9, 0], + [2, 0], + [2, 1], + [0.9, 1], + [0.9, 0], + ], + ], + }; + const box = { south: 0.2, west: 0.5, north: 0.8, east: 1.5 }; + expect(coverageOf(box, [west])).toBe('partial'); + expect(coverageOf(box, [west, east])).toBe('covered'); + }); + + it('PGEO-011: a region whose boundary is unknown counts by its box alone', () => { + expect(coverageOf({ south: 0, west: 0, north: 0.5, east: 0.5 }, [null])).toBe('covered'); + }); +}); + +describe('boxOfBoundary', () => { + it('PGEO-012: spans the outer rings of every polygon', () => { + expect(boxOfBoundary(holed)).toEqual({ south: 49, west: 11, north: 50, east: 13 }); + expect(boxOfBoundary(twoIslands)).toEqual({ south: 0, west: 0, north: 6, east: 6 }); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.controller.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.controller.test.ts new file mode 100644 index 000000000..b54d4ef64 --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.controller.test.ts @@ -0,0 +1,109 @@ +/** + * The two controllers of PAD-05A over stub services: their routes, guards, + * status codes and response contracts, and how each hands its input on. The + * HTTP answers with the real guards are pinned in the e2e suite. + */ +import { AddonGuard } from '../../../../src/nest/addons/addon.guard'; +import { REQUIRE_ADDON } from '../../../../src/nest/addons/require-addon.decorator'; +import { AdminGuard } from '../../../../src/nest/auth-core/admin.guard'; +import { JwtAuthGuard } from '../../../../src/nest/auth-core/jwt-auth.guard'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { RESPONSE_CONTRACT_KEY } from '../../../../src/nest/common/response-contract'; +import { + PaddleObstaclesController, + PaddleRegionsController, +} from '../../../../src/nest/paddle-obstacles/paddle-obstacles.controller'; +import type { PaddleObstaclesService } from '../../../../src/nest/paddle-obstacles/paddle-obstacles.service'; +import type { PaddleRegionsService } from '../../../../src/nest/paddle-obstacles/paddle-regions.service'; +import { RequestMethod } from '@nestjs/common'; +import { GUARDS_METADATA, HTTP_CODE_METADATA, METHOD_METADATA, PATH_METADATA } from '@nestjs/common/constants'; +import { + paddleObstaclesResponseSchema, + paddleRegionCatalogResponseSchema, + paddleRegionListResponseSchema, + paddleRegionSchema, + successResponseSchema, + type PaddleRegion, +} from '@trek/shared'; + +import { describe, expect, it, vi } from 'vitest'; + +const region: PaddleRegion = { + id: 'oberpfalz', + name: 'Oberpfalz', + status: 'queued', + source_timestamp: null, + imported_at: null, + feature_count: 0, + error: null, + progress: null, +}; + +describe('PaddleObstaclesController', () => { + it('PADCTL-001: GET api/paddle/obstacles sits behind the Tours addon ahead of the session check', () => { + expect(Reflect.getMetadata(PATH_METADATA, PaddleObstaclesController)).toBe('api/paddle'); + expect(Reflect.getMetadata(GUARDS_METADATA, PaddleObstaclesController)).toEqual([AddonGuard, JwtAuthGuard]); + expect(Reflect.getMetadata(REQUIRE_ADDON, PaddleObstaclesController)).toEqual({ addonId: 'tours', label: 'Tours' }); + const handler = PaddleObstaclesController.prototype.inBox; + expect(Reflect.getMetadata(METHOD_METADATA, handler)).toBe(RequestMethod.GET); + expect(Reflect.getMetadata(PATH_METADATA, handler)).toBe('obstacles'); + expect(Reflect.getMetadata(RESPONSE_CONTRACT_KEY, handler)).toBe(paddleObstaclesResponseSchema); + }); + + it('PADCTL-002: hands the box on and passes a refusal through as it is', async () => { + const answer = { features: [], coverage: { state: 'none' as const, regions: [] }, truncated: false }; + const inBox = vi.fn().mockResolvedValue(answer); + const controller = new PaddleObstaclesController({ inBox } as unknown as PaddleObstaclesService); + + expect(await controller.inBox({ bbox: '49,12,49.1,12.1' })).toBe(answer); + expect(inBox).toHaveBeenCalledWith('49,12,49.1,12.1'); + + const refusal = new DomainError(400, 'bbox must be south,west,north,east and at most one degree a side'); + inBox.mockRejectedValueOnce(refusal); + await expect(controller.inBox({ bbox: '0,0,5,5' })).rejects.toBe(refusal); + }); +}); + +describe('PaddleRegionsController', () => { + it('PADCTL-003: admin only, with the routes, status codes and contracts of the admin surface', () => { + expect(Reflect.getMetadata(PATH_METADATA, PaddleRegionsController)).toBe('api/admin/paddle-regions'); + expect(Reflect.getMetadata(GUARDS_METADATA, PaddleRegionsController)).toEqual([JwtAuthGuard, AdminGuard]); + const p = PaddleRegionsController.prototype; + const routes = [ + [p.list, RequestMethod.GET, '/', undefined, paddleRegionListResponseSchema], + [p.catalog, RequestMethod.GET, 'catalog', undefined, paddleRegionCatalogResponseSchema], + [p.add, RequestMethod.POST, '/', undefined, paddleRegionSchema], + [p.refresh, RequestMethod.POST, 'refresh', 200, paddleRegionSchema], + [p.remove, RequestMethod.POST, 'remove', 200, successResponseSchema], + ] as const; + for (const [handler, method, path, code, schema] of routes) { + expect(Reflect.getMetadata(METHOD_METADATA, handler)).toBe(method); + expect(Reflect.getMetadata(PATH_METADATA, handler)).toBe(path); + expect(Reflect.getMetadata(HTTP_CODE_METADATA, handler)).toBe(code); + expect(Reflect.getMetadata(RESPONSE_CONTRACT_KEY, handler)).toBe(schema); + } + }); + + it('PADCTL-004: wraps the lists in { regions }, hands the ids on and answers { success: true } on remove', async () => { + const service = { + list: vi.fn().mockResolvedValue([region]), + catalog: vi.fn().mockResolvedValue([{ id: 'oberpfalz', name: 'Oberpfalz', path: ['Europe'] }]), + add: vi.fn().mockResolvedValue(region), + refresh: vi.fn().mockResolvedValue(region), + remove: vi.fn().mockResolvedValue(undefined), + }; + const controller = new PaddleRegionsController(service as unknown as PaddleRegionsService); + + expect(await controller.list()).toEqual({ regions: [region] }); + expect(await controller.catalog({ q: 'ober' })).toEqual({ + regions: [{ id: 'oberpfalz', name: 'Oberpfalz', path: ['Europe'] }], + }); + expect(service.catalog).toHaveBeenCalledWith('ober'); + expect(await controller.add({ id: 'oberpfalz' })).toBe(region); + expect(service.add).toHaveBeenCalledWith('oberpfalz'); + expect(await controller.refresh({ id: 'oberpfalz' })).toBe(region); + expect(service.refresh).toHaveBeenCalledWith('oberpfalz'); + expect(await controller.remove({ id: 'oberpfalz' })).toEqual({ success: true }); + expect(service.remove).toHaveBeenCalledWith('oberpfalz'); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.service.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.service.test.ts new file mode 100644 index 000000000..d0b0b36a4 --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-obstacles.service.test.ts @@ -0,0 +1,209 @@ +/** + * PaddleObstaclesService over a snapshot db: the objects of a box with how far + * the regions cover it (none, partial, covered), the cap, the span limit, an + * object two overlapping regions both hold answered once, and a stored row + * that does not read left out. + */ +import { PaddleFeatures } from '../../../../src/db/entities/PaddleFeatures.entity'; +import { PaddleRegions } from '../../../../src/db/entities/PaddleRegions.entity'; +import { AppConfigModule } from '../../../../src/nest/app-config/app-config.module'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { PaddleObstaclesModule } from '../../../../src/nest/paddle-obstacles/paddle-obstacles.module'; +import { + PADDLE_BOX_INVALID_MESSAGE, + PaddleObstaclesService, +} from '../../../../src/nest/paddle-obstacles/paddle-obstacles.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { insertRow, insertRows, updateRows } from '../../../helpers/factories/rows'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { MikroORM } from '@mikro-orm/core'; +import { PADDLE_OBSTACLES_CAP, paddleObstaclesResponseSchema } from '@trek/shared'; + +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const log = vi.hoisted(() => ({ logInfo: vi.fn(), logWarn: vi.fn(), logError: vi.fn(), logDebug: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + ...log, +})); + +const db = createSnapshotTestDb(); +let t: TestModule | undefined; +let service: PaddleObstaclesService; +const orm = () => { + if (!t) throw new Error('No container booted'); + return t.get(MikroORM); +}; + +/** The Oberpfalz as a square, 48.9 to 50 north and 11.5 to 13 east. */ +const oberpfalz = { + type: 'Polygon', + coordinates: [ + [ + [11.5, 48.9], + [13, 48.9], + [13, 50], + [11.5, 50], + [11.5, 48.9], + ], + ], +}; + +async function region(id: string, options: { boundary?: object | null; imported?: boolean } = {}) { + const boundary = options.boundary === undefined ? oberpfalz : options.boundary; + await insertRow(orm(), PaddleRegions, { + id, + name: id.charAt(0).toUpperCase() + id.slice(1), + boundary: boundary ? JSON.stringify(boundary) : null, + min_lat: 48.9, + min_lng: 11.5, + max_lat: 50, + max_lng: 13, + status: options.imported === false ? 'queued' : 'ready', + feature_count: 0, + source_timestamp: '2026-10-01T00:00:00.000Z', + imported_at: options.imported === false ? null : '2026-10-02 04:20:00', + }); +} + +const weir = (regionId: string, ref: string, lat: number, lng: number) => ({ + region: regionId, + osm_ref: ref, + kind: 'weir', + group_key: null, + name: 'Wehr', + geometry: JSON.stringify({ type: 'Point', coordinates: [lng, lat] }), + tags: JSON.stringify({ waterway: 'weir', name: 'Wehr' }), + tag_hash: `hash-${ref}`, + min_lat: lat, + min_lng: lng, + max_lat: lat, + max_lng: lng, +}); + +beforeEach(async () => { + resetTestDb(db); + await t?.close(); + t = await createTestModule({ db, imports: [AppConfigModule, PaddleObstaclesModule] }); + service = t.get(PaddleObstaclesService); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +afterAll(async () => { + await t?.close(); + db.close(); +}); + +describe('PaddleObstaclesService.inBox', () => { + it('PADOBS-001: an area outside every region answers none, not an empty all-clear', async () => { + const res = await service.inBox('53.2,12.8,53.3,13.0'); + expect(res).toEqual({ features: [], coverage: { state: 'none', regions: [] }, truncated: false }); + expect(paddleObstaclesResponseSchema.safeParse(res).success).toBe(true); + }); + + it('PADOBS-002: an area inside a region answers its objects and covered, with the date of the data', async () => { + await region('oberpfalz'); + await insertRows(orm(), PaddleFeatures, [weir('oberpfalz', 'way/29256405', 49.1235, 12.131)]); + + const res = await service.inBox('49.1,12.1,49.2,12.2'); + + expect(res.coverage).toEqual({ + state: 'covered', + regions: [ + { + id: 'oberpfalz', + name: 'Oberpfalz', + source_timestamp: '2026-10-01T00:00:00.000Z', + imported_at: '2026-10-02T04:20:00Z', + }, + ], + }); + expect(res.features).toEqual([ + { + ref: 'way/29256405', + kind: 'weir', + group_key: null, + name: 'Wehr', + geometry: { type: 'Point', coordinates: [12.131, 49.1235] }, + tags: { waterway: 'weir', name: 'Wehr' }, + tag_hash: 'hash-way/29256405', + region_id: 'oberpfalz', + }, + ]); + expect(paddleObstaclesResponseSchema.safeParse(res).success).toBe(true); + }); + + it('PADOBS-003: an area reaching past the region is partly covered; a covered area without objects stays covered', async () => { + await region('oberpfalz'); + expect((await service.inBox('49.9,12.9,50.1,13.1')).coverage.state).toBe('partial'); + const empty = await service.inBox('49.3,12.3,49.4,12.4'); + expect(empty).toMatchObject({ features: [], coverage: { state: 'covered' }, truncated: false }); + }); + + it('PADOBS-004: a region being read again, or whose last read failed, still counts with the data it has', async () => { + await region('oberpfalz'); + await updateRows(orm(), PaddleRegions, { id: 'oberpfalz' }, { status: 'importing' }); + expect((await service.inBox('49.1,12.1,49.2,12.2')).coverage.state).toBe('covered'); + }); + + it('PADOBS-005: a region not read yet does not count', async () => { + await region('oberpfalz', { imported: false }); + expect((await service.inBox('49.1,12.1,49.2,12.2')).coverage.state).toBe('none'); + }); + + it('PADOBS-006: a region without a boundary counts by its box', async () => { + await region('oberpfalz', { boundary: null }); + expect((await service.inBox('49.1,12.1,49.2,12.2')).coverage.state).toBe('covered'); + }); + + it('PADOBS-007: an object two overlapping regions both hold is answered once', async () => { + await region('oberpfalz'); + await region('bayern'); + await insertRows(orm(), PaddleFeatures, [ + weir('oberpfalz', 'node/1', 49.15, 12.15), + weir('bayern', 'node/1', 49.15, 12.15), + weir('bayern', 'node/2', 49.16, 12.16), + ]); + const res = await service.inBox('49.1,12.1,49.2,12.2'); + expect(res.features.map((feature) => feature.ref)).toEqual(['node/1', 'node/2']); + expect(res.coverage.regions.map((entry) => entry.id)).toEqual(['bayern', 'oberpfalz']); + }); + + it('PADOBS-008: cuts the answer at the cap and says so', async () => { + await region('oberpfalz'); + const many = Array.from({ length: PADDLE_OBSTACLES_CAP + 5 }, (_, i) => + weir('oberpfalz', `node/${i + 1}`, 49.1 + (i % 100) * 0.0005, 12.1 + Math.floor(i / 100) * 0.0005), + ); + await insertRows(orm(), PaddleFeatures, many); + const res = await service.inBox('49.1,12.1,49.2,12.2'); + expect(res.features).toHaveLength(PADDLE_OBSTACLES_CAP); + expect(res.truncated).toBe(true); + }); + + it('PADOBS-009: leaves out a stored object that does not read, and logs it', async () => { + await region('oberpfalz'); + await insertRows(orm(), PaddleFeatures, [ + { ...weir('oberpfalz', 'node/1', 49.15, 12.15), geometry: '{broken' }, + { ...weir('oberpfalz', 'node/2', 49.15, 12.15), kind: 'castle' }, + { ...weir('oberpfalz', 'node/3', 49.15, 12.15), tags: '[]' }, + ]); + const res = await service.inBox('49.1,12.1,49.2,12.2'); + expect(res.features.map((feature) => [feature.ref, feature.tags])).toEqual([['node/3', {}]]); + expect(log.logWarn).toHaveBeenCalledWith('[Paddle] stored object oberpfalz/node/1 does not read; left out'); + expect(log.logWarn).toHaveBeenCalledWith('[Paddle] stored object oberpfalz/node/2 does not read; left out'); + expect(log.logWarn).toHaveBeenCalledWith(expect.stringContaining('paddle_features.tags (oberpfalz/node/3)')); + }); + + it.each(['49,12,50.5,12.5', '49.2,12,49.1,12.1', '1,2,3'])( + 'PADOBS-010: refuses the box %s with the bbox message', + async (bbox) => { + const err = await service.inBox(bbox).catch((e: unknown) => e); + expect(err).toBeInstanceOf(DomainError); + expect(err).toMatchObject({ status: 400, publicMessage: PADDLE_BOX_INVALID_MESSAGE }); + }, + ); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-regions.job.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-regions.job.test.ts new file mode 100644 index 000000000..8aee914ea --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-regions.job.test.ts @@ -0,0 +1,84 @@ +/** + * PaddleRegionsJob: the boot resume and the monthly refresh on the cron + * registrar, and neither letting a failure out. + */ +import { PaddleRegionsJob } from '../../../../src/nest/paddle-obstacles/paddle-regions.job'; +import type { PaddleRegionsService } from '../../../../src/nest/paddle-obstacles/paddle-regions.service'; +import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const log = vi.hoisted(() => ({ logInfo: vi.fn(), logWarn: vi.fn(), logError: vi.fn(), logDebug: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + ...log, +})); + +function registrarStub(enabled = true) { + return { + isEnabled: vi.fn(() => enabled), + register: vi.fn(() => enabled), + runOnBoot: vi.fn(async (_name: string, fn: () => Promise | void) => { + await fn(); + }), + }; +} + +function regionsStub() { + return { + resumeAfterRestart: vi.fn<() => Promise>().mockResolvedValue(undefined), + queueRefresh: vi.fn<() => Promise>().mockResolvedValue(0), + }; +} + +const job = (regions: ReturnType, registrar: ReturnType) => + new PaddleRegionsJob(regions as unknown as PaddleRegionsService, registrar as unknown as CronRegistrarService); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe('PaddleRegionsJob', () => { + it('PADJOB-001: resumes the reads on boot and registers the refresh for 04:20 on the 3rd of the month', async () => { + const regions = regionsStub(); + const registrar = registrarStub(); + + job(regions, registrar).onApplicationBootstrap(); + + expect(registrar.runOnBoot).toHaveBeenCalledWith('paddle-regions-resume', expect.any(Function)); + expect(registrar.register).toHaveBeenCalledWith('paddle-regions-refresh', '20 4 3 * *', expect.any(Function)); + await vi.waitFor(() => expect(regions.resumeAfterRestart).toHaveBeenCalledOnce()); + }); + + it('PADJOB-002: does nothing where crons are off (tests, a worker without the scheduler)', () => { + const regions = regionsStub(); + const registrar = registrarStub(false); + job(regions, registrar).onApplicationBootstrap(); + expect(registrar.runOnBoot).not.toHaveBeenCalled(); + expect(registrar.register).not.toHaveBeenCalled(); + }); + + it('PADJOB-003: the tick queues the regions and logs how many', async () => { + const regions = regionsStub(); + regions.queueRefresh.mockResolvedValueOnce(3).mockResolvedValueOnce(0); + const subject = job(regions, registrarStub()); + + await subject.tick(); + expect(log.logInfo).toHaveBeenCalledWith('[Paddle] monthly refresh: 3 region(s) queued'); + log.logInfo.mockClear(); + await subject.tick(); + expect(log.logInfo).not.toHaveBeenCalled(); + }); + + it('PADJOB-004: a failing tick or boot resume is logged and goes no further', async () => { + const regions = regionsStub(); + regions.queueRefresh.mockRejectedValueOnce(new Error('database is locked')); + regions.resumeAfterRestart.mockRejectedValueOnce(new Error('disk full')); + const subject = job(regions, registrarStub()); + + await expect(subject.tick()).resolves.toBeUndefined(); + await expect(subject.resume()).resolves.toBeUndefined(); + expect(log.logError).toHaveBeenCalledWith('[Paddle] monthly refresh: database is locked'); + expect(log.logError).toHaveBeenCalledWith('[Paddle] resuming the reads after a restart: disk full'); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-regions.service.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-regions.service.test.ts new file mode 100644 index 000000000..54a997680 --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-regions.service.test.ts @@ -0,0 +1,429 @@ +/** + * PaddleRegionsService over a snapshot db with Geofabrik faked: a region + * added, read in the background from an extract the test writes, its objects + * stored; the queue in order; a failed read that keeps the previous data; a + * region removed while it is read; the monthly and the boot requeue. + */ +import { PaddleFeatures } from '../../../../src/db/entities/PaddleFeatures.entity'; +import { PaddleRegions } from '../../../../src/db/entities/PaddleRegions.entity'; +import type { PaddleRegionsRepository } from '../../../../src/db/repositories/PaddleRegions.repository'; +import { AppConfigModule } from '../../../../src/nest/app-config/app-config.module'; +import { DataPathsService } from '../../../../src/nest/app-config/data-paths.service'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { GeofabrikClient, type GeofabrikRegion } from '../../../../src/nest/paddle-obstacles/geofabrik.client'; +import { PaddleObstaclesModule } from '../../../../src/nest/paddle-obstacles/paddle-obstacles.module'; +import { + CATALOG_UNAVAILABLE_MESSAGE, + PaddleRegionsService, + REGION_NOT_FOUND_MESSAGE, + REGION_UNKNOWN_MESSAGE, +} from '../../../../src/nest/paddle-obstacles/paddle-regions.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { countRows, deleteRows, findRow, findRows, updateRows } from '../../../helpers/factories/rows'; +import { pbfFile } from '../../../helpers/osm-pbf-fixture'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { MikroORM } from '@mikro-orm/core'; +import { getRepositoryToken } from '@mikro-orm/nestjs'; +import { paddleRegionSchema } from '@trek/shared'; + +import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const log = vi.hoisted(() => ({ logInfo: vi.fn(), logWarn: vi.fn(), logError: vi.fn(), logDebug: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + ...log, +})); + +const db = createSnapshotTestDb(); + +const boundary = { + type: 'Polygon' as const, + coordinates: [ + [ + [11.5, 48.9], + [13, 48.9], + [13, 50], + [11.5, 50], + [11.5, 48.9], + ] as [number, number][], + ], +}; +const region = (id: string, name: string, parent: string | null = null): GeofabrikRegion => ({ + id, + name, + parent, + pbfUrl: `https://download.geofabrik.de/${id}-latest.osm.pbf`, + boundary: id === 'oberpfalz' ? boundary : null, +}); +const catalogue = new Map( + [ + region('europe', 'Europe'), + region('germany', 'Germany', 'europe'), + region('bayern', 'Bayern', 'germany'), + region('oberpfalz', 'Oberpfalz', 'bayern'), + region('oberbayern', 'Oberbayern', 'bayern'), + region('mecklenburg-vorpommern', 'Mecklenburg-Vorpommern', 'germany'), + region('limburg-nl', 'Limburg', 'europe'), + ].map((entry) => [entry.id, entry]), +); + +/** The Lindenmühle weir and a put-in, current to 1 Oct 2026. */ +const extract = pbfFile( + [ + { + dense: [ + { id: 1, lat: 49.1231, lon: 12.1301 }, + { id: 2, lat: 49.124, lon: 12.1315 }, + { id: 3, lat: 49.13, lon: 12.14, tags: { canoe: 'put_in' } }, + ], + }, + { ways: [{ id: 29256405, refs: [1, 2], tags: { waterway: 'weir', name: 'Wehr Lindenmühle' } }] }, + ], + { timestamp: Date.UTC(2026, 9, 1) / 1000 }, +); + +type DownloadOptions = Parameters[2]; +const regions = vi.fn<() => Promise>>(); +const download = vi.fn<(url: string, target: string, options?: DownloadOptions) => Promise>(); + +let dir: string; +let t: TestModule | undefined; +let service: PaddleRegionsService; +const orm = () => { + if (!t) throw new Error('No container booted'); + return t.get(MikroORM); +}; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-paddle-regions-')); +}); + +beforeEach(async () => { + resetTestDb(db); + await rm(join(dir, 'paddle'), { recursive: true, force: true }); + regions.mockReset(); + regions.mockResolvedValue(catalogue); + download.mockReset(); + download.mockImplementation((_url, target) => writeFile(target, extract)); + await t?.close(); + t = await createTestModule({ + db, + imports: [AppConfigModule, PaddleObstaclesModule], + overrides: [ + { provide: GeofabrikClient, useValue: { regions, download } }, + { provide: DataPathsService, useValue: { tmpDir: dir } }, + ], + }); + service = t.get(PaddleRegionsService); +}); + +afterEach(async () => { + // A case that failed before letting its download go must not hold the next one up. + for (const release of held) release(); + held.clear(); + await service.idle(); + vi.restoreAllMocks(); + vi.clearAllMocks(); +}); + +afterAll(async () => { + await t?.close(); + db.close(); + await rm(dir, { recursive: true, force: true }); +}); + +const held = new Set<() => void>(); + +/** A download that waits until the test lets it finish or the read is called off. */ +function heldDownload(): { release: () => void; signal: () => AbortSignal | undefined } { + let release: () => void = () => {}; + let signal: AbortSignal | undefined; + download.mockImplementationOnce( + (_url, target, options) => + new Promise((resolve, reject) => { + let settled = false; + signal = options?.signal; + options?.onProgress?.(0.25); + options?.signal?.addEventListener('abort', () => { + settled = true; + reject(new Error('This operation was aborted')); + }); + release = () => { + if (settled) return; + settled = true; + void writeFile(target, extract).then(resolve, reject); + }; + held.add(release); + }), + ); + return { release: () => release(), signal: () => signal }; +} + +describe('PaddleRegionsService.add and the background read', () => { + it('PREG-001: queues the region, reads its extract and stores its objects with the date of the data', async () => { + const added = await service.add('oberpfalz'); + expect(added).toMatchObject({ id: 'oberpfalz', name: 'Oberpfalz', feature_count: 0, progress: null }); + expect(['queued', 'importing']).toContain(added.status); + expect(paddleRegionSchema.safeParse(added).success).toBe(true); + + await service.idle(); + + const [listed] = await service.list(); + expect(listed).toMatchObject({ + id: 'oberpfalz', + status: 'ready', + feature_count: 2, + error: null, + source_timestamp: '2026-10-01T00:00:00.000Z', + progress: null, + }); + expect(listed?.imported_at).toMatch(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/); + const stored = await findRows(orm(), PaddleFeatures, {}, { osm_ref: 'asc' }); + expect(stored.map((row) => [row.osm_ref, row.kind])).toEqual([ + ['node/3', 'put_in'], + ['way/29256405', 'weir'], + ]); + const row = await findRow(orm(), PaddleRegions, { id: 'oberpfalz' }); + expect(row).toMatchObject({ min_lat: 48.9, max_lat: 50, min_lng: 11.5, max_lng: 13 }); + expect(download).toHaveBeenCalledWith( + 'https://download.geofabrik.de/oberpfalz-latest.osm.pbf', + expect.stringContaining(join(dir, 'paddle', 'oberpfalz-')), + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + // The extract is gone once it is read. + expect(await readdir(join(dir, 'paddle'))).toEqual([]); + }); + + it('PREG-002: refuses a region Geofabrik does not name, and says so when its list cannot be read', async () => { + const unknown = await service.add('atlantis').catch((err: unknown) => err); + expect(unknown).toBeInstanceOf(DomainError); + expect(unknown).toMatchObject({ publicMessage: REGION_UNKNOWN_MESSAGE, status: 400 }); + + regions.mockRejectedValue(new Error('Geofabrik index 503')); + const add = await service.add('oberpfalz').catch((err: unknown) => err); + expect(add).toMatchObject({ publicMessage: CATALOG_UNAVAILABLE_MESSAGE, status: 502 }); + const catalog = await service.catalog('ober').catch((err: unknown) => err); + expect(catalog).toMatchObject({ publicMessage: CATALOG_UNAVAILABLE_MESSAGE, status: 502 }); + expect(await countRows(orm(), PaddleRegions)).toBe(0); + }); + + it('PREG-003: adding a region already there reads it again', async () => { + await service.add('oberpfalz'); + await service.idle(); + const again = await service.add('oberpfalz'); + expect(again.status).toBe('queued'); + await service.idle(); + expect(download).toHaveBeenCalledTimes(2); + expect(await countRows(orm(), PaddleRegions)).toBe(1); + }); + + it('PREG-004: a failed read marks the region failed with the reason and keeps the objects it had', async () => { + await service.add('oberpfalz'); + await service.idle(); + download.mockRejectedValueOnce(new Error('Geofabrik download 500')); + + await service.refresh('oberpfalz'); + await service.idle(); + + const [listed] = await service.list(); + expect(listed).toMatchObject({ status: 'failed', error: 'Geofabrik download 500', feature_count: 2 }); + expect(await countRows(orm(), PaddleFeatures)).toBe(2); + }); + + it('PREG-005: a region Geofabrik no longer lists at the time of its read fails with that reason', async () => { + await service.add('oberpfalz'); + regions.mockResolvedValue(new Map()); + await service.idle(); + const [listed] = await service.list(); + expect(listed).toMatchObject({ status: 'failed', error: REGION_UNKNOWN_MESSAGE }); + }); + + it('PREG-006: shows how far the read has come while it downloads', async () => { + const held = heldDownload(); + await service.add('oberpfalz'); + await vi.waitFor(() => expect(download).toHaveBeenCalledTimes(1)); + expect((await service.list())[0]).toMatchObject({ + status: 'importing', + progress: { phase: 'download', fraction: 0.25 }, + }); + + held.release(); + await service.idle(); + expect((await service.list())[0]).toMatchObject({ status: 'ready', progress: null }); + }); +}); + +describe('PaddleRegionsService: the queue', () => { + it('PREG-007: reads queued regions one after another, a region queued while another is read included', async () => { + const held = heldDownload(); + await service.add('oberpfalz'); + await vi.waitFor(() => expect(download).toHaveBeenCalledTimes(1)); + await service.add('mecklenburg-vorpommern'); + expect(download).toHaveBeenCalledTimes(1); + + held.release(); + await service.idle(); + + expect(download.mock.calls.map(([url]) => url)).toEqual([ + 'https://download.geofabrik.de/oberpfalz-latest.osm.pbf', + 'https://download.geofabrik.de/mecklenburg-vorpommern-latest.osm.pbf', + ]); + expect((await service.list()).map((entry) => [entry.id, entry.status])).toEqual([ + ['mecklenburg-vorpommern', 'ready'], + ['oberpfalz', 'ready'], + ]); + }); + + it('PREG-008: removing a region while it is read calls the read off and keeps nothing of it', async () => { + const held = heldDownload(); + await service.add('oberpfalz'); + await vi.waitFor(() => expect(download).toHaveBeenCalledTimes(1)); + + await service.remove('oberpfalz'); + await service.idle(); + + expect(held.signal()?.aborted).toBe(true); + expect(await countRows(orm(), PaddleRegions)).toBe(0); + expect(await countRows(orm(), PaddleFeatures)).toBe(0); + expect(await readdir(join(dir, 'paddle'))).toEqual([]); + }); + + it('PREG-009: a region whose row went while its file was read keeps no objects', async () => { + download.mockImplementationOnce(async (_url, target) => { + await writeFile(target, extract); + await deleteRows(orm(), PaddleRegions, { id: 'oberpfalz' }); + }); + await service.add('oberpfalz'); + await service.idle(); + expect(await countRows(orm(), PaddleFeatures)).toBe(0); + }); + + it('PREG-010: refresh leaves a waiting or running read alone and refuses a region it does not hold', async () => { + const held = heldDownload(); + await service.add('oberpfalz'); + await vi.waitFor(() => expect(download).toHaveBeenCalledTimes(1)); + expect((await service.refresh('oberpfalz')).status).toBe('importing'); + held.release(); + await service.idle(); + expect(download).toHaveBeenCalledTimes(1); + + const missing = await service.refresh('bayern').catch((err: unknown) => err); + expect(missing).toMatchObject({ publicMessage: REGION_NOT_FOUND_MESSAGE, status: 404 }); + const gone = await service.remove('bayern').catch((err: unknown) => err); + expect(gone).toMatchObject({ publicMessage: REGION_NOT_FOUND_MESSAGE, status: 404 }); + }); + + it('PREG-011: the monthly refresh queues every region with data or a failed read; a boot requeues a cut read', async () => { + await service.add('oberpfalz'); + await service.add('mecklenburg-vorpommern'); + await service.idle(); + await updateRows(orm(), PaddleRegions, { id: 'mecklenburg-vorpommern' }, { status: 'failed' }); + + expect(await service.queueRefresh()).toBe(2); + await service.idle(); + expect(download).toHaveBeenCalledTimes(4); + expect(await service.queueRefresh()).toBe(2); + await service.idle(); + + await updateRows(orm(), PaddleRegions, { id: 'oberpfalz' }, { status: 'importing' }); + await service.resumeAfterRestart(); + await service.idle(); + expect(download).toHaveBeenCalledTimes(7); + expect((await service.list()).every((entry) => entry.status === 'ready')).toBe(true); + }); + + it('PREG-012: an error the queue itself meets is logged, and the next start picks the region up', async () => { + const repository = t?.get(getRepositoryToken(PaddleRegions)); + if (!repository) throw new Error('No repository'); + vi.spyOn(repository, 'nextQueued').mockRejectedValueOnce(new Error('database is locked')); + + await service.add('oberpfalz'); + await service.idle(); + expect((await service.list())[0]?.status).toBe('queued'); + expect(log.logError).toHaveBeenCalledWith('[Paddle] the read queue stopped: database is locked'); + + service.kick(); + await service.idle(); + expect((await service.list())[0]?.status).toBe('ready'); + }); +}); + +describe('PaddleRegionsService: restart and shutdown', () => { + it('PREG-015: a shutdown stops the read under way, leaves it marked as being read and starts no other', async () => { + const held = heldDownload(); + await service.add('oberpfalz'); + await service.add('mecklenburg-vorpommern'); + await vi.waitFor(() => expect(download).toHaveBeenCalledTimes(1)); + + service.onApplicationShutdown(); + await service.idle(); + + expect(held.signal()?.aborted).toBe(true); + expect(download).toHaveBeenCalledTimes(1); + expect((await service.list()).map((entry) => [entry.id, entry.status])).toEqual([ + ['mecklenburg-vorpommern', 'queued'], + ['oberpfalz', 'importing'], + ]); + expect(log.logInfo).toHaveBeenCalledWith( + '[Paddle] oberpfalz: stopped by the shutdown; read again on the next start', + ); + }); + + it('PREG-016: the boot after it queues the cut read again and clears what the last process left in its folder', async () => { + await service.add('oberpfalz'); + await service.idle(); + await updateRows(orm(), PaddleRegions, { id: 'oberpfalz' }, { status: 'importing' }); + await mkdir(join(dir, 'paddle'), { recursive: true }); + await writeFile(join(dir, 'paddle', 'oberpfalz-1.osm.pbf'), 'half a download'); + + await service.resumeAfterRestart(); + await service.idle(); + + expect((await service.list())[0]?.status).toBe('ready'); + expect(await readdir(join(dir, 'paddle'))).toEqual([]); + }); + + it('PREG-017: a nested Geofabrik id reads into a file in the one folder', async () => { + const texas = { + ...region('us/texas', 'Texas', 'us'), + pbfUrl: 'https://download.geofabrik.de/north-america/us/texas-latest.osm.pbf', + }; + regions.mockResolvedValue(new Map([['us/texas', texas]])); + await service.add('us/texas'); + await service.idle(); + expect(download.mock.calls[0]?.[1]).toMatch(/[/\\]paddle[/\\]us_texas-\d+\.osm\.pbf$/); + expect((await service.list())[0]).toMatchObject({ id: 'us/texas', status: 'ready', feature_count: 2 }); + // Without a boundary from Geofabrik, the region spans its objects. + expect(await findRow(orm(), PaddleRegions, { id: 'us/texas' })).toMatchObject({ + min_lat: 49.1231, + max_lat: 49.13, + min_lng: 12.1301, + max_lng: 12.14, + }); + }); +}); + +describe('PaddleRegionsService.catalog', () => { + it('PREG-013: finds regions by name or id, names starting with the query first, with the regions they lie in', async () => { + expect(await service.catalog('ober')).toEqual([ + { id: 'oberpfalz', name: 'Oberpfalz', path: ['Europe', 'Germany', 'Bayern'] }, + { id: 'oberbayern', name: 'Oberbayern', path: ['Europe', 'Germany', 'Bayern'] }, + ]); + expect(await service.catalog('vorpommern')).toEqual([ + { id: 'mecklenburg-vorpommern', name: 'Mecklenburg-Vorpommern', path: ['Europe', 'Germany'] }, + ]); + expect(await service.catalog('limburg-nl')).toEqual([{ id: 'limburg-nl', name: 'Limburg', path: ['Europe'] }]); + }); + + it('PREG-014: answers at most 25 matches', async () => { + const many = new Map( + Array.from({ length: 40 }, (_, i) => [`region-${i}`, region(`region-${i}`, `Region ${i}`)] as const), + ); + regions.mockResolvedValue(many); + expect(await service.catalog('region')).toHaveLength(25); + }); +}); diff --git a/server/tests/unit/nest/paddle-obstacles/paddle-tags.test.ts b/server/tests/unit/nest/paddle-obstacles/paddle-tags.test.ts new file mode 100644 index 000000000..8261cbcd7 --- /dev/null +++ b/server/tests/unit/nest/paddle-obstacles/paddle-tags.test.ts @@ -0,0 +1,85 @@ +/** + * Which OpenStreetMap objects PAD-05A reads as paddle objects, by the tag + * variants paddle-tags.ts documents, and which of their tags are kept. + */ +import { keptTags, paddleKindOf, paddleNameOf } from '../../../../src/nest/paddle-obstacles/paddle-tags'; + +import { describe, expect, it } from 'vitest'; + +describe('paddleKindOf', () => { + it.each([ + [{ waterway: 'weir' }, 'weir'], + [{ waterway: 'dam' }, 'dam'], + [{ waterway: 'lock_gate' }, 'lock_gate'], + [{ waterway: 'waterfall' }, 'waterfall'], + [{ lock: 'yes' }, 'lock'], + [{ lock: 'yes', waterway: 'canal' }, 'lock'], + [{ whitewater: 'rapid' }, 'rapids'], + [{ waterway: 'rapids' }, 'rapids'], + [{ whitewater: 'hazard' }, 'hazard'], + [{ canoe: 'portage' }, 'portage'], + [{ portage: 'yes' }, 'portage'], + [{ whitewater: 'portage_way' }, 'portage'], + [{ canoe: 'put_in' }, 'put_in'], + [{ whitewater: 'put_in' }, 'put_in'], + [{ whitewater: 'put_in;egress' }, 'put_in'], + [{ whitewater: 'egress;put_in' }, 'put_in'], + [{ canoe: 'egress' }, 'egress'], + [{ whitewater: 'egress' }, 'egress'], + [{ leisure: 'slipway' }, 'slipway'], + [{ waterway: 'river', hazard: 'yes' }, 'hazard'], + [{ waterway: 'stream', obstacle: 'fallen_tree' }, 'hazard'], + [{ waterway: 'river', canoe: 'no' }, 'access'], + [{ waterway: 'canal', boat: 'private' }, 'access'], + [{ waterway: 'river', 'canoe:conditional': 'no @ (Nov-Mar)' }, 'access'], + [{ waterway: 'fairway', 'boat:conditional': 'yes @ (sunrise-sunset)' }, 'access'], + ] as const)('PTAG-001 %o reads as %s', (tags, kind) => { + expect(paddleKindOf(tags)).toBe(kind); + }); + + it('PTAG-002: an object that is none of them reads as null', () => { + expect(paddleKindOf({})).toBeNull(); + expect(paddleKindOf({ waterway: 'river' })).toBeNull(); + expect(paddleKindOf({ highway: 'path', canoe: 'no' })).toBeNull(); + // A lock on something that is no waterway TREK knows (a dock) is not a paddle lock. + expect(paddleKindOf({ lock: 'yes', waterway: 'dock' })).toBeNull(); + expect(paddleKindOf({ whitewater: 'rapid_class' })).toBeNull(); + }); + + it('PTAG-003: the structure wins over the rule: a weir with a canoe tag stays a weir', () => { + expect(paddleKindOf({ waterway: 'weir', canoe: 'portage' })).toBe('weir'); + expect(paddleKindOf({ waterway: 'dam', whitewater: 'hazard' })).toBe('dam'); + expect(paddleKindOf({ canoe: 'portage', whitewater: 'put_in' })).toBe('portage'); + }); +}); + +describe('keptTags and paddleNameOf', () => { + it('PTAG-004: keeps the tags a paddler is shown and the prefixed rules, nothing else', () => { + expect( + keptTags({ + name: 'Lindenmühle', + waterway: 'weir', + 'canoe:conditional': 'no @ (Nov-Mar)', + 'whitewater:section_grade': '2', + 'portage:access': 'private', + height: '1.5', + source: 'survey', + highway: 'footway', + created_by: 'JOSM', + }), + ).toEqual({ + name: 'Lindenmühle', + waterway: 'weir', + 'canoe:conditional': 'no @ (Nov-Mar)', + 'whitewater:section_grade': '2', + 'portage:access': 'private', + height: '1.5', + }); + }); + + it("PTAG-005: names an object by its own name, else by the lock's", () => { + expect(paddleNameOf({ name: 'Wehr', lock_name: 'Schleuse' })).toBe('Wehr'); + expect(paddleNameOf({ lock_name: 'Schleuse Wesenberg' })).toBe('Schleuse Wesenberg'); + expect(paddleNameOf({ waterway: 'weir' })).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/tours.controller.test.ts b/server/tests/unit/nest/tours.controller.test.ts index eba2d60fd..773f9f6ef 100644 --- a/server/tests/unit/nest/tours.controller.test.ts +++ b/server/tests/unit/nest/tours.controller.test.ts @@ -14,9 +14,9 @@ import { REQUIRE_ADDON } from '../../../src/nest/addons/require-addon.decorator' import { JwtAuthGuard } from '../../../src/nest/auth-core/jwt-auth.guard'; import { TRIP_PERMISSION_KEY, TripAccessGuard } from '../../../src/nest/permissions/trip-access.guard'; import type { PlacesService } from '../../../src/nest/places/places.service'; +import type { TourPaddlePlanService } from '../../../src/nest/tours/tour-paddle-plan.service'; import { ToursImportController } from '../../../src/nest/tours/tours-import.controller'; import { ToursController } from '../../../src/nest/tours/tours.controller'; -import type { TourPaddlePlanService } from '../../../src/nest/tours/tour-paddle-plan.service'; import type { ToursService } from '../../../src/nest/tours/tours.service'; import type { User } from '../../../src/types'; import { HttpException, RequestMethod } from '@nestjs/common'; diff --git a/server/tests/unit/nest/weather/wind.service.test.ts b/server/tests/unit/nest/weather/wind.service.test.ts index aa63c0f41..651d26b19 100644 --- a/server/tests/unit/nest/weather/wind.service.test.ts +++ b/server/tests/unit/nest/weather/wind.service.test.ts @@ -29,7 +29,9 @@ function openMeteo(overrides: Record = {}) { } function answer(body: unknown, status = 200) { - return vi.fn(async () => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } })); + return vi.fn( + async () => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }), + ); } afterEach(() => vi.unstubAllGlobals()); @@ -39,7 +41,12 @@ describe('WindService', () => { const fetchSpy = answer(openMeteo()); vi.stubGlobal('fetch', fetchSpy); const result = await new WindService().forecast({ lat: 47.55, lng: 9.68, date: '2026-10-18' }, NOW); - expect(result).toMatchObject({ status: 'forecast', timezone: 'Europe/Berlin', source: 'Open-Meteo', horizon_days: 16 }); + expect(result).toMatchObject({ + status: 'forecast', + timezone: 'Europe/Berlin', + source: 'Open-Meteo', + horizon_days: 16, + }); expect(result.hours).toEqual([ { hour: 0, speed_kmh: 5, gusts_kmh: 9, direction_deg: 270 }, { hour: 1, speed_kmh: null, gusts_kmh: null, direction_deg: null }, diff --git a/server/vitest.config.ts b/server/vitest.config.ts index b88e8e36a..42c6dbd63 100644 --- a/server/vitest.config.ts +++ b/server/vitest.config.ts @@ -136,6 +136,7 @@ export default defineConfig({ 'src/nest/oauth/**/*.ts': { statements: 96, branches: 95, functions: 97, lines: 97 }, 'src/nest/oidc/**/*.ts': { statements: 88, branches: 83, functions: 91, lines: 92 }, 'src/nest/packing/**/*.ts': { statements: 92, branches: 83, functions: 99, lines: 97 }, + 'src/nest/paddle-obstacles/**/*.ts': { statements: 96, branches: 92, functions: 94, lines: 97 }, 'src/nest/permissions/**/*.ts': { statements: 97, branches: 92, functions: 99, lines: 97 }, 'src/nest/photos/**/*.ts': { statements: 96, branches: 92, functions: 92, lines: 96 }, // Landed on dev without an entry, so it sat on the 80 catch-all while diff --git a/shared/src/paddle/paddle-obstacles.schema.spec.ts b/shared/src/paddle/paddle-obstacles.schema.spec.ts new file mode 100644 index 000000000..b063df0db --- /dev/null +++ b/shared/src/paddle/paddle-obstacles.schema.spec.ts @@ -0,0 +1,150 @@ +import { + paddleAccessOf, + paddleFeatureSchema, + paddleGeometrySchema, + paddleObstaclesQuerySchema, + paddleObstaclesResponseSchema, +} from './paddle-obstacles.schema'; +import { + paddleRegionBoundarySchema, + paddleRegionCatalogQuerySchema, + paddleRegionRequestSchema, + paddleRegionSchema, +} from './paddle-regions.schema'; + +import { describe, expect, it } from 'vitest'; + +const weir = { + ref: 'way/29256405', + kind: 'weir', + group_key: null, + name: 'Wehr Lindenmühle', + geometry: { + type: 'LineString', + coordinates: [ + [12.1301, 49.1231], + [12.1315, 49.124], + ], + }, + tags: { waterway: 'weir' }, + tag_hash: 'abc', + region_id: 'oberpfalz', +}; + +describe('paddle obstacle contracts (PAD-05A)', () => { + it('a weir stays a line, never only a point', () => { + expect(paddleFeatureSchema.parse(weir).geometry.type).toBe('LineString'); + expect(paddleGeometrySchema.safeParse({ type: 'LineString', coordinates: [[12, 49]] }).success).toBe(false); + expect( + paddleGeometrySchema.safeParse({ + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [0, 0], + ], + ], + }).success, + ).toBe(false); + expect(paddleGeometrySchema.safeParse({ type: 'Point', coordinates: [Number.NaN, 1] }).success).toBe(false); + }); + + it('refuses a kind TREK does not know', () => { + expect(paddleFeatureSchema.safeParse({ ...weir, kind: 'castle' }).success).toBe(false); + }); + + it('the box query is four numbers', () => { + expect(paddleObstaclesQuerySchema.safeParse({ bbox: '49.1,12.1,49.2,12.2' }).success).toBe(true); + expect(paddleObstaclesQuerySchema.safeParse({ bbox: '-34,-71,-33.5,-70.5' }).success).toBe(true); + expect(paddleObstaclesQuerySchema.safeParse({ bbox: '49.1,12.1,49.2' }).success).toBe(false); + expect(paddleObstaclesQuerySchema.safeParse({ bbox: '49.1,12.1,49.2,abc' }).success).toBe(false); + }); + + it('an answer carries its coverage and whether it was cut', () => { + const answer = { features: [weir], coverage: { state: 'partial', regions: [] }, truncated: false }; + expect(paddleObstaclesResponseSchema.safeParse(answer).success).toBe(true); + expect( + paddleObstaclesResponseSchema.safeParse({ ...answer, coverage: { state: 'free', regions: [] } }).success, + ).toBe(false); + }); +}); + +describe('paddleAccessOf', () => { + it('reads canoe before boat and boat before access', () => { + expect(paddleAccessOf({ canoe: 'yes', boat: 'no', access: 'private' })).toMatchObject({ + value: 'yes', + source: 'canoe', + }); + expect(paddleAccessOf({ boat: 'no', access: 'yes' })).toMatchObject({ + value: 'no', + source: 'boat', + forbidden: true, + }); + expect(paddleAccessOf({ access: 'private' })).toMatchObject({ value: 'private', source: 'access', private: true }); + }); + + it('keeps a condition as written and never reads a missing value as permission', () => { + expect(paddleAccessOf({ 'canoe:conditional': 'no @ (Nov-Mar)' })).toEqual({ + value: null, + source: null, + condition: 'no @ (Nov-Mar)', + forbidden: false, + private: false, + }); + expect(paddleAccessOf({ 'boat:conditional': 'yes @ (sunrise-sunset)' }).condition).toBe('yes @ (sunrise-sunset)'); + expect(paddleAccessOf({})).toEqual({ + value: null, + source: null, + condition: null, + forbidden: false, + private: false, + }); + }); +}); + +describe('paddle region contracts (PAD-05A)', () => { + it("takes Geofabrik's ids, nested ones included, and nothing that could name a path", () => { + for (const id of ['oberpfalz', 'mecklenburg-vorpommern', 'us/texas']) { + expect(paddleRegionRequestSchema.safeParse({ id }).success).toBe(true); + } + for (const id of ['', 'a', 'Oberpfalz', '../etc', 'us//texas', '/texas', 'us/', 'a b']) { + expect(paddleRegionRequestSchema.safeParse({ id }).success).toBe(false); + } + }); + + it('a catalogue query is two to eighty characters once trimmed', () => { + expect(paddleRegionCatalogQuerySchema.parse({ q: ' ober ' }).q).toBe('ober'); + expect(paddleRegionCatalogQuerySchema.safeParse({ q: ' o ' }).success).toBe(false); + expect(paddleRegionCatalogQuerySchema.safeParse({ q: 'x'.repeat(81) }).success).toBe(false); + }); + + it('a region carries its read, and the progress of one under way', () => { + const region = { + id: 'oberpfalz', + name: 'Oberpfalz', + status: 'importing', + source_timestamp: null, + imported_at: null, + feature_count: 0, + error: null, + progress: { phase: 'download', fraction: 0.4 }, + }; + expect(paddleRegionSchema.safeParse(region).success).toBe(true); + expect(paddleRegionSchema.safeParse({ ...region, progress: { phase: 'read', fraction: null } }).success).toBe(true); + expect(paddleRegionSchema.safeParse({ ...region, progress: { phase: 'read', fraction: 2 } }).success).toBe(false); + expect(paddleRegionSchema.safeParse({ ...region, status: 'done' }).success).toBe(false); + }); + + it('a boundary is a polygon or several', () => { + const square = [ + [0, 0], + [1, 0], + [1, 1], + [0, 0], + ]; + expect(paddleRegionBoundarySchema.safeParse({ type: 'Polygon', coordinates: [square] }).success).toBe(true); + expect(paddleRegionBoundarySchema.safeParse({ type: 'MultiPolygon', coordinates: [[square]] }).success).toBe(true); + expect(paddleRegionBoundarySchema.safeParse({ type: 'Point', coordinates: [0, 0] }).success).toBe(false); + }); +}); diff --git a/shared/src/paddle/paddle-regions.schema.ts b/shared/src/paddle/paddle-regions.schema.ts index 140dc8fda..52aecd12a 100644 --- a/shared/src/paddle/paddle-regions.schema.ts +++ b/shared/src/paddle/paddle-regions.schema.ts @@ -3,7 +3,7 @@ import { z } from 'zod'; /** * PAD-05A, the admin side: the regions TREK reads paddle data for. Each is * one of Geofabrik's regional OpenStreetMap extracts, chosen by its id there - * (`germany/bayern/oberpfalz`); TREK downloads it, reads the paddle objects + * (`oberpfalz`, `us/california`); TREK downloads it, reads the paddle objects * out and keeps nothing else of it. A region is read again every month. */ @@ -21,6 +21,17 @@ export const paddleRegionSchema = z.object({ feature_count: z.number().int(), /** Why the last read failed, as plain text; null otherwise. */ error: z.string().nullable(), + /** + * How far the read under way has come: the download, then the reading of + * the file, each with the share done (null while the size is unknown). + * Null unless the region is being read. + */ + progress: z + .object({ + phase: z.enum(['download', 'read']), + fraction: z.number().min(0).max(1).nullable(), + }) + .nullable(), }); export type PaddleRegion = z.infer; @@ -39,13 +50,21 @@ export const paddleRegionCatalogQuerySchema = z.object({ q: z.string().trim().mi export const paddleRegionCatalogResponseSchema = z.object({ regions: z.array(paddleRegionCatalogEntrySchema) }); export type PaddleRegionCatalogResponse = z.infer; -/** Geofabrik's region ids: lower case words, digits and dashes. */ +/** Geofabrik's region ids: lower case words, digits and dashes, a few of them nested with a slash (`us/texas`). */ export const paddleRegionIdSchema = z .string() .trim() .min(2) .max(120) - .regex(/^[a-z0-9-]+$/); + .regex(/^[a-z0-9-]+(\/[a-z0-9-]+)*$/); + +/** A region's boundary as Geofabrik draws it: a polygon or several, rings of [lng, lat]. */ +const boundaryRing = z.array(z.tuple([z.number(), z.number()])); +export const paddleRegionBoundarySchema = z.discriminatedUnion('type', [ + z.object({ type: z.literal('Polygon'), coordinates: z.array(boundaryRing) }), + z.object({ type: z.literal('MultiPolygon'), coordinates: z.array(z.array(boundaryRing)) }), +]); +export type PaddleRegionBoundary = z.infer; export const paddleRegionRequestSchema = z.object({ id: paddleRegionIdSchema }); export type PaddleRegionRequest = z.infer; diff --git a/shared/src/paddle/paddle-route-relation.spec.ts b/shared/src/paddle/paddle-route-relation.spec.ts new file mode 100644 index 000000000..cf34a9d8e --- /dev/null +++ b/shared/src/paddle/paddle-route-relation.spec.ts @@ -0,0 +1,94 @@ +import type { PaddleFeature, PaddleGeometry } from './paddle-obstacles.schema'; +import { NEAR_M, ON_ROUTE_M, relateObstaclesToRoute } from './paddle-route-relation'; + +import { describe, expect, it } from 'vitest'; + +/** Metres north and east of 49.1°N 12.1°E, near enough for these distances. */ +const M_PER_DEG_LAT = 111_195; +const M_PER_DEG_LNG = 111_195 * Math.cos((49.1 * Math.PI) / 180); +const at = (north: number, east: number): [number, number] => [ + 49.1 + north / M_PER_DEG_LAT, + 12.1 + east / M_PER_DEG_LNG, +]; +const lngLat = (north: number, east: number): [number, number] => { + const [lat, lng] = at(north, east); + return [lng, lat]; +}; + +function feature(ref: string, geometry: PaddleGeometry): PaddleFeature { + return { ref, kind: 'weir', group_key: null, name: null, geometry, tags: {}, tag_hash: ref, region_id: 'oberpfalz' }; +} + +/** A route due east along the river, 2 km long. */ +const route = [at(0, 0), at(0, 1000), at(0, 2000)]; + +describe('relateObstaclesToRoute', () => { + it('relates nothing to a route of fewer than two points', () => { + const weir = feature('node/1', { type: 'Point', coordinates: lngLat(0, 500) }); + expect(relateObstaclesToRoute([], [weir])).toEqual([]); + expect(relateObstaclesToRoute([at(0, 0)], [weir])).toEqual([]); + }); + + it('puts a weir the route crosses on the route, where it crosses', () => { + const weir = feature('way/29256405', { + type: 'LineString', + coordinates: [lngLat(-40, 1200), lngLat(40, 1200)], + }); + expect(relateObstaclesToRoute(route, [weir])).toEqual([ + { feature: weir, relation: 'on_route', distanceM: 0, alongM: 1200 }, + ]); + }); + + it('a point within ON_ROUTE_M is on the route, within NEAR_M near, farther off not related', () => { + const on = feature('node/1', { type: 'Point', coordinates: lngLat(ON_ROUTE_M - 5, 300) }); + const near = feature('node/2', { type: 'Point', coordinates: lngLat(100, 700) }); + const far = feature('node/3', { type: 'Point', coordinates: lngLat(NEAR_M + 30, 900) }); + const related = relateObstaclesToRoute(route, [near, far, on]); + expect(related.map((entry) => [entry.feature.ref, entry.relation, entry.distanceM, entry.alongM])).toEqual([ + ['node/1', 'on_route', 20, 300], + ['node/2', 'near', 100, 700], + ]); + }); + + it('measures from the route to a long line too, so passing its middle counts', () => { + // A dam 60 m north of the route, running 1 km parallel to it: its ends are far, its middle is not. + const dam = feature('way/7', { type: 'LineString', coordinates: [lngLat(60, -2000), lngLat(60, 4000)] }); + const [related] = relateObstaclesToRoute(route, [dam]); + expect(related).toMatchObject({ relation: 'near', distanceM: 60, alongM: 0 }); + }); + + it('a route running inside an area meets it, even without crossing its outline', () => { + const basin = feature('way/9', { + type: 'Polygon', + coordinates: [[lngLat(-500, -500), lngLat(-500, 2500), lngLat(500, 2500), lngLat(500, -500), lngLat(-500, -500)]], + }); + expect(relateObstaclesToRoute(route, [basin])).toEqual([ + { feature: basin, relation: 'on_route', distanceM: 0, alongM: 0 }, + ]); + }); + + it('a route inside a hole of an area does not meet it there', () => { + const ring = feature('way/10', { + type: 'Polygon', + coordinates: [ + [lngLat(-900, -900), lngLat(-900, 2900), lngLat(900, 2900), lngLat(900, -900), lngLat(-900, -900)], + [lngLat(-600, -600), lngLat(-600, 2600), lngLat(600, 2600), lngLat(600, -600), lngLat(-600, -600)], + ], + }); + expect(relateObstaclesToRoute(route, [ring])).toEqual([]); + }); + + it('orders the related objects as the route reaches them', () => { + const late = feature('node/late', { type: 'Point', coordinates: lngLat(0, 1800) }); + const early = feature('node/early', { type: 'Point', coordinates: lngLat(0, 200) }); + expect(relateObstaclesToRoute(route, [late, early]).map((entry) => entry.feature.ref)).toEqual([ + 'node/early', + 'node/late', + ]); + }); + + it('an object on the next arm of the river, 120 m off, stays near and so uncertain', () => { + const otherArm = feature('way/11', { type: 'LineString', coordinates: [lngLat(100, 500), lngLat(160, 500)] }); + expect(relateObstaclesToRoute(route, [otherArm])[0]).toMatchObject({ relation: 'near', distanceM: 100 }); + }); +}); diff --git a/shared/src/paddle/paddle-route-relation.ts b/shared/src/paddle/paddle-route-relation.ts index 035dc504f..626f04a49 100644 --- a/shared/src/paddle/paddle-route-relation.ts +++ b/shared/src/paddle/paddle-route-relation.ts @@ -54,28 +54,112 @@ function crosses(a: Local, b: Local, c: Local, d: Local): number | null { return t >= 0 && t <= 1 && u >= 0 && u <= 1 ? t : null; } -/** The object's outline as segments (a point is one segment of length zero). */ -function segmentsOf(geometry: PaddleGeometry, project: (lat: number, lng: number) => Local): Array<[Local, Local]> { - const line = - geometry.type === 'Point' - ? [geometry.coordinates, geometry.coordinates] - : geometry.type === 'LineString' - ? geometry.coordinates - : (geometry.coordinates[0] ?? []); - const points = line.map(([lng, lat]) => project(lat, lng)); +/** The rings of an object: a point is one ring of one point, a line one open ring, an area its outline and holes. */ +function ringsOf(geometry: PaddleGeometry): ReadonlyArray> { + if (geometry.type === 'Point') return [[geometry.coordinates, geometry.coordinates]]; + if (geometry.type === 'LineString') return [geometry.coordinates]; + return geometry.coordinates; +} + +function segmentsOf(rings: ReadonlyArray): Array<[Local, Local]> { const segments: Array<[Local, Local]> = []; - for (let i = 1; i < points.length; i++) { - const a = points[i - 1]; - const b = points[i]; - if (a && b) segments.push([a, b]); + for (const ring of rings) { + for (let i = 1; i < ring.length; i++) { + const a = ring[i - 1]; + const b = ring[i]; + if (a && b) segments.push([a, b]); + } } return segments; } +function inRing(p: Local, ring: readonly Local[]): boolean { + let inside = false; + for (let i = 0, j = ring.length - 1; i < ring.length; j = i++) { + const a = ring[i] as Local; + const b = ring[j] as Local; + if (a.y > p.y !== b.y > p.y && p.x < ((b.x - a.x) * (p.y - a.y)) / (b.y - a.y) + a.x) inside = !inside; + } + return inside; +} + +/** Inside an area: within its outline and in none of its holes. */ +function inArea(p: Local, rings: readonly (readonly Local[])[]): boolean { + const [outer, ...holes] = rings; + return !!outer && inRing(p, outer) && !holes.some((hole) => inRing(p, hole)); +} + +interface Bounds { + minX: number; + minY: number; + maxX: number; + maxY: number; +} + +function boundsOf(points: readonly Local[], margin: number): Bounds { + const bounds = { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity }; + for (const p of points) { + bounds.minX = Math.min(bounds.minX, p.x - margin); + bounds.minY = Math.min(bounds.minY, p.y - margin); + bounds.maxX = Math.max(bounds.maxX, p.x + margin); + bounds.maxY = Math.max(bounds.maxY, p.y + margin); + } + return bounds; +} + +function overlaps(a: Bounds, b: Bounds): boolean { + return a.minX <= b.maxX && b.minX <= a.maxX && a.minY <= b.maxY && b.minY <= a.maxY; +} + +/** + * The closest the route comes to one object: the distance and how far along + * the route that is. A route that crosses the outline, or runs inside an + * area, meets it at distance 0, at the first place it does. + */ +function closestApproach( + path: readonly Local[], + along: readonly number[], + rings: readonly (readonly Local[])[], + isArea: boolean, +): { distance: number; alongM: number } { + const outline = segmentsOf(rings); + const reach = boundsOf(rings.flat(), NEAR_M); + let best = { distance: Number.POSITIVE_INFINITY, alongM: 0 }; + const consider = (distance: number, alongM: number) => { + if (distance < best.distance || (distance === best.distance && alongM < best.alongM)) best = { distance, alongM }; + }; + path.forEach((p, i) => { + if (isArea && inArea(p, rings)) consider(0, along[i] ?? 0); + }); + for (let i = 1; i < path.length; i++) { + const a = path[i - 1] as Local; + const b = path[i] as Local; + if (!overlaps(boundsOf([a, b], 0), reach)) continue; + const start = along[i - 1] ?? 0; + const length = (along[i] ?? start) - start; + for (const [c, d] of outline) { + const hit = crosses(a, b, c, d); + if (hit !== null) { + consider(0, start + hit * length); + continue; + } + for (const p of [c, d]) { + const near = closestOnSegment(p, a, b); + consider(near.distance, start + near.t * length); + } + consider(closestOnSegment(a, c, d).distance, start); + consider(closestOnSegment(b, c, d).distance, start + length); + } + } + return best; +} + /** * The obstacles on or near a route, in the order the route reaches them. The * route is `[lat, lng]` pairs as the planner draws it; an empty or one-point - * route relates nothing. + * route relates nothing. The distance is measured both ways, from the route + * to the object's outline and from the object to the route, so a long weir + * passed in its middle and a route inside an area both count. */ export function relateObstaclesToRoute( route: readonly LatLng[], @@ -90,27 +174,12 @@ export function relateObstaclesToRoute( const b = path[i] as Local; along.push((along[i - 1] ?? 0) + Math.hypot(b.x - a.x, b.y - a.y)); } + const routeBounds = boundsOf(path, NEAR_M); const related: RelatedObstacle[] = []; for (const feature of features) { - let best = { distance: Number.POSITIVE_INFINITY, alongM: 0 }; - const outline = segmentsOf(feature.geometry, project); - for (let i = 1; i < path.length; i++) { - const a = path[i - 1] as Local; - const b = path[i] as Local; - const start = along[i - 1] ?? 0; - const length = (along[i] ?? start) - start; - for (const [c, d] of outline) { - const hit = crosses(a, b, c, d); - if (hit !== null) { - if (best.distance > 0 || start + hit * length < best.alongM) best = { distance: 0, alongM: start + hit * length }; - continue; - } - for (const p of [c, d]) { - const near = closestOnSegment(p, a, b); - if (near.distance < best.distance) best = { distance: near.distance, alongM: start + near.t * length }; - } - } - } + const rings = ringsOf(feature.geometry).map((ring) => ring.map(([lng, lat]) => project(lat, lng))); + if (!overlaps(boundsOf(rings.flat(), 0), routeBounds)) continue; + const best = closestApproach(path, along, rings, feature.geometry.type === 'Polygon'); if (best.distance > NEAR_M) continue; related.push({ feature, diff --git a/shared/src/tours/paddle-plan.spec.ts b/shared/src/tours/paddle-plan.spec.ts index 54a2b0783..274b69aea 100644 --- a/shared/src/tours/paddle-plan.spec.ts +++ b/shared/src/tours/paddle-plan.spec.ts @@ -1,5 +1,3 @@ -import { describe, expect, it } from 'vitest'; - import { estimatePaddleTime, paddleLayoutOfPoints, @@ -11,12 +9,19 @@ import { import { tourPaddlePlanRequestSchema, trackMarksSchema } from './paddle-plan.schema'; import { tourCreateRequestSchema } from './tours.schema'; +import { describe, expect, it } from 'vitest'; + /** A point `km` kilometres east of 53°N 13°E along the parallel, near enough for these sums. */ function east(km: number): { lat: number; lng: number } { return { lat: 53, lng: 13 + km / (111.19493 * Math.cos((53 * Math.PI) / 180)) }; } -function leg(kind: 'free' | 'portage', from: { lat: number; lng: number }, to: { lat: number; lng: number }, extra?: number) { +function leg( + kind: 'free' | 'portage', + from: { lat: number; lng: number }, + to: { lat: number; lng: number }, + extra?: number, +) { return { kind, geometry: [ @@ -155,7 +160,13 @@ describe('paddle plan (PAD-03, PAD-07)', () => { name: 'Carry', route_geometry: route, waypoints: [ - { lat: 53, lng: 13, role: 'start', sequence: 0, fixed_leg: leg('portage', { lat: 53, lng: 13 }, { lat: 53, lng: 13.01 }) }, + { + lat: 53, + lng: 13, + role: 'start', + sequence: 0, + fixed_leg: leg('portage', { lat: 53, lng: 13 }, { lat: 53, lng: 13.01 }), + }, { lat: 53, lng: 13.01, role: 'end', sequence: 1, lock: { wait_minutes: 5, transit_minutes: 5 } }, ], }; diff --git a/shared/src/tours/paddle-plan.ts b/shared/src/tours/paddle-plan.ts index 9f3c4d483..e91568743 100644 --- a/shared/src/tours/paddle-plan.ts +++ b/shared/src/tours/paddle-plan.ts @@ -142,7 +142,8 @@ export function paddleLayoutOfTrack(totalM: number, marks: TrackMarks | null | u }); cursor = mark.to_m; } - if (totalM > cursor) stretches.push({ kind: 'water', fromM: cursor, toM: totalM, extraMinutes: null, startIndex: null }); + if (totalM > cursor) + stretches.push({ kind: 'water', fromM: cursor, toM: totalM, extraMinutes: null, startIndex: null }); const locks = (marks?.locks ?? []) .filter((lock) => lock.at_m <= totalM) .map((lock) => ({ diff --git a/shared/src/tours/tours.schema.ts b/shared/src/tours/tours.schema.ts index 5987636cd..f8b48f748 100644 --- a/shared/src/tours/tours.schema.ts +++ b/shared/src/tours/tours.schema.ts @@ -1,4 +1,9 @@ -import { paddleLockSchema, paddleStopMinutesSchema, portageSpeedKmhSchema, trackMarksSchema } from './paddle-plan.schema'; +import { + paddleLockSchema, + paddleStopMinutesSchema, + portageSpeedKmhSchema, + trackMarksSchema, +} from './paddle-plan.schema'; import { tourOriginSchema, tourProvenanceSchema } from './tour-provenance.schema'; import { z } from 'zod'; @@ -340,7 +345,8 @@ function tourTypeIssues(value: { (value.waypoints ?? []).forEach((point, index) => { if (point.fixed_leg?.kind === 'portage') issues.push({ path: ['waypoints', index, 'fixed_leg'], message: 'a portage belongs to a paddle tour' }); - if (point.lock != null) issues.push({ path: ['waypoints', index, 'lock'], message: 'a lock belongs to a paddle tour' }); + if (point.lock != null) + issues.push({ path: ['waypoints', index, 'lock'], message: 'a lock belongs to a paddle tour' }); }); return issues; } diff --git a/shared/src/weather/wind.schema.spec.ts b/shared/src/weather/wind.schema.spec.ts new file mode 100644 index 000000000..fcfd24e61 --- /dev/null +++ b/shared/src/weather/wind.schema.spec.ts @@ -0,0 +1,51 @@ +import { COMPASS_POINTS, compassPointOf, windForecastResponseSchema, windQuerySchema } from './wind.schema'; + +import { describe, expect, it } from 'vitest'; + +describe('windQuerySchema (PAD-08)', () => { + it('reads the place from the query string and wants a day', () => { + expect(windQuerySchema.parse({ lat: '47.55', lng: '9.68', date: '2026-10-18' })).toEqual({ + lat: 47.55, + lng: 9.68, + date: '2026-10-18', + }); + expect(windQuerySchema.safeParse({ lat: '47.55', lng: '9.68' }).success).toBe(false); + expect(windQuerySchema.safeParse({ lat: '91', lng: '9.68', date: '2026-10-18' }).success).toBe(false); + expect(windQuerySchema.safeParse({ lat: '47.55', lng: '9.68', date: '18.10.2026' }).success).toBe(false); + }); +}); + +describe('windForecastResponseSchema', () => { + it('a day without a forecast carries its status and no figures; a missing value stays null, never zero', () => { + const past = { + status: 'past', + date: '2026-10-01', + lat: 47.55, + lng: 9.68, + timezone: null, + source: 'Open-Meteo', + fetched_at: '2026-10-16T10:00:00.000Z', + horizon_days: 16, + hours: [], + day: null, + }; + expect(windForecastResponseSchema.safeParse(past).success).toBe(true); + const hour = { hour: 9, speed_kmh: 12, gusts_kmh: null, direction_deg: null }; + expect(windForecastResponseSchema.safeParse({ ...past, status: 'forecast', hours: [hour] }).success).toBe(true); + expect(windForecastResponseSchema.safeParse({ ...past, source: 'Climate' }).success).toBe(false); + }); +}); + +describe('compassPointOf', () => { + it('names the sixteen points, wrapping around north', () => { + expect(compassPointOf(0)).toBe('N'); + expect(compassPointOf(11)).toBe('N'); + expect(compassPointOf(12)).toBe('NNE'); + expect(compassPointOf(90)).toBe('E'); + expect(compassPointOf(225)).toBe('SW'); + expect(compassPointOf(349)).toBe('N'); + expect(compassPointOf(-90)).toBe('W'); + expect(compassPointOf(720 + 180)).toBe('S'); + expect(COMPASS_POINTS).toHaveLength(16); + }); +});