diff --git a/charts/trek/templates/configmap.yaml b/charts/trek/templates/configmap.yaml index 27b84c024..128b4f2c4 100644 --- a/charts/trek/templates/configmap.yaml +++ b/charts/trek/templates/configmap.yaml @@ -118,6 +118,9 @@ data: {{- if .Values.env.TREK_SERVICES_HOSTS }} TREK_SERVICES_HOSTS: {{ .Values.env.TREK_SERVICES_HOSTS | quote }} {{- end }} + {{- if .Values.env.TREK_PHOTON_URL }} + TREK_PHOTON_URL: {{ .Values.env.TREK_PHOTON_URL | quote }} + {{- end }} {{- if .Values.env.GOOGLE_LIST_DATA_BASE }} GOOGLE_LIST_DATA_BASE: {{ .Values.env.GOOGLE_LIST_DATA_BASE | quote }} {{- end }} diff --git a/charts/trek/values.yaml b/charts/trek/values.yaml index 97980fafb..4f1c895de 100644 --- a/charts/trek/values.yaml +++ b/charts/trek/values.yaml @@ -92,7 +92,8 @@ env: # Custom Nominatim base URL for every geocoding call the server makes (place search, reverse lookup, Atlas region # fill). When set it REPLACES the public OpenStreetMap service, with no fallback, so the instance behind it needs # the coverage you expect and has to speak the Nominatim API. TREK keeps its ~1 request per second spacing either - # way. Defaults to https://nominatim.openstreetmap.org. + # way, and search and reverse lookups no longer go to the TREK services' Photon. Defaults to + # https://nominatim.openstreetmap.org. # OVERPASS_URL: "" # Custom Overpass endpoint(s) for the map POI "explore" search, comma-separated. When set, REPLACES the bundled # public mirrors — point it at an internal/self-hosted Overpass instance when the public mirrors are unreachable @@ -117,7 +118,12 @@ env: # database. "off" sends no key; searches then use the shared pool. A key issued to you goes under # secretEnv.TREK_SERVICES_KEY. Defaults to auto. # TREK_SERVICES_HOSTS: "" - # Comma-separated hosts that receive the installation key over https. Defaults to places.liketrek.com. + # Comma-separated hosts that receive the installation key over https. Defaults to + # places.liketrek.com,photon.liketrek.com,valhalla.liketrek.com. + # TREK_PHOTON_URL: "" + # The address search (Photon) of the TREK services. With a key, search and reverse lookups ask it before + # Nominatim and fall back to Nominatim when it does not answer. Never asked when NOMINATIM_URL is set. + # Defaults to https://photon.liketrek.com. # AMAP_API_BASE: "" # Send the Amap (高德地图) calls through an egress proxy, a cache or a gateway that speaks the same API instead of # restapi.amap.com directly. The key itself is a credential and goes under secretEnv.AMAP_API_KEY, or into diff --git a/docker-compose.yml b/docker-compose.yml index ac6e9c97b..a8a0fe820 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,7 @@ services: # - TREK_PLACES_URL=https://places.liketrek.com # Optional. Base URL of a copy of the Places API you run yourself; it has to answer the same /v1 API. Read from configuration rather than typed by a user, so an address on your LAN or Docker network works without ALLOW_INTERNAL_NETWORK. (default: https://places.liketrek.com) # - TREK_SERVICES=off # Optional. Places search identifies this installation with a key it registers by itself on first use. off sends no key; searches then use the shared pool. (default: auto) # - TREK_SERVICES_KEY= # Optional. A key issued for this installation; it wins over the stored one and turns self-registration off. +# - TREK_PHOTON_URL=https://photon.liketrek.com # Optional. The address search of the TREK services. With a key, search and reverse lookups ask it before Nominatim and fall back to Nominatim when it does not answer; never asked when NOMINATIM_URL is set. (default: https://photon.liketrek.com) # - PLACES_API_KEY= # Optional. Google Maps API key (Places API (New) enabled) for place search, photos, details and Google transit. Overrides the key set in Admin > Settings > API Keys, whose field then shows this variable read-only. # - GOOGLE_LIST_DATA_BASE= # Optional. HTTP(S) base URL of a list-data service. Pasted URLs beginning with this base are imported directly; unset keeps the Google flow. # - AMAP_API_KEY= # Optional. Amap (高德地图) Web 服务 key for place search and geocoding inside mainland China. Overrides the key set in Admin > Settings > API Keys. A Web 端 (JS API) key is a different credential and is rejected with INVALID_USER_KEY. diff --git a/server/.env.example b/server/.env.example index 494db90cf..711b08ce5 100644 --- a/server/.env.example +++ b/server/.env.example @@ -63,7 +63,7 @@ DEMO_MODE=false # Demo mode: resets data hourly # MCP_SSE_KEEPALIVE=25 # Seconds between keep-alive pings on an open MCP event stream, so a reverse proxy does not cut it. 0 turns the pings off. (default: 25) # IDEMPOTENCY_TTL_SECONDS=2592000 # How long the server remembers an X-Idempotency-Key. The offline client replays queued changes with their key on reconnect, so keep this longer than the longest time a device stays offline. (default: 2592000, 30 days) -# NOMINATIM_URL=https://nominatim.openstreetmap.org # Custom HTTP(S) Nominatim base URL for every geocoding call the server makes: place search, reverse lookup, place details and the Atlas region fill. When set it REPLACES the public service with no fallback, so whatever answers has to speak the Nominatim API (/search, /reverse, /lookup) and hold the coverage you expect. TREK keeps its ~1 request per second spacing whichever instance answers. Unset/blank uses the public default. +# NOMINATIM_URL=https://nominatim.openstreetmap.org # Custom HTTP(S) Nominatim base URL for every geocoding call the server makes: place search, reverse lookup, place details and the Atlas region fill. When set it REPLACES the public service with no fallback, so whatever answers has to speak the Nominatim API (/search, /reverse, /lookup) and hold the coverage you expect. TREK keeps its ~1 request per second spacing whichever instance answers. Unset/blank uses the public default. Set, it also keeps search and reverse lookups off the TREK services' Photon. # OVERPASS_URL= # Custom Overpass endpoint(s) for the map POI "explore" search, comma-separated. When set, REPLACES the bundled public mirrors — point it at an internal/self-hosted Overpass instance when the public ones are unreachable from your network. Non-http(s) entries are ignored. If you don't self-host Overpass but the public mirrors throttle you, setting APP_URL also gives outbound requests a unique User-Agent the mirrors rate-limit less. # OVERPASS_TIMEOUT_MS=25000 # Per-endpoint timeout (ms) for Overpass POI requests; slower endpoints are abandoned so a faster mirror wins. The query itself gives Overpass 20 seconds of work, so a lower value hangs up on answers that were still coming. Raise it for a slow self-hosted instance. The Tours search along a tour does not read it: it gives Overpass 10 seconds of work and waits 12. (default: 25000) @@ -72,7 +72,8 @@ DEMO_MODE=false # Demo mode: resets data hourly # TREK_PLACES_URL=https://places.liketrek.com # Base URL of a copy of the Places API you run yourself; it has to answer the same /v1 API. Read from configuration rather than typed by a user, so an address on your LAN or Docker network works without ALLOW_INTERNAL_NETWORK. (default: https://places.liketrek.com) # TREK_SERVICES=auto # How this installation identifies itself to the TREK services. On first use it registers itself in the background and keeps a key in the database; nothing to click, no account. off sends no key, and searches use the shared pool. (default: auto) # TREK_SERVICES_KEY= # A key issued to you for this installation. Wins over the stored one and turns self-registration off; meant for hosted and pre-provisioned instances. (default: none) -# TREK_SERVICES_HOSTS=places.liketrek.com # Comma-separated hosts that receive the installation key over https. No other host ever sees it. (default: places.liketrek.com) +# TREK_SERVICES_HOSTS=places.liketrek.com,photon.liketrek.com,valhalla.liketrek.com # Comma-separated hosts that receive the installation key over https. No other host ever sees it. (default: places.liketrek.com,photon.liketrek.com,valhalla.liketrek.com) +# TREK_PHOTON_URL=https://photon.liketrek.com # The address search (Photon) of the TREK services. Once the installation holds a key, search and reverse lookups ask it before Nominatim and fall back to Nominatim whenever it does not answer. Never asked with TREK_SERVICES=off, without a key, or when NOMINATIM_URL is set. (default: https://photon.liketrek.com) # PLACES_API_KEY= # Google Maps API key (Places API (New) enabled) for place search, photos, details and Google transit. Overrides the key set in Admin > Settings > API Keys, whose field then shows this variable read-only. The value never reaches the browser. # GOOGLE_LIST_DATA_BASE= # Optional HTTP(S) base URL of a list-data service, e.g. https://maps.example.com. Paste a URL beginning with this base to fetch and parse it directly. Unset keeps the original Google list import flow. diff --git a/server/src/app-config/boot-derive.ts b/server/src/app-config/boot-derive.ts index 9ec357d43..6d658b1e5 100644 --- a/server/src/app-config/boot-derive.ts +++ b/server/src/app-config/boot-derive.ts @@ -63,7 +63,10 @@ export function deriveKitinerary(raw: RawEnv) { } /** The liketrek hosts that receive the installation key when nothing else is configured. */ -const DEFAULT_SERVICE_HOSTS = ['places.liketrek.com']; +const DEFAULT_SERVICE_HOSTS = ['places.liketrek.com', 'photon.liketrek.com', 'valhalla.liketrek.com']; + +/** The address search of the TREK services, asked before Nominatim while the installation holds a key. */ +const DEFAULT_PHOTON_URL = 'https://photon.liketrek.com'; /** * How this installation identifies itself to the liketrek services. Unset is @@ -77,6 +80,7 @@ export function deriveServices(raw: RawEnv) { mode: mode === 'auto' ? ('auto' as const) : ('off' as const), key: raw.TREK_SERVICES_KEY?.trim() || null, hosts: hosts?.length ? hosts : DEFAULT_SERVICE_HOSTS, + photonUrl: stripTrailingSlashes(raw.TREK_PHOTON_URL?.trim() || DEFAULT_PHOTON_URL), }; } diff --git a/server/src/app-config/derive.ts b/server/src/app-config/derive.ts index f78ab0fb8..871b38442 100644 --- a/server/src/app-config/derive.ts +++ b/server/src/app-config/derive.ts @@ -284,6 +284,8 @@ export function deriveIntegrations(raw: RawEnv) { // treats a blank one as unset, so a padded or whitespace-only value would // otherwise pass startup and then be the string that cannot be fetched. nominatimUrl: stripTrailingSlashes(raw.NOMINATIM_URL?.trim() || 'https://nominatim.openstreetmap.org'), + /** The operator runs a geocoder of their own, so search and reverse stay on it instead of going to Photon. */ + nominatimOwn: Boolean(raw.NOMINATIM_URL?.trim()), overpassUrl: raw.OVERPASS_URL, // Longer than the `[timeout:20]` the query itself carries, or we abort an answer the // mirror was still allowed to be working on. See OVERPASS_QUERY_TIMEOUT_S. diff --git a/server/src/app-config/env.schema.ts b/server/src/app-config/env.schema.ts index 70592706b..937ddf675 100644 --- a/server/src/app-config/env.schema.ts +++ b/server/src/app-config/env.schema.ts @@ -188,6 +188,7 @@ export const envSchema = z.object({ TREK_SERVICES: oneOf(['auto', 'off']), TREK_SERVICES_KEY: optionalWith(isServiceKeyShape, 'must be a TREK services key (tk1...)'), TREK_SERVICES_HOSTS: anyString, + TREK_PHOTON_URL: url, PLACES_API_KEY: anyString, AMAP_API_BASE: url, AMAP_API_KEY: anyString, diff --git a/server/src/nest/maps/maps.module.ts b/server/src/nest/maps/maps.module.ts index e3ad456d8..c2120de3e 100644 --- a/server/src/nest/maps/maps.module.ts +++ b/server/src/nest/maps/maps.module.ts @@ -2,6 +2,7 @@ import { AppSettings } from '../../db/entities/AppSettings.entity'; import { PlaceDetailsCache } from '../../db/entities/PlaceDetailsCache.entity'; import { Places } from '../../db/entities/Places.entity'; import { Users } from '../../db/entities/Users.entity'; +import { AppConfigModule } from '../app-config/app-config.module'; import { GoogleQuotaModule } from '../google-quota/google-quota.module'; import { PlacePhotosModule } from '../place-photos/place-photos.module'; import { StorageModule } from '../storage/storage.module'; @@ -15,6 +16,7 @@ import { PlacesProviderSelector } from './places-provider.selector'; import { PoisAlongResolver } from './pois-along.resolver'; import { GooglePlacesClient } from './providers/google-places.provider'; import { OsmClient } from './providers/osm.client'; +import { PhotonClient } from './providers/photon.client'; import { WikimediaClient } from './providers/wikimedia.client'; import { TrekPlacesLayersService } from './trek-places-layers.service'; import { MikroOrmModule } from '@mikro-orm/nestjs'; @@ -26,7 +28,8 @@ import { Module } from '@nestjs/common'; * MapsService is the orchestrator every consumer injects (BookingImportModule's * geocoding, PlacesModule's search_place tool and list-import enrichment, the * road trip search). Behind it sit one provider per outbound source - * (GooglePlacesClient, OsmClient, WikimediaClient; Amap is built per request by + * (GooglePlacesClient, OsmClient with the PhotonClient it asks before + * Nominatim, WikimediaClient; Amap is built per request by * PlacesProviderSelector) and the resolvers it delegates to. The three clients * are exported too, so the enrichment column asks Google, OpenStreetMap and * Wikimedia directly rather than through a MapsService facade, and @@ -34,7 +37,8 @@ import { Module } from '@nestjs/common'; * routes layers before it asks the Overpass mirrors. Nothing outside * the container consumes this domain, so there is no bridge. * - * MikroOrmModule.forFeature: AppSettings and Users for the key chain + * AppConfigModule for the servicesConfig token PhotonClient reads its address + * from. MikroOrmModule.forFeature: AppSettings and Users for the key chain * (instance-api-keys.ts's resolveApiKey) and the kill switches, * PlaceDetailsCache for the details cache (the entity only, not * place-enrichment's module, so there is no new module edge) and Places for the @@ -42,6 +46,7 @@ import { Module } from '@nestjs/common'; */ @Module({ imports: [ + AppConfigModule, PlacePhotosModule, StorageModule, GoogleQuotaModule, @@ -53,6 +58,7 @@ import { Module } from '@nestjs/common'; MapsMcp, GooglePlacesClient, OsmClient, + PhotonClient, WikimediaClient, PlacesProviderSelector, PlacePhotoResolver, diff --git a/server/src/nest/maps/providers/osm.client.ts b/server/src/nest/maps/providers/osm.client.ts index 6c773422f..32d9eb409 100644 --- a/server/src/nest/maps/providers/osm.client.ts +++ b/server/src/nest/maps/providers/osm.client.ts @@ -15,6 +15,12 @@ * search along a line takes after every Overpass endpoint failed it is the * instance's own: MapsModule provides one client per process. * + * Search and reverse ask Photon first (PhotonClient) where the TREK services + * hold a key and the operator runs no Nominatim of their own; its answers are + * read into Nominatim's shape, so both come out of the same mapping. Lookups + * by id and the identity match stay on Nominatim: they need its id lookup and + * its extratags. + * * Never imports MapsService: providers sit below the orchestrator * (lint:boundaries holds that). */ @@ -41,6 +47,8 @@ import { } from '../maps.helpers'; import { overpassSelector, tagsMatchFilter } from '../osm-filter.helpers'; import { OverpassBusy, hedged, lastAnsweredFirst, retryAfterMs, soonestRetryMs } from '../overpass-hedge.helpers'; +import { PhotonClient } from './photon.client'; +import { asNominatimHit, asNominatimReverse } from './photon.mapping'; import { readWikiIdentity } from './wiki-identity'; import { Injectable } from '@nestjs/common'; import { normalizePlaceWebsite } from '@trek/shared'; @@ -51,7 +59,7 @@ function nominatimCategory(item: { class?: string; type?: string }): string | nu return item.class === 'shop' ? `shop_${item.type}` : item.type; } -interface NominatimResult { +export interface NominatimResult { osm_type: string; osm_id: string; /** The main OSM tag, key and value: `class` "tourism" with `type` "hotel". */ @@ -99,6 +107,35 @@ export interface NominatimReverse { address?: Record; } +/** A search row as TREK's place record. */ +function searchRecord(item: NominatimResult) { + // Number.isFinite, not `|| null`: a place on the equator or prime + // meridian has a legitimate 0 coordinate. + const lat = Number.parseFloat(item.lat); + const lng = Number.parseFloat(item.lon); + return { + google_place_id: null, + google_ftid: null, + osm_id: `${item.osm_type}:${item.osm_id}`, + name: item.name || item.display_name?.split(',')[0] || '', + address: item.display_name || '', + lat: Number.isFinite(lat) ? lat : null, + lng: Number.isFinite(lng) ? lng : null, + rating: null, + website: null, + phone: null, + // What the place is, so the add form can preselect a category (#2282). A shop's + // value is the ware ("bakery"), so the key goes first and says it is a shop. + category: nominatimCategory(item), + source: 'openstreetmap', + ...readWikiIdentity(item.extratags), + }; +} + +function isHit(hit: NominatimResult | null): hit is NominatimResult { + return hit !== null; +} + // Tighter than the wiki calls, because this one sits at the FRONT of a chain: // identity, then sitelinks, then the extract. Nominatim answers a bounded // search in 0.2-0.7s in practice, so anything past a couple of seconds is a bad @@ -402,6 +439,8 @@ export class OsmClient { /** Whether this cooldown's first skip was logged, so the log holds one line per cooldown. */ private overpassAlongSkipLogged = false; + constructor(private readonly photon: PhotonClient) {} + // ── Nominatim search ─────────────────────────────────────────────────────── /** @@ -420,6 +459,8 @@ export class OsmClient { lane: GeoLane = 'interactive', bias?: { lat: number; lng: number }, ) { + const photon = await this.photon.search(query, { lang, bias }); + if (photon) return photon.map(asNominatimHit).filter(isHit).map(searchRecord); const params = new URLSearchParams({ q: query, format: 'json', @@ -450,29 +491,7 @@ export class OsmClient { ); } const data = (await response.json()) as NominatimResult[]; - return data.map((item) => { - // Number.isFinite, not `|| null`: a place on the equator or prime - // meridian has a legitimate 0 coordinate. - const lat = Number.parseFloat(item.lat); - const lng = Number.parseFloat(item.lon); - return { - google_place_id: null, - google_ftid: null, - osm_id: `${item.osm_type}:${item.osm_id}`, - name: item.name || item.display_name?.split(',')[0] || '', - address: item.display_name || '', - lat: Number.isFinite(lat) ? lat : null, - lng: Number.isFinite(lng) ? lng : null, - rating: null, - website: null, - phone: null, - // What the place is, so the add form can preselect a category (#2282). A shop's - // value is the ware ("bakery"), so the key goes first and says it is a shop. - category: nominatimCategory(item), - source: 'openstreetmap', - ...readWikiIdentity(item.extratags), - }; - }); + return data.map(searchRecord); } /** Suggestions out of a Nominatim search, for an install with neither the index nor a key. */ @@ -630,13 +649,38 @@ export class OsmClient { // ── Nominatim reverse ────────────────────────────────────────────────────── - /** The name and address Nominatim gives a point, or nulls when it answers with an error. */ + /** The name and address Photon or Nominatim gives a point, or nulls when Nominatim answers with an error. */ async reverse( lat: string, lng: string, lang?: string, opts?: { lane?: GeoLane; timeoutMs?: number; locality?: boolean }, ): Promise<{ name: string | null; address: string | null }> { + const data = (await this.reversePhoton(lat, lng, lang)) ?? (await this.reverseNominatim(lat, lng, lang, opts)); + if (!data) return { name: null, address: null }; + const addr = data.address || {}; + const name = opts?.locality + ? addr.city || addr.town || addr.village || addr.municipality || data.name || null + : data.name || addr.tourism || addr.amenity || addr.shop || addr.building || addr.road || null; + return { name, address: data.display_name || null }; + } + + /** + * Photon's nearest feature, or null for Nominatim to answer. Photon looks + * within a radius and can find nothing at all, where Nominatim names the + * nearest object at any distance, so an empty answer is asked again there. + */ + private async reversePhoton(lat: string, lng: string, lang?: string): Promise { + const found = await this.photon.reverse(Number.parseFloat(lat), Number.parseFloat(lng), lang); + return found?.[0] ? asNominatimReverse(found[0]) : null; + } + + private async reverseNominatim( + lat: string, + lng: string, + lang?: string, + opts?: { lane?: GeoLane; timeoutMs?: number; locality?: boolean }, + ): Promise { const params = new URLSearchParams({ lat, lon: lng, @@ -646,21 +690,18 @@ export class OsmClient { 'accept-language': toApiLang(lang), }); const response = await nominatimFetch('reverse', params, opts); - if (!response.ok) return { name: null, address: null }; - const data = (await response.json()) as NominatimReverse; - const addr = data.address || {}; - const name = opts?.locality - ? addr.city || addr.town || addr.village || addr.municipality || data.name || null - : data.name || addr.tourism || addr.amenity || addr.shop || addr.building || addr.road || null; - return { name, address: data.display_name || null }; + if (!response.ok) return null; + return (await response.json()) as NominatimReverse; } /** - * The raw reverse answer for a point a pasted link carried, in Nominatim's - * default language. A non-ok answer (5xx, 429) is an empty object: the + * The raw reverse answer for a point a pasted link carried, in the names + * used on the spot. A non-ok answer (5xx, 429) is an empty object: the * coordinates are already known, so the caller still has something to return. */ async reverseRaw(lat: number, lng: number): Promise { + const found = await this.photon.reverse(lat, lng); + if (found?.[0]) return asNominatimReverse(found[0]); const res = await nominatimFetch( 'reverse', new URLSearchParams({ lat: String(lat), lon: String(lng), format: 'json', addressdetails: '1' }), diff --git a/server/src/nest/maps/providers/photon.client.ts b/server/src/nest/maps/providers/photon.client.ts new file mode 100644 index 000000000..56f53cfad --- /dev/null +++ b/server/src/nest/maps/providers/photon.client.ts @@ -0,0 +1,115 @@ +/** + * Photon, the address search of the TREK services, asked before Nominatim. + * + * It answers the forward search and the reverse lookup for an installation + * that holds a key for the TREK services and has not set NOMINATIM_URL: an + * operator who runs a geocoder of their own keeps it. Photon has no shared + * pool, so without a key it is not asked at all. + * + * Every miss is null and the caller asks Nominatim instead: a refused key, a + * spent allowance, a timeout, an error or an answer that is not Photon's all + * read to the traveller as the search they had before. After a few misses in + * a row it stops asking for a minute, as the Places client does, so an + * installation that cannot reach it does not pay the deadline on every search. + * + * Never imports MapsService: providers sit below the orchestrator. + */ +import { readEnv } from '../../../app-config'; +import { capResponse } from '../../../utils/ssrfGuard'; +import { servicesConfig } from '../../app-config/tokens'; +import { logDebug, logWarn } from '../../audit/audit-log.logger'; +import { serviceIdentitySlot } from '../../service-identity/service-identity.interface'; +import { photonAnswerSchema, photonLang, type PhotonFeature } from './photon.mapping'; +import { Inject, Injectable } from '@nestjs/common'; +import type { ConfigType } from '@nestjs/config'; + +/** Short, because Nominatim still has to fit behind it inside the browser's wait. */ +const TIMEOUT_MS = 2500; +/** Ten hits are a few kB. */ +const MAX_BYTES = 512_000; +const SEARCH_LIMIT = 10; +/** + * How far the bias reaches. Nominatim is handed a box of a degree around the + * point; zoom 10 is Photon's spelling of about that. + */ +const BIAS_ZOOM = 10; +const MISSES_BEFORE_PAUSE = 4; +const PAUSE_MS = 60_000; + +interface PhotonSearchOptions { + lang?: string; + bias?: { lat: number; lng: number }; +} + +@Injectable() +export class PhotonClient { + private misses = 0; + private pausedUntil = 0; + + constructor(@Inject(servicesConfig.KEY) private readonly config: ConfigType) {} + + /** Places matching the query, best first, or null when Nominatim has to answer. */ + search(query: string, opts: PhotonSearchOptions = {}): Promise { + const params = new URLSearchParams({ q: query, limit: String(SEARCH_LIMIT), lang: photonLang(opts.lang) }); + if (opts.bias && Number.isFinite(opts.bias.lat) && Number.isFinite(opts.bias.lng)) { + params.set('lat', String(opts.bias.lat)); + params.set('lon', String(opts.bias.lng)); + params.set('zoom', String(BIAS_ZOOM)); + } + return this.ask('api', params); + } + + /** What lies at a point, nearest first, or null when Nominatim has to answer. */ + reverse(lat: number, lng: number, lang?: string): Promise { + if (!Number.isFinite(lat) || !Number.isFinite(lng)) return Promise.resolve(null); + const params = new URLSearchParams({ lat: String(lat), lon: String(lng), limit: '1', lang: photonLang(lang) }); + return this.ask('reverse', params); + } + + private async ask(path: 'api' | 'reverse', params: URLSearchParams): Promise { + if (!this.mayAsk()) return null; + const url = new URL(`${this.config.photonUrl}/${path}`); + url.search = params.toString(); + const identity = serviceIdentitySlot.get(); + const headers = identity.headersFor(url); + if (!headers.Authorization) return null; + try { + const res = await fetch(url, { + headers: { Accept: 'application/json', ...headers }, + // The key goes to this host alone, never on to wherever it points. + redirect: 'error', + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + identity.observe(url, res); + if (!res.ok) { + await res.body?.cancel(); + throw new Error(`answered ${res.status}`); + } + const { features } = photonAnswerSchema.parse(await capResponse(res, MAX_BYTES).json()); + this.misses = 0; + return features; + } catch (err) { + this.noteMiss(err); + return null; + } + } + + /** Off with the services, beside an operator's own Nominatim, and during the pause. */ + private mayAsk(): boolean { + if (this.config.mode === 'off' || readEnv().integrations.nominatimOwn) return false; + return Date.now() >= this.pausedUntil; + } + + private noteMiss(err: unknown): void { + const reason = err instanceof Error ? err.message : String(err); + this.misses += 1; + if (this.misses < MISSES_BEFORE_PAUSE) { + logDebug(`[Photon] ${reason}; asking Nominatim instead`); + return; + } + this.pausedUntil = Date.now() + PAUSE_MS; + // One more miss after the pause starts the next one. + this.misses = MISSES_BEFORE_PAUSE - 1; + logWarn(`[Photon] ${reason}; asking Nominatim alone for a minute`); + } +} diff --git a/server/src/nest/maps/providers/photon.mapping.ts b/server/src/nest/maps/providers/photon.mapping.ts new file mode 100644 index 000000000..c32c2296d --- /dev/null +++ b/server/src/nest/maps/providers/photon.mapping.ts @@ -0,0 +1,160 @@ +/** + * Photon's answers in the shape Nominatim gives, so search and reverse read + * the same to every caller whichever of the two answered. + * + * Photon is a GeoJSON service: one feature per hit, the coordinate as + * [lon, lat], the OSM type as a letter and the address as flat properties. + * Nominatim names the type in full, spells the coordinate as text and carries + * the address as one comma-joined line. The OsmClient keeps one mapping from + * Nominatim's shape to TREK's place record, and this file feeds it. + */ +import { toApiLang } from '../maps.helpers'; +import type { NominatimResult, NominatimReverse } from './osm.client'; + +import { z } from 'zod'; + +/** The languages the TREK services imported names for; anything else gets the names used on the spot. */ +const PHOTON_LANGS: ReadonlySet = new Set([ + 'ar', + 'az', + 'ca', + 'cs', + 'de', + 'el', + 'en', + 'es', + 'et', + 'fr', + 'hu', + 'id', + 'it', + 'ja', + 'ko', + 'nl', + 'pl', + 'pt', + 'ru', + 'sk', + 'sv', + 'th', + 'tr', + 'uk', + 'vi', + 'zh', +]); + +const OSM_TYPES: Readonly> = { N: 'node', W: 'way', R: 'relation' }; + +const text = z.string().optional(); + +const photonFeatureSchema = z.object({ + geometry: z.object({ coordinates: z.array(z.number()).min(2) }), + properties: z.object({ + osm_type: text, + osm_id: z.number().int().optional(), + osm_key: text, + osm_value: text, + type: text, + name: text, + housenumber: text, + street: text, + postcode: text, + locality: text, + district: text, + city: text, + county: text, + state: text, + country: text, + countrycode: text, + extra: z.record(z.string(), z.unknown()).optional(), + }), +}); + +export const photonAnswerSchema = z.object({ features: z.array(photonFeatureSchema) }); + +export type PhotonFeature = z.infer; +type PhotonProperties = PhotonFeature['properties']; + +/** + * The language Photon is asked in: TREK's code through toApiLang (`br` is + * Brazilian Portuguese, `gr` Greek), cut to its base (`pt-BR` to `pt`, `zh-TW` + * to `zh`). A language the import has no names for, or none at all, asks for + * `default`, the names written on the spot. + */ +export function photonLang(lang: string | undefined): string { + if (!lang?.trim()) return 'default'; + const base = toApiLang(lang).split('-')[0].toLowerCase(); + return PHOTON_LANGS.has(base) ? base : 'default'; +} + +/** + * The address line Nominatim calls `display_name`: the place's own name first, + * then street, quarter, town, county, region, postcode and country. A part + * that repeats one before it is left out, so Berlin reads "Berlin, Germany" + * rather than naming the city state twice. + */ +export function photonDisplayName(p: PhotonProperties): string { + const street = [p.street, p.housenumber].filter(Boolean).join(' '); + const parts = [p.name, street, p.district ?? p.locality, p.city, p.county, p.state, p.postcode, p.country]; + const seen = new Set(); + for (const part of parts) { + const value = part?.trim(); + if (value) seen.add(value); + } + return [...seen].join(', '); +} + +/** A search hit as Nominatim would have answered it, or null for one without an OpenStreetMap id. */ +export function asNominatimHit(feature: PhotonFeature): NominatimResult | null { + const p = feature.properties; + const osmType = p.osm_type ? OSM_TYPES[p.osm_type] : undefined; + if (!osmType || p.osm_id === undefined) return null; + const [lon, lat] = feature.geometry.coordinates; + return { + osm_type: osmType, + osm_id: String(p.osm_id), + class: p.osm_key, + type: p.osm_value, + name: p.name, + display_name: photonDisplayName(p), + lat: String(lat), + lon: String(lon), + extratags: stringTags(p.extra), + }; +} + +/** + * The point a reverse lookup found, in the fields Nominatim's reverse answer + * has. A town itself is its own `city`, and a named place is filed under its + * main tag the way Nominatim files it (`tourism` for the Eiffel Tower), so the + * readers that look for a sight's name before the road find it here too. + */ +export function asNominatimReverse(feature: PhotonFeature): NominatimReverse { + const p = feature.properties; + const address: Record = {}; + const put = (key: string, value: string | undefined): void => { + if (value?.trim() && !(key in address)) address[key] = value.trim(); + }; + put('road', p.street); + put('house_number', p.housenumber); + put('postcode', p.postcode); + put('suburb', p.district ?? p.locality); + put('city', p.type === 'city' ? p.name : p.city); + put('county', p.county); + put('state', p.state); + put('country', p.country); + put('country_code', p.countrycode?.toLowerCase()); + if (p.osm_key && p.type !== 'city') put(p.osm_key, p.name); + const displayName = photonDisplayName(p); + return { display_name: displayName || undefined, name: p.name, address }; +} + +/** The extra tags Photon passes through, the text ones only. */ +function stringTags(extra: Record | undefined): Record | null { + if (!extra) return null; + const tags: Record = {}; + for (const [key, value] of Object.entries(extra)) { + if (typeof value === 'string') tags[key] = value; + } + return tags; +} diff --git a/server/tests/e2e/maps-photon.e2e.test.ts b/server/tests/e2e/maps-photon.e2e.test.ts new file mode 100644 index 000000000..87d4cb9c1 --- /dev/null +++ b/server/tests/e2e/maps-photon.e2e.test.ts @@ -0,0 +1,182 @@ +/** + * /api/maps search and reverse e2e with Photon in front of Nominatim: the real + * JwtAuthGuard, Zod pipe and response contracts over MapsModule on a migrated + * temp database, the TREK Places index switched off so the OpenStreetMap half + * answers alone. + * + * Replaced: the global fetch, routed by host to a fake Photon and a fake + * Nominatim, and the installation identity, a fake holding a key or none. + */ +import { servicesConfig } from '../../src/nest/app-config/tokens'; +import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; +import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { MapsModule } from '../../src/nest/maps/maps.module'; +import { serviceIdentitySlot } from '../../src/nest/service-identity/service-identity.interface'; +import { createSnapshotTestDb } from '../helpers/db-mock'; +import { makeUser } from '../helpers/factories/users'; +import { FakeServiceIdentity, gateFeature, photonAnswer } from '../helpers/photon'; +import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm'; +import { TestUnitOfWorkModule } from '../helpers/test-uow'; +import { sessionCookie } from './harness'; +import { Test } from '@nestjs/testing'; + +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const db = createSnapshotTestDb(); +const USER = 1; +const SERVICES = { + mode: 'auto', + key: null, + hosts: ['places.liketrek.com', 'photon.liketrek.com', 'valhalla.liketrek.com'], + photonUrl: 'https://photon.liketrek.com', +}; + +const NOMINATIM_HIT = { + osm_type: 'node', + osm_id: '42', + class: 'amenity', + type: 'cafe', + name: 'Café Nominatim', + display_name: 'Café Nominatim, Pariser Platz, Berlin, Deutschland', + lat: '52.516', + lon: '13.378', +}; + +let orm: TestOrm; +let app: Awaited>; +let identity: FakeServiceIdentity; +let photonReply: () => Response; +const asked: URL[] = []; + +async function build() { + const moduleRef = await Test.createTestingModule({ + imports: [await TestUnitOfWorkModule.forRoot(db), await createTestMikroOrmModule(db), MapsModule], + }) + .overrideProvider(servicesConfig.KEY) + .useValue(SERVICES) + .compile(); + const nest = moduleRef.createNestApplication(); + nest.use(cookieParser()); + nest.useGlobalFilters(new TrekExceptionFilter()); + nest.useGlobalPipes(new ZodValidationPipe()); + await nest.init(); + return nest; +} + +const hostsAsked = () => asked.map((url) => url.hostname); +const as = (userId: number) => ({ Cookie: sessionCookie(userId) }); + +beforeAll(async () => { + orm = await createTestOrm(db); + await makeUser(orm, { id: USER, username: 'traveler', email: 'traveler@example.test' }); + app = await build(); +}); + +afterAll(async () => { + await app.close(); + await orm.close(); +}); + +beforeEach(() => { + identity = new FakeServiceIdentity(); + serviceIdentitySlot.install(identity); + photonReply = () => photonAnswer([gateFeature()]); + asked.length = 0; + vi.stubEnv('TREK_PLACES_ENABLED', 'false'); + vi.stubEnv('NOMINATIM_URL', ''); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => { + const url = new URL(String(input)); + asked.push(url); + if (url.hostname === 'photon.liketrek.com') return photonReply(); + const body = url.pathname.endsWith('/reverse') + ? { display_name: NOMINATIM_HIT.display_name, name: NOMINATIM_HIT.name, address: {} } + : [NOMINATIM_HIT]; + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }); + }), + ); +}); + +afterEach(() => { + serviceIdentitySlot.release(identity); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +describe('POST /api/maps/search', () => { + it('MAPS-PHOTON-E2E-001: answers from Photon with the key, in the reader language and around the bias', async () => { + const res = await request(app.getHttpServer()) + .post('/api/maps/search?lang=de') + .set(as(USER)) + .send({ query: 'Brandenburger Tor', locationBias: { lat: 52.5, lng: 13.4 } }); + expect(res.status).toBe(200); + expect(res.body.source).toBe('openstreetmap'); + expect(res.body.places[0]).toMatchObject({ + osm_id: 'way:518071791', + name: 'Brandenburger Tor', + address: 'Brandenburger Tor, Pariser Platz, Mitte, Berlin, 10117, Deutschland', + lat: 52.5163, + lng: 13.3777, + wikidata: 'Q82425', + }); + expect(hostsAsked()).toEqual(['photon.liketrek.com']); + expect(Object.fromEntries(asked[0].searchParams)).toMatchObject({ lang: 'de', lat: '52.5', lon: '13.4' }); + expect(identity.observed).toEqual([{ host: 'photon.liketrek.com', status: 200 }]); + }); + + it('MAPS-PHOTON-E2E-002: falls back to Nominatim when Photon refuses, and the answer reads the same', async () => { + photonReply = () => photonAnswer([], 429); + const res = await request(app.getHttpServer()).post('/api/maps/search').set(as(USER)).send({ query: 'Café' }); + expect(res.status).toBe(200); + expect(res.body.places[0]).toMatchObject({ osm_id: 'node:42', name: 'Café Nominatim', category: 'cafe' }); + expect(hostsAsked()).toEqual(['photon.liketrek.com', 'nominatim.openstreetmap.org']); + }); + + it('MAPS-PHOTON-E2E-003: never asks Photon without a key', async () => { + identity.bearer = null; + const res = await request(app.getHttpServer()).post('/api/maps/search').set(as(USER)).send({ query: 'Café' }); + expect(res.status).toBe(200); + expect(hostsAsked()).toEqual(['nominatim.openstreetmap.org']); + }); + + it("MAPS-PHOTON-E2E-004: keeps an operator's own Nominatim", async () => { + vi.stubEnv('NOMINATIM_URL', 'http://nominatim.internal:8080'); + const res = await request(app.getHttpServer()).post('/api/maps/search').set(as(USER)).send({ query: 'Café' }); + expect(res.status).toBe(200); + expect(hostsAsked()).toEqual(['nominatim.internal']); + }); +}); + +describe('GET /api/maps/reverse', () => { + it('MAPS-PHOTON-E2E-005: names the point from Photon', async () => { + const res = await request(app.getHttpServer()) + .get('/api/maps/reverse') + .set(as(USER)) + .query({ lat: '52.5163', lng: '13.3777', lang: 'de' }); + expect(res.status).toBe(200); + expect(res.body).toEqual({ + name: 'Brandenburger Tor', + address: 'Brandenburger Tor, Pariser Platz, Mitte, Berlin, 10117, Deutschland', + }); + expect(asked[0].pathname).toBe('/reverse'); + }); + + it('MAPS-PHOTON-E2E-006: asks Nominatim when Photon is down', async () => { + photonReply = () => photonAnswer([], 503); + const res = await request(app.getHttpServer()) + .get('/api/maps/reverse') + .set(as(USER)) + .query({ lat: '52.5163', lng: '13.3777' }); + expect(res.body).toEqual({ name: 'Café Nominatim', address: NOMINATIM_HIT.display_name }); + expect(hostsAsked()).toEqual(['photon.liketrek.com', 'nominatim.openstreetmap.org']); + }); + + it('MAPS-PHOTON-E2E-007: answers 401 without a session and asks nobody', async () => { + const res = await request(app.getHttpServer()).get('/api/maps/reverse').query({ lat: '52.5', lng: '13.4' }); + expect(res.status).toBe(401); + expect(asked).toEqual([]); + }); +}); diff --git a/server/tests/helpers/maps-service.ts b/server/tests/helpers/maps-service.ts index 470236715..499567ae4 100644 --- a/server/tests/helpers/maps-service.ts +++ b/server/tests/helpers/maps-service.ts @@ -14,6 +14,7 @@ import { OsmClient } from '../../src/nest/maps/providers/osm.client'; import { WikimediaClient } from '../../src/nest/maps/providers/wikimedia.client'; import type { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import { noGoogleQuota } from './google-quota'; +import { photonOff } from './photon'; /** MapsService and the outbound clients it was wired with, for tests that reach a client directly. */ export interface MapsParts { @@ -39,7 +40,7 @@ export function buildMapsParts( googleQuota: GoogleQuotaService = noGoogleQuota, ): MapsParts { const google = new GooglePlacesClient(googleQuota); - const osm = new OsmClient(); + const osm = new OsmClient(photonOff()); const wiki = new WikimediaClient(); const selector = new PlacesProviderSelector(appSettings, users, googleQuota); const photos = new PlacePhotoResolver(photoCache, places, google, wiki, selector); diff --git a/server/tests/helpers/photon.ts b/server/tests/helpers/photon.ts new file mode 100644 index 000000000..c4414ff19 --- /dev/null +++ b/server/tests/helpers/photon.ts @@ -0,0 +1,60 @@ +import { deriveServices } from '../../src/app-config/boot-derive'; +import { PhotonClient } from '../../src/nest/maps/providers/photon.client'; +import { ServiceIdentityPort } from '../../src/nest/service-identity/service-identity.interface'; + +/** Photon with the TREK services off, so an OsmClient built by hand asks Nominatim as it always did. */ +export function photonOff(): PhotonClient { + return new PhotonClient(deriveServices({ TREK_SERVICES: 'off' })); +} + +/** + * The installation identity as the service-identity module hands it out, with + * a key to send or none, and the status of every answer it was shown. + */ +export class FakeServiceIdentity extends ServiceIdentityPort { + bearer: string | null = 'tk1.kid.payload.signature'; + readonly observed: { host: string; status: number }[] = []; + + headersFor(): Record { + const headers: Record = { 'User-Agent': 'TREK/test' }; + if (this.bearer) headers.Authorization = `Bearer ${this.bearer}`; + return headers; + } + + observe(url: URL, response: Response): void { + this.observed.push({ host: url.hostname, status: response.status }); + } +} + +/** The Brandenburg Gate as Photon answers a search for it. */ +export function gateFeature(properties: Record = {}) { + return { + type: 'Feature', + geometry: { type: 'Point', coordinates: [13.3777, 52.5163] }, + properties: { + osm_type: 'W', + osm_id: 518071791, + osm_key: 'tourism', + osm_value: 'attraction', + type: 'house', + name: 'Brandenburger Tor', + street: 'Pariser Platz', + postcode: '10117', + district: 'Mitte', + city: 'Berlin', + state: 'Berlin', + country: 'Deutschland', + countrycode: 'DE', + extra: { wikidata: 'Q82425', wikipedia: 'de:Brandenburger Tor' }, + ...properties, + }, + }; +} + +/** A Photon answer over the given features, as the service sends it. */ +export function photonAnswer(features: unknown[], status = 200): Response { + return new Response(JSON.stringify({ type: 'FeatureCollection', features }), { + status, + headers: { 'content-type': 'application/json' }, + }); +} diff --git a/server/tests/unit/app-config/config-templates.test.ts b/server/tests/unit/app-config/config-templates.test.ts index 382a4bc69..9ae3b9626 100644 --- a/server/tests/unit/app-config/config-templates.test.ts +++ b/server/tests/unit/app-config/config-templates.test.ts @@ -163,6 +163,14 @@ describe('the TREK services switches in the templates', () => { expect(envExample).toContain('# TREK_SERVICES_HOSTS='); }); + it('every template names the Photon address and the default hosts it joined', () => { + expect(envExample).toContain('# TREK_PHOTON_URL=https://photon.liketrek.com '); + expect(compose).toContain('# - TREK_PHOTON_URL=https://photon.liketrek.com '); + expect(helmValues).toContain('# TREK_PHOTON_URL:'); + expect(helmConfigMap).toContain('TREK_PHOTON_URL: {{ .Values.env.TREK_PHOTON_URL | quote }}'); + expect(envExample).toContain('(default: places.liketrek.com,photon.liketrek.com,valhalla.liketrek.com)'); + }); + it('the Helm ConfigMap passes the switch and the hosts, the Secret carries the key', () => { for (const name of ['TREK_SERVICES', 'TREK_SERVICES_HOSTS']) { expect(helmValues).toContain(`# ${name}:`); diff --git a/server/tests/unit/app-config/derive.test.ts b/server/tests/unit/app-config/derive.test.ts index dbf298aa2..6b3492527 100644 --- a/server/tests/unit/app-config/derive.test.ts +++ b/server/tests/unit/app-config/derive.test.ts @@ -249,6 +249,14 @@ describe('deriveIntegrations', () => { expect(deriveIntegrations({ OVERPASS_TIMEOUT_MS: '-1' }).overpassTimeoutMs).toBe(OVERPASS_TIMEOUT_DEFAULT_MS); }); + it('says whether the operator set a Nominatim of their own, which keeps search off Photon', () => { + expect(deriveIntegrations({}).nominatimOwn).toBe(false); + expect(deriveIntegrations({ NOMINATIM_URL: ' ' }).nominatimOwn).toBe(false); + expect(deriveIntegrations({ NOMINATIM_URL: 'http://nominatim:8080' }).nominatimOwn).toBe(true); + // The public address spelled out is still a choice the operator made. + expect(deriveIntegrations({ NOMINATIM_URL: 'https://nominatim.openstreetmap.org' }).nominatimOwn).toBe(true); + }); + it('floors the LLM ceiling to a whole number — undici rejects a fractional headersTimeout', () => { expect(deriveIntegrations({}).llmTimeoutMs).toBe(900_000); expect(deriveIntegrations({ LLM_TIMEOUT_MS: '-1' }).llmTimeoutMs).toBe(900_000); @@ -298,13 +306,22 @@ describe('deriveServices (servicesConfig token)', () => { expect(deriveServices({}).key).toBeNull(); }); - it('sends the key to places.liketrek.com unless TREK_SERVICES_HOSTS names others', () => { - expect(deriveServices({}).hosts).toEqual(['places.liketrek.com']); + it('sends the key to the three liketrek services unless TREK_SERVICES_HOSTS names others', () => { + const all = ['places.liketrek.com', 'photon.liketrek.com', 'valhalla.liketrek.com']; + expect(deriveServices({}).hosts).toEqual(all); expect(deriveServices({ TREK_SERVICES_HOSTS: 'Places.LikeTrek.com, , photon.liketrek.com' }).hosts).toEqual([ 'places.liketrek.com', 'photon.liketrek.com', ]); - expect(deriveServices({ TREK_SERVICES_HOSTS: ' , ' }).hosts).toEqual(['places.liketrek.com']); + expect(deriveServices({ TREK_SERVICES_HOSTS: ' , ' }).hosts).toEqual(all); + }); + + it('asks photon.liketrek.com unless TREK_PHOTON_URL names another copy, trimmed and without trailing slashes', () => { + expect(deriveServices({}).photonUrl).toBe('https://photon.liketrek.com'); + expect(deriveServices({ TREK_PHOTON_URL: ' ' }).photonUrl).toBe('https://photon.liketrek.com'); + expect(deriveServices({ TREK_PHOTON_URL: ' https://geo.example.org/photon// ' }).photonUrl).toBe( + 'https://geo.example.org/photon', + ); }); }); diff --git a/server/tests/unit/nest/maps.identity.test.ts b/server/tests/unit/nest/maps.identity.test.ts index e4df6c95d..8a9f6738d 100644 --- a/server/tests/unit/nest/maps.identity.test.ts +++ b/server/tests/unit/nest/maps.identity.test.ts @@ -8,6 +8,7 @@ import { toWikiLang, haversineMetres, namesOverlap } from '../../../src/nest/maps/maps.helpers'; import { OsmClient } from '../../../src/nest/maps/providers/osm.client'; import { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; +import { photonOff } from '../../helpers/photon'; import { describe, it, expect, vi, afterEach } from 'vitest'; @@ -22,7 +23,7 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({ })); const wikiOf = () => new WikimediaClient(); -const osmOf = () => new OsmClient(); +const osmOf = () => new OsmClient(photonOff()); // The Brandenburg Gate and the underground station named after it, 250m apart. const GATE = { lat: 52.5163, lng: 13.3777 }; diff --git a/server/tests/unit/nest/maps.overpass-names.test.ts b/server/tests/unit/nest/maps.overpass-names.test.ts index 059816f09..a7054b462 100644 --- a/server/tests/unit/nest/maps.overpass-names.test.ts +++ b/server/tests/unit/nest/maps.overpass-names.test.ts @@ -6,10 +6,11 @@ * had there. */ import { OsmClient } from '../../../src/nest/maps/providers/osm.client'; +import { photonOff } from '../../helpers/photon'; import { describe, it, expect, vi, afterEach } from 'vitest'; -const osm = new OsmClient(); +const osm = new OsmClient(photonOff()); afterEach(() => { vi.unstubAllGlobals(); diff --git a/server/tests/unit/nest/osm.photon.test.ts b/server/tests/unit/nest/osm.photon.test.ts new file mode 100644 index 000000000..8162e156e --- /dev/null +++ b/server/tests/unit/nest/osm.photon.test.ts @@ -0,0 +1,150 @@ +/** + * OSM-PHOTON-001..009: the OpenStreetMap search and reverse lookup ask Photon + * first where the installation holds a key, answer in the shape Nominatim's + * answers always had, and fall back to Nominatim whenever Photon gives none. + * Lookups by id stay on Nominatim. The network is a stubbed global fetch + * routed by host. + */ +import { deriveServices } from '../../../src/app-config/boot-derive'; +import { OsmClient } from '../../../src/nest/maps/providers/osm.client'; +import { PhotonClient } from '../../../src/nest/maps/providers/photon.client'; +import { serviceIdentitySlot } from '../../../src/nest/service-identity/service-identity.interface'; +import { FakeServiceIdentity, gateFeature, photonAnswer } from '../../helpers/photon'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const NOMINATIM_GATE = { + osm_type: 'way', + osm_id: '518071791', + class: 'tourism', + type: 'attraction', + name: 'Brandenburger Tor', + display_name: 'Brandenburger Tor, Pariser Platz, Mitte, Berlin, 10117, Deutschland', + lat: '52.5163', + lon: '13.3777', + extratags: { wikidata: 'Q82425', wikipedia: 'de:Brandenburger Tor' }, +}; + +const NOMINATIM_REVERSE = { + display_name: 'Unter den Linden 77, Mitte, Berlin, Deutschland', + name: '', + address: { road: 'Unter den Linden', city: 'Berlin' }, +}; + +let identity: FakeServiceIdentity; +let photonReply: () => Response; +let osm: OsmClient; +const hosts: string[] = []; + +beforeEach(() => { + identity = new FakeServiceIdentity(); + serviceIdentitySlot.install(identity); + photonReply = () => photonAnswer([gateFeature()]); + hosts.length = 0; + vi.stubEnv('NOMINATIM_URL', ''); + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => { + const url = new URL(String(input)); + hosts.push(url.hostname); + if (url.hostname === 'photon.liketrek.com') return photonReply(); + const body = url.pathname.endsWith('/reverse') ? NOMINATIM_REVERSE : [NOMINATIM_GATE]; + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }); + }), + ); + osm = new OsmClient(new PhotonClient(deriveServices({}))); +}); + +afterEach(() => { + serviceIdentitySlot.release(identity); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); +}); + +describe('search', () => { + it('OSM-PHOTON-001: answers from Photon in the record a Nominatim hit gives, and leaves Nominatim alone', async () => { + const fromPhoton = await osm.searchNominatim('Brandenburger Tor', 'de'); + expect(hosts).toEqual(['photon.liketrek.com']); + + identity.bearer = null; + const fromNominatim = await osm.searchNominatim('Brandenburger Tor', 'de'); + expect(hosts).toEqual(['photon.liketrek.com', 'nominatim.openstreetmap.org']); + + expect(fromPhoton).toEqual(fromNominatim); + expect(fromPhoton[0]).toMatchObject({ + osm_id: 'way:518071791', + name: 'Brandenburger Tor', + lat: 52.5163, + lng: 13.3777, + category: 'attraction', + source: 'openstreetmap', + wikidata: 'Q82425', + }); + }); + + it('OSM-PHOTON-002: asks Nominatim when Photon fails, with the bias it would have had', async () => { + photonReply = () => photonAnswer([], 503); + const places = await osm.searchNominatim('Brandenburger Tor', 'de', 'interactive', { lat: 52.5, lng: 13.4 }); + expect(hosts).toEqual(['photon.liketrek.com', 'nominatim.openstreetmap.org']); + expect(places[0]).toMatchObject({ osm_id: 'way:518071791' }); + }); + + it('OSM-PHOTON-003: leaves out a Photon hit without an OpenStreetMap id', async () => { + photonReply = () => photonAnswer([gateFeature({ osm_type: undefined }), gateFeature({ osm_id: 7, osm_type: 'N' })]); + const places = await osm.searchNominatim('Brandenburger Tor'); + expect(places.map((p) => p.osm_id)).toEqual(['node:7']); + }); + + it('OSM-PHOTON-004: the suggestions of a keyless install come out of Photon too', async () => { + const { suggestions } = await osm.autocompleteNominatim('Brandenb', 'de'); + expect(hosts).toEqual(['photon.liketrek.com']); + expect(suggestions).toEqual([ + { + placeId: 'way:518071791', + mainText: 'Brandenburger Tor', + secondaryText: 'Pariser Platz, Mitte, Berlin, 10117, Deutschland', + }, + ]); + }); +}); + +describe('reverse', () => { + it('OSM-PHOTON-005: names a point from Photon, the sight before its street', async () => { + expect(await osm.reverse('52.5163', '13.3777', 'de')).toEqual({ + name: 'Brandenburger Tor', + address: 'Brandenburger Tor, Pariser Platz, Mitte, Berlin, 10117, Deutschland', + }); + expect(hosts).toEqual(['photon.liketrek.com']); + }); + + it('OSM-PHOTON-006: names the town for a locality', async () => { + expect((await osm.reverse('52.5163', '13.3777', 'de', { locality: true })).name).toBe('Berlin'); + }); + + it('OSM-PHOTON-007: asks Nominatim when Photon fails or finds nothing there', async () => { + photonReply = () => photonAnswer([], 429); + expect(await osm.reverse('52.5', '13.4', 'de')).toEqual({ + name: 'Unter den Linden', + address: 'Unter den Linden 77, Mitte, Berlin, Deutschland', + }); + photonReply = () => photonAnswer([]); + expect((await osm.reverse('52.5', '13.4', 'de')).name).toBe('Unter den Linden'); + expect(hosts).toEqual([ + 'photon.liketrek.com', + 'nominatim.openstreetmap.org', + 'photon.liketrek.com', + 'nominatim.openstreetmap.org', + ]); + }); + + it('OSM-PHOTON-008: the raw answer for a pasted link comes from Photon, else from Nominatim', async () => { + expect((await osm.reverseRaw(52.5163, 13.3777)).address?.tourism).toBe('Brandenburger Tor'); + photonReply = () => photonAnswer([], 500); + expect(await osm.reverseRaw(52.5, 13.4)).toEqual(NOMINATIM_REVERSE); + }); +}); + +it('OSM-PHOTON-009: a lookup by id stays on Nominatim', async () => { + await osm.lookupNominatim('way', '518071791', 'de'); + expect(hosts).toEqual(['nominatim.openstreetmap.org']); +}); diff --git a/server/tests/unit/nest/osm.pois-along.test.ts b/server/tests/unit/nest/osm.pois-along.test.ts index 33d68e53a..78c6181bc 100644 --- a/server/tests/unit/nest/osm.pois-along.test.ts +++ b/server/tests/unit/nest/osm.pois-along.test.ts @@ -16,6 +16,7 @@ import { OsmClient, OverpassCoolingDown, } from '../../../src/nest/maps/providers/osm.client'; +import { photonOff } from '../../helpers/photon'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -61,7 +62,7 @@ const queryOf = (call: unknown[] | undefined) => decodeURIComponent(String((call?.[1] as RequestInit | undefined)?.body ?? '')).replace(/^data=/, ''); beforeEach(() => { - osm = new OsmClient(); + osm = new OsmClient(photonOff()); logWarn.mockClear(); vi.spyOn(console, 'error').mockImplementation(() => {}); }); diff --git a/server/tests/unit/nest/photon.client.test.ts b/server/tests/unit/nest/photon.client.test.ts new file mode 100644 index 000000000..593e1856e --- /dev/null +++ b/server/tests/unit/nest/photon.client.test.ts @@ -0,0 +1,164 @@ +/** + * PHOTON-001..014: when Photon is asked, what it is asked, and every way it + * hands the question back to Nominatim. The network is a stubbed global fetch, + * the installation identity a fake in the slot the service-identity module fills. + */ +import { deriveServices } from '../../../src/app-config/boot-derive'; +import { PhotonClient } from '../../../src/nest/maps/providers/photon.client'; +import { serviceIdentitySlot } from '../../../src/nest/service-identity/service-identity.interface'; +import { FakeServiceIdentity, gateFeature, photonAnswer } from '../../helpers/photon'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +let identity: FakeServiceIdentity; +let fetchMock: ReturnType; + +const client = (env: Record = {}) => new PhotonClient(deriveServices(env)); + +/** The URL and headers of the n-th request that went out. */ +function request(n = 0): { url: URL; headers: Record } { + const [url, init] = fetchMock.mock.calls[n] as [URL, RequestInit & { headers: Record }]; + return { url: new URL(String(url)), headers: init.headers }; +} + +beforeEach(() => { + identity = new FakeServiceIdentity(); + serviceIdentitySlot.install(identity); + fetchMock = vi.fn(async () => photonAnswer([gateFeature()])); + vi.stubGlobal('fetch', fetchMock); + vi.stubEnv('NOMINATIM_URL', ''); +}); + +afterEach(() => { + serviceIdentitySlot.release(identity); + vi.unstubAllGlobals(); + vi.unstubAllEnvs(); + vi.useRealTimers(); +}); + +describe('which installations ask Photon', () => { + it('PHOTON-001: one with a key for the TREK services, at photon.liketrek.com with the key', async () => { + const found = await client().search('Brandenburger Tor', { lang: 'de' }); + expect(found).toHaveLength(1); + const { url, headers } = request(); + expect(`${url.origin}${url.pathname}`).toBe('https://photon.liketrek.com/api'); + expect(headers.Authorization).toBe('Bearer tk1.kid.payload.signature'); + expect(headers['User-Agent']).toBe('TREK/test'); + expect(identity.observed).toEqual([{ host: 'photon.liketrek.com', status: 200 }]); + }); + + it('PHOTON-002: none without a key, since Photon has no shared pool', async () => { + identity.bearer = null; + expect(await client().search('Brandenburger Tor')).toBeNull(); + expect(await client().reverse(52.5, 13.4)).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('PHOTON-003: none with the TREK services off', async () => { + expect(await client({ TREK_SERVICES: 'off' }).search('Brandenburger Tor')).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('PHOTON-004: none beside a Nominatim the operator runs, which keeps answering', async () => { + vi.stubEnv('NOMINATIM_URL', 'http://nominatim:8080'); + expect(await client().search('Brandenburger Tor')).toBeNull(); + expect(await client().reverse(52.5, 13.4)).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('PHOTON-005: one pointed at another copy by TREK_PHOTON_URL', async () => { + await client({ TREK_PHOTON_URL: 'https://geo.example.org/photon/' }).search('Rostock'); + const { url } = request(); + expect(`${url.origin}${url.pathname}`).toBe('https://geo.example.org/photon/api'); + }); +}); + +describe('what Photon is asked', () => { + it('PHOTON-006: a search carries the query, the limit, the language and the bias', async () => { + await client().search('Hase-dera', { lang: 'br', bias: { lat: 35.31, lng: 139.53 } }); + const { searchParams } = request().url; + expect(Object.fromEntries(searchParams)).toEqual({ + q: 'Hase-dera', + limit: '10', + lang: 'pt', + lat: '35.31', + lon: '139.53', + zoom: '10', + }); + }); + + it('PHOTON-007: a search without a bias carries none, and one without a language the names on the spot', async () => { + await client().search('Rostock'); + const { searchParams } = request().url; + expect(searchParams.has('lat')).toBe(false); + expect(searchParams.get('lang')).toBe('default'); + }); + + it('PHOTON-008: a reverse lookup asks /reverse for the nearest feature', async () => { + await client().reverse(52.5163, 13.3777, 'gr'); + const { url } = request(); + expect(url.pathname).toBe('/reverse'); + expect(Object.fromEntries(url.searchParams)).toEqual({ lat: '52.5163', lon: '13.3777', limit: '1', lang: 'el' }); + }); + + it('PHOTON-009: a point that is not one is not sent', async () => { + expect(await client().reverse(Number.NaN, 13.4)).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +describe('when Nominatim has to answer instead', () => { + it.each([401, 403, 429, 500, 503])('PHOTON-010: an answer of %i', async (status) => { + fetchMock.mockResolvedValueOnce(photonAnswer([], status)); + expect(await client().search('Rostock')).toBeNull(); + expect(identity.observed).toEqual([{ host: 'photon.liketrek.com', status }]); + }); + + it('PHOTON-011: no answer at all, or none in time', async () => { + fetchMock.mockRejectedValueOnce(new TypeError('fetch failed')); + expect(await client().search('Rostock')).toBeNull(); + fetchMock.mockRejectedValueOnce(new DOMException('The operation was aborted due to timeout', 'TimeoutError')); + expect(await client().search('Rostock')).toBeNull(); + }); + + it("PHOTON-012: an answer that is not Photon's, or one far too long", async () => { + fetchMock.mockResolvedValueOnce(new Response('{"results":[]}', { status: 200 })); + expect(await client().search('Rostock')).toBeNull(); + fetchMock.mockResolvedValueOnce(new Response('', { status: 200 })); + expect(await client().search('Rostock')).toBeNull(); + fetchMock.mockResolvedValueOnce(photonAnswer([gateFeature({ name: 'x'.repeat(600_000) })])); + expect(await client().search('Rostock')).toBeNull(); + }); + + it('PHOTON-013: an empty answer is an answer, not a miss', async () => { + fetchMock.mockResolvedValueOnce(photonAnswer([])); + expect(await client().search('Qwxzv')).toEqual([]); + }); +}); + +describe('the pause after misses in a row', () => { + it('PHOTON-014: four misses stop the asking for a minute, then one try goes out, and an answer resets it', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-29T10:00:00Z')); + const photon = client(); + fetchMock.mockImplementation(async () => photonAnswer([], 503)); + for (let i = 0; i < 4; i++) expect(await photon.search('Rostock')).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(4); + + expect(await photon.search('Rostock')).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(4); + + vi.setSystemTime(new Date('2026-10-29T10:01:01Z')); + expect(await photon.search('Rostock')).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(5); + expect(await photon.search('Rostock')).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(5); + + vi.setSystemTime(new Date('2026-10-29T10:02:02Z')); + fetchMock.mockImplementation(async () => photonAnswer([gateFeature()])); + expect(await photon.search('Rostock')).toHaveLength(1); + fetchMock.mockImplementation(async () => photonAnswer([], 503)); + for (let i = 0; i < 3; i++) expect(await photon.search('Rostock')).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(9); + }); +}); diff --git a/server/tests/unit/nest/photon.mapping.test.ts b/server/tests/unit/nest/photon.mapping.test.ts new file mode 100644 index 000000000..dd44060da --- /dev/null +++ b/server/tests/unit/nest/photon.mapping.test.ts @@ -0,0 +1,118 @@ +/** + * PHOTON-MAP-001..012: Photon's answers read into the shape Nominatim gives, + * and the language Photon is asked in. + */ +import { + asNominatimHit, + asNominatimReverse, + photonAnswerSchema, + photonDisplayName, + photonLang, + type PhotonFeature, +} from '../../../src/nest/maps/providers/photon.mapping'; +import { gateFeature } from '../../helpers/photon'; + +import { describe, expect, it } from 'vitest'; + +const feature = (properties: Record = {}): PhotonFeature => + photonAnswerSchema.parse({ features: [gateFeature(properties)] }).features[0]; + +describe('photonLang', () => { + it('PHOTON-MAP-001: asks in the languages the import holds names for', () => { + expect(photonLang('de')).toBe('de'); + expect(photonLang('en')).toBe('en'); + expect(photonLang('ja')).toBe('ja'); + }); + + it("PHOTON-MAP-002: reads TREK's own codes through toApiLang first", () => { + expect(photonLang('br')).toBe('pt'); + expect(photonLang('gr')).toBe('el'); + }); + + it('PHOTON-MAP-003: drops the region, whatever its case', () => { + expect(photonLang('pt-BR')).toBe('pt'); + expect(photonLang('zh-TW')).toBe('zh'); + expect(photonLang('EN-gb')).toBe('en'); + }); + + it('PHOTON-MAP-004: asks for the names on the spot in a language without names, and without one', () => { + expect(photonLang('eo')).toBe('default'); + expect(photonLang('xx-YY')).toBe('default'); + expect(photonLang(undefined)).toBe('default'); + expect(photonLang(' ')).toBe('default'); + }); +}); + +describe('photonDisplayName', () => { + it('PHOTON-MAP-005: names the place, then the street, quarter, town, region, postcode and country', () => { + expect(photonDisplayName(feature({ housenumber: '1' }).properties)).toBe( + 'Brandenburger Tor, Pariser Platz 1, Mitte, Berlin, 10117, Deutschland', + ); + }); + + it('PHOTON-MAP-006: leaves out what is missing and what repeats', () => { + const city = feature({ name: 'Berlin', street: undefined, district: undefined, postcode: undefined, type: 'city' }); + expect(photonDisplayName(city.properties)).toBe('Berlin, Deutschland'); + }); +}); + +describe('asNominatimHit', () => { + it('PHOTON-MAP-007: spells the hit the way a Nominatim search row is spelt', () => { + expect(asNominatimHit(feature())).toEqual({ + osm_type: 'way', + osm_id: '518071791', + class: 'tourism', + type: 'attraction', + name: 'Brandenburger Tor', + display_name: 'Brandenburger Tor, Pariser Platz, Mitte, Berlin, 10117, Deutschland', + lat: '52.5163', + lon: '13.3777', + extratags: { wikidata: 'Q82425', wikipedia: 'de:Brandenburger Tor' }, + }); + }); + + it('PHOTON-MAP-008: names nodes and relations in full, and keeps only text tags', () => { + expect(asNominatimHit(feature({ osm_type: 'N' }))?.osm_type).toBe('node'); + expect(asNominatimHit(feature({ osm_type: 'R' }))?.osm_type).toBe('relation'); + expect(asNominatimHit(feature({ extra: { wikidata: 'Q1', capacity: 12 } }))?.extratags).toEqual({ wikidata: 'Q1' }); + expect(asNominatimHit(feature({ extra: undefined }))?.extratags).toBeNull(); + }); + + it('PHOTON-MAP-009: has nothing for a hit without an OpenStreetMap id', () => { + expect(asNominatimHit(feature({ osm_type: undefined }))).toBeNull(); + expect(asNominatimHit(feature({ osm_type: 'X' }))).toBeNull(); + expect(asNominatimHit(feature({ osm_id: undefined }))).toBeNull(); + }); +}); + +describe('asNominatimReverse', () => { + it('PHOTON-MAP-010: fills the address fields Nominatim fills, the sight under its main tag', () => { + expect(asNominatimReverse(feature({ housenumber: '1' }))).toEqual({ + display_name: 'Brandenburger Tor, Pariser Platz 1, Mitte, Berlin, 10117, Deutschland', + name: 'Brandenburger Tor', + address: { + road: 'Pariser Platz', + house_number: '1', + postcode: '10117', + suburb: 'Mitte', + city: 'Berlin', + state: 'Berlin', + country: 'Deutschland', + country_code: 'de', + tourism: 'Brandenburger Tor', + }, + }); + }); + + it('PHOTON-MAP-011: makes a town its own city and files nothing more under its tag', () => { + const town = feature({ name: 'Warnemünde', type: 'city', osm_key: 'place', city: undefined, street: undefined }); + const reverse = asNominatimReverse(town); + expect(reverse.address?.city).toBe('Warnemünde'); + expect(reverse.address).not.toHaveProperty('place'); + }); + + it('PHOTON-MAP-012: a feature with nothing to name has no display name', () => { + const bare = photonAnswerSchema.parse({ features: [{ geometry: { coordinates: [0, 0] }, properties: {} }] }); + expect(asNominatimReverse(bare.features[0])).toEqual({ display_name: undefined, name: undefined, address: {} }); + }); +});