From 0c8788ce44c4867e17f5036629371555c0da7ea0 Mon Sep 17 00:00:00 2001 From: Maurice Date: Mon, 10 Aug 2026 09:53:08 +0200 Subject: [PATCH] Give a Google place its free half back --- server/src/nest/maps/maps.helpers.ts | 67 +++++ server/src/nest/maps/maps.service.ts | 214 ++++++++++++-- .../place-enrichment.service.ts | 128 +++++++-- server/tests/e2e/place-enrichment.e2e.test.ts | 9 + server/tests/unit/nest/maps.identity.test.ts | 213 ++++++++++++++ .../nest/place-enrichment.description.test.ts | 270 ++++++++++++++++++ .../nest/place-enrichment.service.test.ts | 16 +- 7 files changed, 872 insertions(+), 45 deletions(-) create mode 100644 server/tests/unit/nest/maps.identity.test.ts create mode 100644 server/tests/unit/nest/place-enrichment.description.test.ts diff --git a/server/src/nest/maps/maps.helpers.ts b/server/src/nest/maps/maps.helpers.ts index b9ea5a1fa..2cdd394fb 100644 --- a/server/src/nest/maps/maps.helpers.ts +++ b/server/src/nest/maps/maps.helpers.ts @@ -40,6 +40,73 @@ export function toApiLang(lang: string | undefined, fallback = 'en'): string { return API_LANG_OVERRIDES[code] ?? code; } +/** + * A wiki subdomain out of a TREK language code. + * + * Not the same normalisation as `toApiLang`, and the difference bites: TREK + * calls Brazilian Portuguese `br`, but `br.wikipedia.org` exists and is the + * BRETON Wikipedia. Passing the raw code through would not fail — it would + * quietly return Breton articles. `pt-br.wikipedia.org` does not resolve at + * all, so the region subtag has to go as well. + */ +const WIKI_LANG_OVERRIDES: Record = { + br: 'pt', + gr: 'el', +}; +export function toWikiLang(lang: string | undefined, fallback = 'en'): string { + const raw = (lang || '').trim(); + if (!raw) return fallback; + const mapped = WIKI_LANG_OVERRIDES[raw] ?? raw; + const base = mapped.split('-')[0].toLowerCase(); + return /^[a-z]{2,3}$/.test(base) ? base : fallback; +} + +/** Metres between two coordinates. Spherical earth is plenty at this scale. */ +export function haversineMetres(aLat: number, aLng: number, bLat: number, bLng: number): number { + const toRad = (deg: number) => (deg * Math.PI) / 180; + const dLat = toRad(bLat - aLat); + const dLng = toRad(bLng - aLng); + const h = + Math.sin(dLat / 2) ** 2 + Math.cos(toRad(aLat)) * Math.cos(toRad(bLat)) * Math.sin(dLng / 2) ** 2; + return 2 * 6371000 * Math.asin(Math.min(1, Math.sqrt(h))); +} + +/** + * Whether two place names plausibly refer to the same thing. + * + * The gate that keeps a coordinate-based lookup from confidently describing the + * wrong building. "Hamburg Airport" and "Flughafen Hamburg" pass on "hamburg"; + * "Hamburg Airport" and "Bahnhof Ohlsdorf" do not. + * + * One shared word is enough only when it carries some weight — four letters or + * more. Otherwise two have to match. A single short word is almost always an + * article or a generic noun, and TREK speaks 23 languages, so "Der Kiosk" and + * "Der Bahnhof" would sail through a plain word-overlap test while a stopword + * list for all of them is its own maintenance problem. Anything the pair rule + * lets through has already survived the distance check. + */ +export function namesOverlap(a: string, b: string): boolean { + const words = (value: string): Set => + new Set( + value + .toLowerCase() + .normalize('NFD') + .replace(/[̀-ͯ]/g, '') + .split(/[^a-z0-9]+/) + .filter((word) => word.length > 2), + ); + const left = words(a); + if (left.size === 0) return false; + + let shared = 0; + for (const word of words(b)) { + if (!left.has(word)) continue; + if (word.length >= 4) return true; + if (++shared >= 2) return true; + } + return false; +} + const GOOGLE_FTID_RE = /^0x[0-9a-f]+:0x[0-9a-f]+$/i; // Extracts a Google Maps feature id (ftid, 0x..:0x..) from a URL's ?ftid= param. diff --git a/server/src/nest/maps/maps.service.ts b/server/src/nest/maps/maps.service.ts index 2aa581357..36a1a225d 100644 --- a/server/src/nest/maps/maps.service.ts +++ b/server/src/nest/maps/maps.service.ts @@ -27,6 +27,9 @@ import { resolveOverpassTimeoutMs, stripWikiMarkup, parseWikipediaTag, + toWikiLang, + haversineMetres, + namesOverlap, type GoogleOpeningHours, type OverpassPoi, } from './maps.helpers'; @@ -275,6 +278,12 @@ interface GooglePlaceDetails extends GooglePlaceResult { // a slow provider is dropped rather than waited out. const WIKI_TIMEOUT_MS = 6000; +// Tighter than the wiki calls, because this one sits at the FRONT of a chain: +// identity, then sitelinks, then the extract. Nominatim answers a bounded +// search in 0.2-0.7s in practice, so anything past a couple of seconds is a bad +// day at the provider rather than a slow answer worth waiting for. +const IDENTITY_TIMEOUT_MS = 2500; + const MAX_CONCURRENT_PHOTO_FETCHES = 5; let photoFetchActive = 0; const photoFetchQueue: Array<() => void> = []; @@ -552,6 +561,91 @@ export class MapsService { }); } + /** + * Finds the OpenStreetMap record for a place we only know by name and + * coordinate, and hands back its tags. + * + * This is what gives a Google place a free identity. Google's payload has no + * `wikidata`, no `wikipedia` and no `wikimedia_commons` — it never had — so + * without this the entire free half of the enrichment column is unreachable + * for anyone who configured a Google key, which is the opposite of how this + * feature is meant to work. OSM knows these places perfectly well; nobody was + * asking it. + * + * Two gates keep it from describing the wrong building, because a confident + * description of somewhere else is worse than none: + * - the match has to be within `maxDistanceM` of where we are looking, and + * - it has to share a substantial word with the name we are looking for. + * Among what survives, Nominatim's own `importance` decides — that is what + * separates the Brandenburg Gate from the underground station named after it. + */ + async resolveOsmIdentity( + name: string, + lat: number, + lng: number, + opts: { lang?: string; maxDistanceM?: number; signal?: AbortSignal } = {}, + ): Promise<{ tags: Record; osmUrl: string | null; matchedName: string } | null> { + const query = (name || '').trim(); + if (!query || !Number.isFinite(lat) || !Number.isFinite(lng)) return null; + const maxDistanceM = opts.maxDistanceM ?? 2000; + + // ~2km box around the point, so Nominatim ranks locally instead of handing + // back the most famous place on earth with this name. + const d = 0.02; + const params = new URLSearchParams({ + q: query, + format: 'jsonv2', + extratags: '1', + limit: '5', + bounded: '1', + viewbox: `${lng - d},${lat - d},${lng + d},${lat + d}`, + 'accept-language': toApiLang(opts.lang), + }); + + try { + const res = await fetch(`https://nominatim.openstreetmap.org/search?${params}`, { + headers: { 'User-Agent': UA }, + signal: opts.signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS), + }); + if (!res.ok) return null; + // Nominatim answers rate limiting in plain text, not JSON. + const data = (await res.json()) as (NominatimResult & { importance?: number })[]; + if (!Array.isArray(data)) return null; + + const best = data + .map((item) => ({ + item, + lat: parseFloat(item.lat), + lng: parseFloat(item.lon), + })) + .filter(({ item, lat: hitLat, lng: hitLng }) => { + if (!Number.isFinite(hitLat) || !Number.isFinite(hitLng)) return false; + if (haversineMetres(lat, lng, hitLat, hitLng) > maxDistanceM) return false; + const label = item.name || item.display_name?.split(',')[0] || ''; + return namesOverlap(query, label); + }) + .sort((a, b) => { + const byImportance = (b.item.importance ?? 0) - (a.item.importance ?? 0); + if (byImportance !== 0) return byImportance; + return ( + haversineMetres(lat, lng, a.lat, a.lng) - haversineMetres(lat, lng, b.lat, b.lng) + ); + })[0]; + + if (!best) return null; + return { + tags: best.item.extratags ?? {}, + osmUrl: + best.item.osm_type && best.item.osm_id + ? `https://www.openstreetmap.org/${best.item.osm_type}/${best.item.osm_id}` + : null, + matchedName: best.item.name || best.item.display_name?.split(',')[0] || query, + }; + } catch { + return null; + } + } + // ── Nominatim lookup (by OSM ID) ─────────────────────────────────────────── async lookupNominatim( @@ -871,35 +965,105 @@ export class MapsService { }); for (const host of ['wikivoyage', 'wikipedia'] as const) { - try { - const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) continue; - const data = (await res.json()) as { - query?: { pages?: Record }; - }; - const pages = data.query?.pages; - if (!pages) continue; - for (const page of Object.values(pages)) { - const text = page.extract?.trim(); - // A missing article comes back as a page with no extract, not a 404. - if (!text) continue; - const title = page.title ?? parsed.title; - return { - text, - sourceUrl: `https://${parsed.lang}.${host}.org/wiki/${encodeURIComponent(title)}`, - source: host, - }; - } - } catch { - /* try the next wiki */ - } + const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title); + if (hit) return hit; } return null; } + /** + * The lead paragraph of one named article on one named wiki. + * + * Split out from `fetchWikiExtract` because the article is not always found + * through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag + * at all (Berlin Hauptbahnhof is exactly that), and then the title comes from + * the item's sitelinks instead. + */ + async fetchWikiExtractFor( + host: 'wikivoyage' | 'wikipedia', + lang: string, + title: string, + signal?: AbortSignal, + ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { + if (!lang || !title) return null; + // Two sentences, not three: this sits next to a form, and a fourth line of + // prose pushes the pictures out of view. + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: title, + prop: 'extracts', + exintro: '1', + explaintext: '1', + exsentences: '2', + redirects: '1', + }); + try { + const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) return null; + const data = (await res.json()) as { + query?: { pages?: Record }; + }; + for (const page of Object.values(data.query?.pages ?? {})) { + const text = page.extract?.trim(); + // A missing article comes back as a page with no extract, not a 404. + if (!text) continue; + const resolved = page.title ?? title; + return { + text, + sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`, + source: host, + }; + } + return null; + } catch { + return null; + } + } + + /** + * Which articles a Wikidata item is linked to, for the wikis we care about. + * + * The way to an article when a place has a Wikidata id but no `wikipedia` + * tag — which is most of them, because mappers add one or the other. One + * request, a few hundred bytes. + */ + async fetchWikidataSitelinks( + wikidataId: string, + sites: string[], + signal?: AbortSignal, + ): Promise> { + const qid = wikidataId.trim(); + if (!/^Q\d+$/.test(qid) || sites.length === 0) return {}; + const params = new URLSearchParams({ + action: 'wbgetentities', + props: 'sitelinks', + ids: qid, + sitefilter: sites.join('|'), + format: 'json', + }); + try { + const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS), + }); + if (!res.ok) return {}; + const data = (await res.json()) as { + entities?: Record }>; + }; + const out: Record = {}; + for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) { + if (link?.title) out[site] = link.title; + } + return out; + } catch { + return {}; + } + } + /** * The pictures Wikidata records for a place, best first. * diff --git a/server/src/nest/place-enrichment/place-enrichment.service.ts b/server/src/nest/place-enrichment/place-enrichment.service.ts index 1f04923dd..6f0160a0b 100644 --- a/server/src/nest/place-enrichment/place-enrichment.service.ts +++ b/server/src/nest/place-enrichment/place-enrichment.service.ts @@ -9,8 +9,8 @@ import type { } from '@trek/shared'; import { safeFetchFollow } from '../../utils/ssrfGuard'; import { DatabaseService } from '../database/database.service'; -import { MapsService, withPhotoFetchSlot, type CommonsCandidate } from '../maps/maps.service'; -import { isGooglePlaceId, rankCommonsCandidates } from '../maps/maps.helpers'; +import { MapsService, readWikiIdentity, withPhotoFetchSlot, type CommonsCandidate, type WikiIdentity } from '../maps/maps.service'; +import { isGooglePlaceId, parseWikipediaTag, rankCommonsCandidates, toWikiLang } from '../maps/maps.helpers'; import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service'; /** @@ -59,6 +59,15 @@ export function candidateKey(placeId: string, identity: string): string { return `${placeId}~p${digest}`; } +/** + * What the free sources need to know about a place, kept apart from whatever + * the maps provider returned. See `resolveIdentity` for why they do not merge. + */ +interface PlaceIdentity extends WikiIdentity { + /** Full OSM tag set, when the identity came from a Nominatim lookup. */ + osmTags: Record | null; +} + /** A ranked Commons pick, tagged with the rung of the ladder it came from. */ type CommonsPick = CommonsCandidate & { rung: 'wikidata' | 'wikipedia' | 'category' | 'nearby' }; @@ -204,10 +213,11 @@ export class PlaceEnrichmentService { // comes along with the request. Refetching cost 12.8 seconds on a large // OSM relation, and it ran in parallel with the client's own lookup. const details = req.details ?? (await this.readDetails(userId, placeId, lang)); + const identity = await this.resolveIdentity(req, details); const [photos, description] = await Promise.all([ - this.collectPhotos(userId, placeId, req, details), - this.collectDescription(userId, placeId, req, details), + this.collectPhotos(userId, placeId, req, identity), + this.collectDescription(userId, placeId, req, details, identity), ]); const result: CachedEnrichment = { photos, description, facts: collectFacts(details) }; @@ -226,20 +236,53 @@ export class PlaceEnrichmentService { return { credit: this.photoCache.get(key)?.attribution ?? null }; } + /** + * Which encyclopaedia entry, Wikidata item and Commons category describe this + * place. + * + * Kept beside the provider payload rather than merged into it, and that is + * not tidiness. `collectFacts` and the OSM branch of `collectDescription` + * both gate on `details.source`, so OSM tags folded into a Google payload + * would be carried around and never read — or, worse, would make a Google + * place start claiming to be an OpenStreetMap one. + * + * The lookup only runs when the payload brought nothing, which is every + * Google place: Google's response has no wiki tags and never had, so before + * this a configured API key silently switched the free half of this feature + * off. + */ + private async resolveIdentity( + req: MapsPlaceEnrichmentRequest, + details: Record | null, + ): Promise { + const fromPayload = (key: string): string | null => + typeof details?.[key] === 'string' && (details[key] as string).trim() ? (details[key] as string).trim() : null; + + const carried: PlaceIdentity = { + wikipedia: fromPayload('wikipedia'), + wikidata: fromPayload('wikidata'), + wikimedia_commons: fromPayload('wikimedia_commons'), + osmTags: null, + }; + if (carried.wikipedia || carried.wikidata || carried.wikimedia_commons) return carried; + + const resolved = await this.maps.resolveOsmIdentity(req.name, req.lat, req.lng, { lang: req.lang }); + if (!resolved) return carried; + return { ...readWikiIdentity(resolved.tags), osmTags: resolved.tags }; + } + // ── Photos ───────────────────────────────────────────────────────────────── private async collectPhotos( userId: number, placeId: string, req: MapsPlaceEnrichmentRequest, - details: Record | null, + identity: PlaceIdentity, ): Promise { const apiKey = this.maps.getMapsKey(userId); const wantsGoogle = !!apiKey && !this.maps.photosDisabled() && isGooglePlaceId(placeId); - const tag = (key: string): string | null => (typeof details?.[key] === 'string' ? (details[key] as string) : null); - const wikidata = tag('wikidata'); - const wikipedia = tag('wikipedia'); + const { wikidata, wikipedia } = identity; // Google's listing is one call for the whole strip and knows nothing about // the free sources, so it runs alongside them rather than in the ladder. @@ -258,7 +301,7 @@ export class PlaceEnrichmentService { // gate by the underground station beneath it. It is the fallback, and it // only runs when the ladder above came up short. const commonsPool: CommonsPick[] = []; - let categoryName = tag('wikimedia_commons'); + let categoryName = identity.wikimedia_commons; if (wikidata) { const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP); @@ -392,6 +435,7 @@ export class PlaceEnrichmentService { placeId: string, req: MapsPlaceEnrichmentRequest, details: Record | null, + identity: PlaceIdentity, ): Promise { // OpenStreetMap first: it costs nothing, it is already fetched, and a // description someone wrote into the map data beats a generated blurb. @@ -405,6 +449,16 @@ export class PlaceEnrichmentService { }; } + // Then the encyclopaedias. This used to sit BELOW Google, which read as a + // detail and was in fact the whole problem: an instance with a key asked + // Google, got nothing back for most places, and stopped — while a perfectly + // good Wikivoyage article sat one call away. Free sources are the normal + // case here and Google is the addition, so the order says so. + const extract = await this.fetchWikiDescription(identity, req.lang); + if (extract) { + return { text: extract.text, source: extract.source, sourceUrl: extract.sourceUrl, license: 'CC BY-SA 4.0' }; + } + const apiKey = this.maps.getMapsKey(userId); if (apiKey && !this.maps.detailsDisabled() && isGooglePlaceId(placeId)) { const summary = await this.maps.fetchEditorialSummary(placeId, apiKey, req.lang); @@ -418,16 +472,56 @@ export class PlaceEnrichmentService { } } - // Wikipedia last, and only when the place carries a wiki tag. Looking the - // article up by name lands on the wrong one for anything ambiguous, and a - // confident description of somewhere else is worse than none. - const wikipediaTag = typeof details?.wikipedia === 'string' ? details.wikipedia : null; - const extract = await this.maps.fetchWikiExtract(wikipediaTag); - if (extract) { - return { text: extract.text, source: extract.source, sourceUrl: extract.sourceUrl, license: 'CC BY-SA 4.0' }; + return null; + } + + /** + * The article about this place, in the reader's language where one exists. + * + * Two ways in, and a place usually has only one of them: mappers tag either + * `wikipedia` or `wikidata`, rarely both. The Wikidata route goes through + * sitelinks, which is also what makes the language choice honest — asking + * `.wikipedia.org` for a title taken from the German tag returns + * nothing, so without sitelinks a German article is all a Korean reader could + * ever get. + * + * Wikivoyage before Wikipedia at every step: it describes a place for someone + * about to go there, where Wikipedia opens with area in square kilometres. + */ + private async fetchWikiDescription( + identity: PlaceIdentity, + lang: string | undefined, + ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { + const userLang = toWikiLang(lang); + const tag = parseWikipediaTag(identity.wikipedia); + + if (identity.wikidata) { + // The tag's own language is in the list because the place named that + // article specifically; it beats falling through to English. + const wanted: { site: string; host: 'wikivoyage' | 'wikipedia'; lang: string }[] = [ + { site: `${userLang}wikivoyage`, host: 'wikivoyage', lang: userLang }, + { site: `${userLang}wiki`, host: 'wikipedia', lang: userLang }, + ...(tag && tag.lang !== userLang + ? [{ site: `${tag.lang}wiki`, host: 'wikipedia' as const, lang: tag.lang }] + : []), + { site: 'enwikivoyage', host: 'wikivoyage', lang: 'en' }, + { site: 'enwiki', host: 'wikipedia', lang: 'en' }, + ]; + const sitelinks = await this.maps.fetchWikidataSitelinks( + identity.wikidata, + wanted.map((w) => w.site), + ); + for (const { site, host, lang: hostLang } of wanted) { + const title = sitelinks[site]; + if (!title) continue; + const hit = await this.maps.fetchWikiExtractFor(host, hostLang, title); + if (hit) return hit; + } } - return null; + // No Wikidata id, or its sitelinks led nowhere: fall back to the tag, which + // names an article directly. + return identity.wikipedia ? this.maps.fetchWikiExtract(identity.wikipedia) : null; } /** diff --git a/server/tests/e2e/place-enrichment.e2e.test.ts b/server/tests/e2e/place-enrichment.e2e.test.ts index a0e75ff91..47253f547 100644 --- a/server/tests/e2e/place-enrichment.e2e.test.ts +++ b/server/tests/e2e/place-enrichment.e2e.test.ts @@ -86,6 +86,15 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => vi.spyOn(maps, 'fetchWikiExtract').mockClear().mockResolvedValue(null); vi.spyOn(maps, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]); vi.spyOn(maps, 'details').mockClear().mockResolvedValue({ place: null }); + // Every provider the service can reach has to be stubbed here, not just the + // ones a given case cares about: anything left open goes out over the real + // network from CI, which is both slow and rude to the provider. + vi.spyOn(maps, 'resolveOsmIdentity').mockClear().mockResolvedValue(null); + vi.spyOn(maps, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({}); + vi.spyOn(maps, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null); + vi.spyOn(maps, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null }); + vi.spyOn(maps, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null); + vi.spyOn(maps, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map()); }); it('401 without a session cookie', async () => { diff --git a/server/tests/unit/nest/maps.identity.test.ts b/server/tests/unit/nest/maps.identity.test.ts new file mode 100644 index 000000000..d7d3a68e3 --- /dev/null +++ b/server/tests/unit/nest/maps.identity.test.ts @@ -0,0 +1,213 @@ +/** + * MAPS-157..170 — resolving a place's identity from a name and a coordinate. + * + * This is what gives a Google place its free half back: Google's payload has no + * wiki tags, OpenStreetMap does. The two gates below are the whole safety + * argument — a confident description of the wrong building is worse than none. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +vi.mock('../../../src/db/database', () => ({ + db: { prepare: () => ({ get: () => undefined, run: () => undefined, all: () => [] }) }, +})); + +vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) })); + +vi.mock('../../../src/utils/ssrfGuard', () => ({ + safeFetchFollow: vi.fn(), + checkSsrf: vi.fn(async () => ({ allowed: true })), + SsrfBlockedError: class extends Error {}, +})); + +import { db } from '../../../src/db/database'; +import { DatabaseService } from '../../../src/nest/database/database.service'; +import { MapsService } from '../../../src/nest/maps/maps.service'; +import { toWikiLang, haversineMetres, namesOverlap } from '../../../src/nest/maps/maps.helpers'; + +const svcOf = () => new MapsService(new DatabaseService(db as never), {} as never); + +// The Brandenburg Gate and the underground station named after it, 250m apart. +const GATE = { lat: 52.5163, lng: 13.3777 }; +const hit = (over: Record = {}) => ({ + osm_type: 'way', + osm_id: '518071791', + name: 'Brandenburger Tor', + display_name: 'Brandenburger Tor, Pariser Platz, Berlin', + lat: '52.5163', + lon: '13.3777', + importance: 0.72, + extratags: { wikidata: 'Q82425', wikimedia_commons: 'Category:Brandenburg Gate' }, + ...over, +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('toWikiLang', () => { + it('MAPS-157: maps TREK\'s "br" to Portuguese, not Breton', () => { + // br.wikipedia.org EXISTS and answers 200 — it is the Breton Wikipedia. + // Passing the raw code through fails silently, in Breton. + expect(toWikiLang('br')).toBe('pt'); + }); + + it('MAPS-158: drops the region subtag, which no wiki subdomain has', () => { + // pt-br.wikipedia.org does not resolve at all. + expect(toWikiLang('pt-BR')).toBe('pt'); + expect(toWikiLang('zh-TW')).toBe('zh'); + expect(toWikiLang('en-US')).toBe('en'); + }); + + it('MAPS-159: maps gr to el and falls back to English on nonsense', () => { + expect(toWikiLang('gr')).toBe('el'); + expect(toWikiLang('')).toBe('en'); + expect(toWikiLang(undefined)).toBe('en'); + // Anything that is not shaped like a language code at all. + expect(toWikiLang('deutsch')).toBe('en'); + expect(toWikiLang('12')).toBe('en'); + }); +}); + +describe('namesOverlap', () => { + it('MAPS-160: matches across languages on a shared substantial word', () => { + expect(namesOverlap('Hamburg Airport', 'Flughafen Hamburg')).toBe(true); + }); + + it('MAPS-161: rejects two different places in the same area', () => { + expect(namesOverlap('Hamburg Airport', 'Bahnhof Ohlsdorf')).toBe(false); + }); + + it('MAPS-162: ignores accents', () => { + expect(namesOverlap('Café Zürich', 'Cafe Zurich')).toBe(true); + }); + + it('MAPS-162b: will not match on a single short word', () => { + // An article, and TREK speaks 23 languages — a stopword list for all of + // them is its own problem, so weight decides instead. + expect(namesOverlap('Der Kiosk', 'Der Bahnhof')).toBe(false); + // Two short words together are evidence enough. + expect(namesOverlap('Bar zum Tor', 'Cafe zum Tor')).toBe(true); + // And one substantial word on its own is. + expect(namesOverlap('Brandenburger Tor', 'Brandenburger Tor (U-Bahn)')).toBe(true); + }); +}); + +describe('haversineMetres', () => { + it('MAPS-163: measures the gate to its underground station at roughly 250m', () => { + const metres = haversineMetres(GATE.lat, GATE.lng, 52.5166047, 13.3809897); + expect(metres).toBeGreaterThan(150); + expect(metres).toBeLessThan(400); + }); + + it('MAPS-164: is zero for the same point', () => { + expect(haversineMetres(GATE.lat, GATE.lng, GATE.lat, GATE.lng)).toBeCloseTo(0, 5); + }); +}); + +describe('resolveOsmIdentity', () => { + it('MAPS-165: bounds the search to the area around the point', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => [] }); + vi.stubGlobal('fetch', fetchMock); + + await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng, { lang: 'de' }); + + const url = decodeURIComponent(String(fetchMock.mock.calls[0][0])); + // Without bounded+viewbox, Nominatim answers with the most famous place on + // earth carrying this name rather than the one being looked at. + expect(url).toContain('bounded=1'); + expect(url).toContain('viewbox='); + expect(url).toContain('extratags=1'); + }); + + it('MAPS-166: hands back the tags of the best local match', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [hit()] })); + + const out = await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng); + + expect(out?.tags.wikidata).toBe('Q82425'); + expect(out?.osmUrl).toBe('https://www.openstreetmap.org/way/518071791'); + }); + + it('MAPS-167: prefers the more important match over the merely closer one', async () => { + // The gate and the station share a name and sit 250m apart. Distance alone + // picks whichever the search coordinate happened to land on. + const station = hit({ + osm_id: '3862767512', + osm_type: 'node', + lat: '52.5166047', + lon: '13.3809897', + importance: 0.41, + extratags: { wikidata: 'Q477185' }, + }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [station, hit()] })); + + const out = await svcOf().resolveOsmIdentity('Brandenburger Tor', 52.5166, 13.3809); + + expect(out?.tags.wikidata).toBe('Q82425'); + }); + + it('MAPS-168: refuses a match too far from where we are looking', async () => { + const faraway = hit({ lat: '48.8584', lon: '2.2945' }); // Paris + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [faraway] })); + + expect(await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng)).toBeNull(); + }); + + it('MAPS-169: refuses a match that shares no word with the name', async () => { + // Right area, wrong building — this is the gate that stops the column + // describing the neighbour. + const neighbour = hit({ name: 'Hotel Adlon', display_name: 'Hotel Adlon, Berlin' }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [neighbour] })); + + expect(await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng)).toBeNull(); + }); + + it('MAPS-170: survives an empty name, a bad response, a throw and a non-array body', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + expect(await svcOf().resolveOsmIdentity('', GATE.lat, GATE.lng)).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => [] })); + expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull(); + + // Nominatim answers rate limiting in plain text, not JSON. + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => 'Bandwidth limit exceeded' })); + expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull(); + + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); + expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull(); + }); +}); + +describe('fetchWikidataSitelinks', () => { + it('MAPS-171: asks only for the sites it will use', async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ entities: { Q1097: { sitelinks: { dewiki: { title: 'Berlin Hauptbahnhof' } } } } }), + }); + vi.stubGlobal('fetch', fetchMock); + + const out = await svcOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']); + + expect(decodeURIComponent(String(fetchMock.mock.calls[0][0]))).toContain('sitefilter=dewikivoyage|dewiki|enwiki'); + expect(out).toEqual({ dewiki: 'Berlin Hauptbahnhof' }); + }); + + it('MAPS-172: refuses a non-Q id and an empty site list without calling out', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + expect(await svcOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({}); + expect(await svcOf().fetchWikidataSitelinks('Q1', [])).toEqual({}); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('MAPS-173: yields an empty map on a bad response or a throw', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); + expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); + + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); + expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); + }); +}); diff --git a/server/tests/unit/nest/place-enrichment.description.test.ts b/server/tests/unit/nest/place-enrichment.description.test.ts new file mode 100644 index 000000000..ccf71faff --- /dev/null +++ b/server/tests/unit/nest/place-enrichment.description.test.ts @@ -0,0 +1,270 @@ +/** + * ENRICH-070..082 — where a description comes from. + * + * The cases the column got wrong on dev1: L'Osteria Rostock, Hamburg Airport + * and Berlin Brandenburg Airport all showed no description at all with a Google + * key configured, and Berlin Hauptbahnhof showed none without one. Two distinct + * causes, both pinned here — Google short-circuiting the free chain, and the + * chain itself only ever starting from an OSM `wikipedia` tag. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const { mockDbGet, mockDbRun } = vi.hoisted(() => ({ + mockDbGet: vi.fn(() => undefined as any), + mockDbRun: vi.fn(), +})); + +vi.mock('../../../src/db/database', () => ({ + db: { + prepare: (sql: string) => ({ + get: (...params: unknown[]) => mockDbGet(sql, ...params), + run: (...params: unknown[]) => mockDbRun(sql, ...params), + all: () => [], + }), + }, +})); + +vi.mock('../../../src/utils/ssrfGuard', () => ({ + safeFetchFollow: vi.fn(async () => ({ ok: false, arrayBuffer: async () => new ArrayBuffer(0) })), + checkSsrf: vi.fn(async () => ({ allowed: true })), + SsrfBlockedError: class extends Error {}, +})); + +vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) })); + +import { db } from '../../../src/db/database'; +import { DatabaseService } from '../../../src/nest/database/database.service'; +import { PlaceEnrichmentService } from '../../../src/nest/place-enrichment/place-enrichment.service'; +import type { MapsService } from '../../../src/nest/maps/maps.service'; +import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; + +const GOOGLE_REQ = { lat: 53.6323, lng: 10.0067, name: 'Hamburg Airport', placeId: 'ChIJham', lang: 'de' }; +const OSM_REQ = { lat: 52.525, lng: 13.3694, name: 'Berlin Hauptbahnhof', placeId: 'relation:3600565', lang: 'de' }; + +const EXTRACT = { + text: 'Der Berliner Hauptbahnhof ist der größte Kreuzungsbahnhof Europas.', + sourceUrl: 'https://de.wikipedia.org/wiki/Berlin_Hauptbahnhof', + source: 'wikipedia' as const, +}; + +function mapsStub(over: Partial> = {}) { + return { + getMapsKey: vi.fn(() => null as string | null), + photosDisabled: vi.fn(() => false), + detailsDisabled: vi.fn(() => false), + fetchGooglePhotoRefs: vi.fn(async () => []), + fetchGooglePhotoBytes: vi.fn(async () => null), + fetchCommonsCandidates: vi.fn(async () => []), + fetchCommonsCategoryCandidates: vi.fn(async () => []), + fetchWikidataCandidates: vi.fn(async () => ({ candidates: [], commonsCategory: null })), + fetchCommonsFilesByName: vi.fn(async () => new Map()), + fetchWikiLeadImageName: vi.fn(async () => null), + fetchEditorialSummary: vi.fn(async () => null as string | null), + fetchWikiExtract: vi.fn(async () => null as typeof EXTRACT | null), + fetchWikiExtractFor: vi.fn(async () => null as typeof EXTRACT | null), + fetchWikidataSitelinks: vi.fn(async () => ({}) as Record), + resolveOsmIdentity: vi.fn(async () => null as { tags: Record; osmUrl: string | null; matchedName: string } | null), + details: vi.fn(async () => ({ place: null })), + ...over, + } as unknown as MapsService; +} + +const cacheStub = () => + ({ get: vi.fn(() => null), put: vi.fn(async () => ({ photoUrl: '/x', filePath: '/x', attribution: null })) }) as unknown as PlacePhotoCacheService; + +const make = (maps: MapsService) => new PlaceEnrichmentService(new DatabaseService(db as never), maps, cacheStub()); + +beforeEach(() => { + mockDbGet.mockReset(); + mockDbGet.mockReturnValue(undefined); + mockDbRun.mockReset(); +}); + +describe('identity resolution', () => { + it('ENRICH-070: looks up a Google place in OpenStreetMap, which is where its wiki tags live', async () => { + // Google's payload has no wikidata, no wikipedia and no wikimedia_commons — + // it never had. Before this, configuring an API key silently switched off + // the free half of the column. + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + details: vi.fn(async () => ({ place: { source: 'google', name: 'Hamburg Airport' } })), + resolveOsmIdentity: vi.fn(async () => ({ + tags: { wikidata: 'Q27706', wikipedia: 'de:Flughafen Hamburg' }, + osmUrl: 'https://www.openstreetmap.org/way/147122729', + matchedName: 'Hamburg Airport', + })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Hamburg-Fuhlsbüttel' })), + fetchWikiExtractFor: vi.fn(async () => ({ ...EXTRACT, source: 'wikivoyage' as const })), + }); + + const out = await make(maps).enrich(1, GOOGLE_REQ); + + expect(maps.resolveOsmIdentity).toHaveBeenCalledWith('Hamburg Airport', 53.6323, 10.0067, { lang: 'de' }); + expect(out.description).toMatchObject({ source: 'wikivoyage', license: 'CC BY-SA 4.0' }); + }); + + it('ENRICH-071: does not spend a lookup when the payload already carries tags', async () => { + const maps = mapsStub({ + details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof' })), + fetchWikiExtractFor: vi.fn(async () => EXTRACT), + }); + + await make(maps).enrich(1, OSM_REQ); + + expect(maps.resolveOsmIdentity).not.toHaveBeenCalled(); + }); + + it('ENRICH-072: keeps OSM tags out of a Google payload', async () => { + // collectFacts and the OSM branch of collectDescription both gate on + // details.source. Merging the looked-up tags in would either be dead weight + // or would make a Google place claim to be an OpenStreetMap one. + const details = { source: 'google', name: 'Hamburg Airport', summary: null }; + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + details: vi.fn(async () => ({ place: details })), + resolveOsmIdentity: vi.fn(async () => ({ + tags: { wikidata: 'Q27706', cuisine: 'german' }, + osmUrl: null, + matchedName: 'Hamburg Airport', + })), + }); + + const out = await make(maps).enrich(1, GOOGLE_REQ); + + expect(details).toEqual({ source: 'google', name: 'Hamburg Airport', summary: null }); + // Cuisine is an OSM fact and this is a Google place, so it stays out. + expect(out.facts.some((f) => f.kind === 'cuisine')).toBe(false); + }); +}); + +describe('description source order', () => { + it('ENRICH-073: prefers a free article over Google, even with a key configured', async () => { + // This is the whole point of the reordering: Google used to be asked first + // and answered for almost nothing, so the column stayed empty while a + // perfectly good article sat one call away. + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + details: vi.fn(async () => ({ place: { source: 'google', wikidata: 'Q1097' } })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof' })), + fetchWikiExtractFor: vi.fn(async () => EXTRACT), + fetchEditorialSummary: vi.fn(async () => 'A large railway station.'), + }); + + const out = await make(maps).enrich(1, { ...OSM_REQ, placeId: 'ChIJhbf' }); + + expect(out.description).toMatchObject({ source: 'wikipedia' }); + // Not merely lower priority — the billed call is never made. + expect(maps.fetchEditorialSummary).not.toHaveBeenCalled(); + }); + + it('ENRICH-074: still falls back to Google when nothing free describes the place', async () => { + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + details: vi.fn(async () => ({ place: { source: 'google', google_maps_url: 'https://maps.google/x' } })), + fetchEditorialSummary: vi.fn(async () => 'Casual chain for wood-fired pizza.'), + }); + + const out = await make(maps).enrich(1, { ...GOOGLE_REQ, name: "L'Osteria Rostock" }); + + expect(out.description).toMatchObject({ + source: 'google', + text: 'Casual chain for wood-fired pizza.', + sourceUrl: 'https://maps.google/x', + license: null, + }); + }); + + it('ENRICH-075: keeps the OpenStreetMap description ahead of everything', async () => { + const maps = mapsStub({ + details: vi.fn(async () => ({ + place: { source: 'openstreetmap', summary: ' Ein Bahnhof. ', osm_url: 'https://osm.org/r/1', wikidata: 'Q1097' }, + })), + fetchWikiExtractFor: vi.fn(async () => EXTRACT), + }); + + const out = await make(maps).enrich(1, OSM_REQ); + + expect(out.description).toMatchObject({ text: 'Ein Bahnhof.', source: 'osm', license: 'ODbL 1.0' }); + expect(maps.fetchWikidataSitelinks).not.toHaveBeenCalled(); + }); +}); + +describe('article language choice', () => { + it('ENRICH-076: asks for the reader\'s language before English', async () => { + const maps = mapsStub({ + details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof', enwiki: 'Berlin Hauptbahnhof' })), + fetchWikiExtractFor: vi.fn(async () => EXTRACT), + }); + + await make(maps).enrich(1, OSM_REQ); + + const sites = (maps.fetchWikidataSitelinks as unknown as { mock: { calls: [string, string[]][] } }).mock.calls[0][1]; + expect(sites.slice(0, 2)).toEqual(['dewikivoyage', 'dewiki']); + expect(sites).toContain('enwiki'); + expect(maps.fetchWikiExtractFor).toHaveBeenCalledWith('wikipedia', 'de', 'Berlin Hauptbahnhof'); + }); + + it('ENRICH-077: puts Wikivoyage ahead of Wikipedia in the same language', async () => { + // Wikivoyage describes a place for somebody about to go there; Wikipedia + // opens with area in square kilometres. + const maps = mapsStub({ + details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Berlin', dewiki: 'Berlin Hauptbahnhof' })), + fetchWikiExtractFor: vi.fn(async () => ({ ...EXTRACT, source: 'wikivoyage' as const })), + }); + + await make(maps).enrich(1, OSM_REQ); + + expect(maps.fetchWikiExtractFor).toHaveBeenNthCalledWith(1, 'wikivoyage', 'de', 'Berlin'); + }); + + it('ENRICH-078: keeps the language the place itself named, ahead of English', async () => { + // A Korean reader looking at the Brandenburg Gate: no kowiki sitelink, but + // the OSM tag names a German article somebody curated for this place. + const maps = mapsStub({ + details: vi.fn(async () => ({ + place: { source: 'openstreetmap', wikidata: 'Q82425', wikipedia: 'de:Brandenburger Tor' }, + })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Brandenburger Tor', enwiki: 'Brandenburg Gate' })), + fetchWikiExtractFor: vi.fn(async () => EXTRACT), + }); + + await make(maps).enrich(1, { ...OSM_REQ, lang: 'ko' }); + + const sites = (maps.fetchWikidataSitelinks as unknown as { mock: { calls: [string, string[]][] } }).mock.calls[0][1]; + expect(sites.indexOf('dewiki')).toBeLessThan(sites.indexOf('enwiki')); + }); + + it('ENRICH-079: falls back to the tag when the item has no usable sitelink', async () => { + const maps = mapsStub({ + details: vi.fn(async () => ({ + place: { source: 'openstreetmap', wikidata: 'Q1', wikipedia: 'de:Museum Ludwig' }, + })), + fetchWikidataSitelinks: vi.fn(async () => ({})), + fetchWikiExtract: vi.fn(async () => EXTRACT), + }); + + const out = await make(maps).enrich(1, OSM_REQ); + + expect(maps.fetchWikiExtract).toHaveBeenCalledWith('de:Museum Ludwig'); + expect(out.description).toMatchObject({ source: 'wikipedia' }); + }); + + it('ENRICH-080: tries the next site when one has no article', async () => { + const maps = mapsStub({ + details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })), + fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Nowhere', dewiki: 'Berlin Hauptbahnhof' })), + fetchWikiExtractFor: vi + .fn() + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(EXTRACT), + }); + + const out = await make(maps).enrich(1, OSM_REQ); + + expect(maps.fetchWikiExtractFor).toHaveBeenCalledTimes(2); + expect(out.description).toMatchObject({ source: 'wikipedia' }); + }); +}); diff --git a/server/tests/unit/nest/place-enrichment.service.test.ts b/server/tests/unit/nest/place-enrichment.service.test.ts index 0fca91f8b..7940acae0 100644 --- a/server/tests/unit/nest/place-enrichment.service.test.ts +++ b/server/tests/unit/nest/place-enrichment.service.test.ts @@ -58,6 +58,9 @@ function mapsStub(over: Partial> = {}) { fetchWikidataCandidates: vi.fn(async () => ({ candidates: [] as any[], commonsCategory: null as string | null })), fetchCommonsFilesByName: vi.fn(async () => new Map()), fetchWikiLeadImageName: vi.fn(async () => null as string | null), + resolveOsmIdentity: vi.fn(async () => null as { tags: Record; osmUrl: string | null; matchedName: string } | null), + fetchWikidataSitelinks: vi.fn(async () => ({}) as Record), + fetchWikiExtractFor: vi.fn(async () => null as { text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null), details: vi.fn(async () => ({ place: null })), ...over, } as unknown as MapsService; @@ -443,7 +446,10 @@ describe('collectDescription', () => { await make(maps, cacheStub()).enrich(1, OSM_REQ); - expect(maps.fetchWikiExtract).toHaveBeenCalledWith(null); + // Not a usable tag, so it counts as no identity at all: the lookup that + // finds one is tried instead, and the wiki itself is never asked. + expect(maps.resolveOsmIdentity).toHaveBeenCalled(); + expect(maps.fetchWikiExtract).not.toHaveBeenCalled(); }); it('ENRICH-017: honours the details kill switch and the missing key', async () => { @@ -477,12 +483,16 @@ describe('collectDescription', () => { }); }); - it('ENRICH-019: passes null on when the place carries no wiki tag', async () => { + it('ENRICH-019: looks the place up when it carries no wiki tag, and gives up quietly when that finds nothing', async () => { + // The kiosk case. A place with no tag and no OSM match genuinely has no + // description, and inventing one from a name search is how you confidently + // describe a different building. const maps = mapsStub({ details: vi.fn(async () => ({ place: { name: 'Kiosk' } })) }); const out = await make(maps, cacheStub()).enrich(1, OSM_REQ); - expect(maps.fetchWikiExtract).toHaveBeenCalledWith(null); + expect(maps.resolveOsmIdentity).toHaveBeenCalledWith('Museum Ludwig', 50.9, 6.96, { lang: undefined }); + expect(maps.fetchWikiExtract).not.toHaveBeenCalled(); expect(out.description).toBeNull(); });