From 0cedff94a0072ef567717ad753733d985c159995 Mon Sep 17 00:00:00 2001 From: Maurice Date: Sat, 10 Oct 2026 12:06:54 +0200 Subject: [PATCH] refactor(maps): wikimedia lookups and brand logos move into their own client --- server/scripts/import-boundaries.mjs | 1 + server/src/nest/maps/maps.module.ts | 5 +- server/src/nest/maps/maps.service.ts | 881 +----------------- server/src/nest/maps/photo-fetch-slot.ts | 41 + .../nest/maps/providers/wikimedia.client.ts | 830 +++++++++++++++++ .../place-enrichment.service.ts | 18 +- server/tests/e2e/place-enrichment.e2e.test.ts | 25 +- server/tests/helpers/maps-service.ts | 7 +- .../unit/nest/maps.curated-images.test.ts | 5 +- server/tests/unit/nest/maps.identity.test.ts | 15 +- server/tests/unit/nest/maps.service.test.ts | 80 +- .../unit/nest/place-enrichment.brand.test.ts | 5 +- .../nest/place-enrichment.description.test.ts | 5 +- .../nest/place-enrichment.service.test.ts | 5 +- 14 files changed, 977 insertions(+), 946 deletions(-) create mode 100644 server/src/nest/maps/photo-fetch-slot.ts create mode 100644 server/src/nest/maps/providers/wikimedia.client.ts diff --git a/server/scripts/import-boundaries.mjs b/server/scripts/import-boundaries.mjs index c78bbe956..2953a7043 100644 --- a/server/scripts/import-boundaries.mjs +++ b/server/scripts/import-boundaries.mjs @@ -102,6 +102,7 @@ export const PUBLIC_FILES = [ // asks Google and Wikimedia directly instead of through a MapsService facade. 'maps/providers/google-places.provider.ts', 'maps/providers/osm.client.ts', + 'maps/providers/wikimedia.client.ts', ]; function walk(dir, files = []) { diff --git a/server/src/nest/maps/maps.module.ts b/server/src/nest/maps/maps.module.ts index 5df215e5e..26742f3ed 100644 --- a/server/src/nest/maps/maps.module.ts +++ b/server/src/nest/maps/maps.module.ts @@ -5,6 +5,7 @@ import { MapsService } from './maps.service'; import { MapsMcp } from './maps.mcp'; import { GooglePlacesClient } from './providers/google-places.provider'; import { OsmClient } from './providers/osm.client'; +import { WikimediaClient } from './providers/wikimedia.client'; import { PlacePhotosModule } from '../place-photos/place-photos.module'; import { StorageModule } from '../storage/storage.module'; import { GoogleQuotaModule } from '../google-quota/google-quota.module'; @@ -30,7 +31,7 @@ import { Places } from '../../db/entities/Places.entity'; @Module({ imports: [PlacePhotosModule, StorageModule, GoogleQuotaModule, MikroOrmModule.forFeature([AppSettings, Users, PlaceDetailsCache, Places])], controllers: [MapsController], - providers: [MapsService, MapsMcp, GooglePlacesClient, OsmClient], - exports: [MapsService, GooglePlacesClient, OsmClient], + providers: [MapsService, MapsMcp, GooglePlacesClient, OsmClient, WikimediaClient], + exports: [MapsService, GooglePlacesClient, OsmClient, WikimediaClient], }) export class MapsModule {} diff --git a/server/src/nest/maps/maps.service.ts b/server/src/nest/maps/maps.service.ts index 0a72f308c..d22eff78c 100644 --- a/server/src/nest/maps/maps.service.ts +++ b/server/src/nest/maps/maps.service.ts @@ -8,10 +8,9 @@ import type { MapsReverseResult, MapsResolveUrlResult, } from '@trek/shared'; -import { Jimp } from 'jimp'; import { readEnv } from '../../app-config'; import { safeFetchFollow, SsrfBlockedError } from '../../utils/ssrfGuard'; -import { discardBody, exceedsDeclaredLength, readCapped, readCappedText } from '../../utils/cappedFetch'; +import { discardBody, exceedsDeclaredLength, readCappedText } from '../../utils/cappedFetch'; import { resolveApiKey, type ApiKeySource } from '../settings/instance-api-keys'; import { InjectRepository } from '@mikro-orm/nestjs'; import { AppSettings } from '../../db/entities/AppSettings.entity'; @@ -25,6 +24,8 @@ import type { PlacesRepository } from '../../db/repositories/Places.repository'; import { isPlacesProviderChoice, type PlacesProviderChoice } from './providers/places-provider'; import { GooglePlacesClient } from './providers/google-places.provider'; import { OsmClient, POI_RESULT_CAP, type PoiSearchResult } from './providers/osm.client'; +import { WikimediaClient, type BrandLogo } from './providers/wikimedia.client'; +import { withPhotoFetchSlot } from './photo-fetch-slot'; import { AMAP_SHORT_HOSTS, AmapPlacesProvider, @@ -57,9 +58,6 @@ import { googleFtidFromMapsUrl, OSM_PLACE_ID, parsePoiCategories, - stripWikiMarkup, - parseWikipediaTag, - toWikiLang, mergeSearchResults, } from './maps.helpers'; import { @@ -73,153 +71,9 @@ import { export { readBrandIdentity, readWikiIdentity, type WikiIdentity } from './providers/wiki-identity'; -interface WikidataSnak { - mainsnak?: { datavalue?: { value?: string } }; - rank?: 'preferred' | 'normal' | 'deprecated'; -} -type WikidataClaims = Record; +export { withPhotoFetchSlot } from './photo-fetch-slot'; +export type { BrandLogo, CommonsCandidate } from './providers/wikimedia.client'; -/** - * Wikidata properties that name a picture of a place, in the order we want them. - * - * P18 is the representative image. The rest exist because a station or a - * monument is not one view: asking for the interior, the night shot, the - * panorama and the aerial gives a picker four genuinely different pictures - * instead of four frames of the same façade. - */ -const WIKIDATA_IMAGE_PROPERTIES = [ - 'P18', // image - 'P5775', // interior view - 'P3451', // night view - 'P8592', // aerial view - 'P4291', // panoramic view - 'P5252', // winter view - 'P948', // Wikivoyage banner -] as const; - -/** First non-empty string value of a claim list. */ -function claimValue(snaks: WikidataSnak[] | undefined): string | null { - for (const snak of snaks ?? []) { - const value = snak.mainsnak?.datavalue?.value; - if (typeof value === 'string' && value.trim()) return value.trim(); - } - return null; -} - -/** - * File names from an item's picture properties, best first. - * - * Within P18 the statement rank decides: an item with several images marks one - * `preferred`, and that is the one an editor considers representative. The API - * returns statements in edit order, not rank order, so taking `[0]` picks - * whichever was added first — for the Brandenburg Gate that is a coin toss - * between the morning shot and a wide overview. - */ -function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] { - const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const; - const names: string[] = []; - const seen = new Set(); - - for (const property of WIKIDATA_IMAGE_PROPERTIES) { - const snaks = [...(claims[property] ?? [])] - .filter((snak) => snak.rank !== 'deprecated') - .sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1)); - for (const snak of snaks) { - const value = snak.mainsnak?.datavalue?.value; - if (typeof value !== 'string' || !value.trim()) continue; - const key = normalizeFileTitle(value); - if (seen.has(key)) continue; - seen.add(key); - names.push(value.trim()); - if (names.length >= limit) return names; - } - } - return names; -} - -/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */ -function normalizeFileTitle(title: string): string { - return title.replace(/^File:/i, '').replaceAll('_', ' ').trim().toLowerCase(); -} - -/** - * A bare Commons category name out of whatever an OSM `wikimedia_commons` tag - * holds. - * - * The tag is free text and mappers put three different things in it: a bare - * name, a prefixed `Category:…`, or — against the wiki's own advice — a single - * `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`, - * which matches nothing and fell through to the coordinate search without a - * word. Localised prefixes (`Kategorie:`) appear too. - */ -function normalizeCategoryName(raw: string): string | null { - const value = raw.trim(); - if (!value) return null; - // A file name is not a category, and there is nothing sensible to derive. - if (/^(file|datei|image|bild)\s*:/i.test(value)) return null; - return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null; -} - -interface WikiCommonsPage { - pageid?: number; - title?: string; - imageinfo?: { - url?: string; - thumburl?: string; - mime?: string; - width?: number; - height?: number; - /** The file description page — where the full licence terms live. */ - descriptionurl?: string; - extmetadata?: { - Artist?: { value?: string }; - LicenseShortName?: { value?: string }; - LicenseUrl?: { value?: string }; - UsageTerms?: { value?: string }; - /** Used to spot survey imagery and diagrams, which are not pictures of a place. */ - Categories?: { value?: string }; - ObjectName?: { value?: string }; - ImageDescription?: { value?: string }; - }; - }[]; -} - -/** - * One Commons image with everything needed to credit it. Commons is mostly - * CC BY / CC BY-SA, so a candidate that cannot be attributed is not usable in a - * picker — the fields are nullable because Commons metadata is user-maintained - * and genuinely incomplete on some files, not because they are optional to show. - */ -export interface CommonsCandidate { - photoUrl: string; - attribution: string | null; - license: string | null; - licenseUrl: string | null; - sourceUrl: string | null; - /** - * Commons page id — the only stable identity a file has across the four ways - * we reach it. The thumbnail URL is not: the same file comes back from - * commons.wikimedia.org and from a language Wikipedia with different query - * strings, so deduplicating on the URL silently lets the same picture through - * twice. It is also what keys the cached bytes, so it must survive. - */ - pageId: number | null; - /** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */ - title: string | null; - width: number | null; - height: number | null; - /** Free text used to reject survey imagery, floor plans and logos. */ - descriptors: string | null; -} - -// ── Concurrency limiter for outbound photo fetches ─────────────────────────── -// Caps simultaneous Wikimedia/Google photo requests so a bulk import of hundreds -// of places cannot monopolise the event loop or trigger external API rate limits. -// Module-scoped ON PURPOSE (permissions-cache precedent): the bridge instance and -// the DI singleton must share one limiter, one POI cache and one call counter. -// Wikimedia is normally well under a second, but a cold TLS handshake from a -// fresh container has been seen at eight. Enrichment answers a live dialog, so -// a slow provider is dropped rather than waited out. // A Google Maps place page is a few hundred KB; the coordinates sit in the // embedded map data near the top, so two megabytes is plenty and keeps an // unbounded body out of memory. @@ -239,144 +93,6 @@ export function isGoogleMapsHost(hostname: string): boolean { || /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname); } -const WIKI_TIMEOUT_MS = 6000; - -// ── Brand logos ────────────────────────────────────────────────────────────── -// -// A road trip corridor is mostly chains — Shell, Aral, JET — and the brand is the -// fastest thing to recognise on a map. OSM carries `brand:wikidata` on most of them, -// Wikidata carries the logo (P154), and Commons serves it. -// -// The bytes are proxied rather than linked so the browser never talks to Wikimedia: -// one self-hosted instance asking for a handful of logos is a very different egress -// profile from every visitor's browser announcing which petrol stations they are -// looking at. The cache is in memory on purpose — a GET must not write to the DB, and -// there are only so many fuel brands. -const BRAND_LOGO_CACHE_TTL_MS = 24 * 60 * 60 * 1000; -const BRAND_LOGO_CACHE_MAX = 300; -/** Well past any logo; a Commons original can be a multi-megabyte SVG or print-res PNG. */ -const BRAND_LOGO_MAX_BYTES = 512 * 1024; -/** A place photo is a full-size image rather than a 128px mark, so its own ceiling. */ -const WIKIMEDIA_PHOTO_MAX_BYTES = 8 * 1024 * 1024; -const BRAND_LOGO_WIDTH = 128; -/** The square the logo is centred in, and the breathing room around it. */ -const BRAND_LOGO_CANVAS = 96; -const BRAND_LOGO_PADDING = 8; -const WIKIDATA_ID_RE = /^Q[1-9][0-9]{0,11}$/; - -export interface BrandLogo { - bytes: Buffer; - contentType: string; -} - -/** - * Puts a logo on a background it can actually be seen against. - * - * Half the fuel brands ship a white wordmark with a transparent background — - * TotalEnergies, Esso and JET among them — and on the white pill the marker used to - * draw they were invisible. Which background is right is a property of the image, not - * of the brand, so it is measured: the mean brightness of the pixels that are actually - * opaque decides between a white and a near-black backdrop. - * - * Returns the flattened PNG, or null when the bytes cannot be read at all — the pin - * then keeps its category icon, which is a fine outcome. - */ -async function flattenBrandLogo(bytes: Buffer): Promise { - try { - const image = await Jimp.read(bytes); - const width = image.bitmap.width; - const height = image.bitmap.height; - if (!width || !height) return null; - - let sum = 0; - let opaque = 0; - for (let y = 0; y < height; y++) { - for (let x = 0; x < width; x++) { - const idx = (y * width + x) * 4; - const alpha = image.bitmap.data[idx + 3]; - // Half-transparent pixels are anti-aliasing, not the mark itself. - if (alpha < 128) continue; - const r = image.bitmap.data[idx]; - const g = image.bitmap.data[idx + 1]; - const b = image.bitmap.data[idx + 2]; - sum += 0.2126 * r + 0.7152 * g + 0.0722 * b; - opaque++; - } - } - // A logo with nothing opaque in it is not a logo. - if (opaque === 0) return null; - - // Square, with the logo scaled to fit inside it. Most brand marks are wide wordmarks - // — TotalEnergies is three times as wide as it is tall — and a round pin showing the - // middle of one is unreadable. Fitting it into a square means the pin can crop to a - // circle without ever cutting the mark itself. - const scale = Math.min( - (BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / width, - (BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / height, - ); - // Never upscale: a 16-pixel favicon blown up to 96 looks worse than a small one. - if (scale < 1) image.scale(scale); - - const light = sum / opaque > 150; - const canvas = new Jimp({ - width: BRAND_LOGO_CANVAS, - height: BRAND_LOGO_CANVAS, - color: light ? 0x111827ff : 0xffffffff, - }); - canvas.composite( - image, - Math.round((BRAND_LOGO_CANVAS - image.bitmap.width) / 2), - Math.round((BRAND_LOGO_CANVAS - image.bitmap.height) / 2), - ); - const out = await canvas.getBuffer('image/png'); - return { bytes: Buffer.from(out), contentType: 'image/png' }; - } catch { - return null; - } -} - -// Tighter than the wiki calls, because this one sits at the FRONT of a chain: -// identity, then sitelinks, then the extract. Nominatim answers a bounded -// search in 0.2-0.7s in practice, so anything past a couple of seconds is a bad -// day at the provider rather than a slow answer worth waiting for. -const IDENTITY_TIMEOUT_MS = 2500; - -const MAX_CONCURRENT_PHOTO_FETCHES = 5; -let photoFetchActive = 0; -const photoFetchQueue: Array<() => void> = []; - -function acquirePhotoFetchSlot(): Promise { - if (photoFetchActive < MAX_CONCURRENT_PHOTO_FETCHES) { - photoFetchActive++; - return Promise.resolve(); - } - return new Promise((resolve) => photoFetchQueue.push(resolve)); -} - -function releasePhotoFetchSlot(): void { - const next = photoFetchQueue.shift(); - if (next) { - next(); - } else { - photoFetchActive--; - } -} - -/** - * Runs an outbound photo fetch under the shared slot limit. Exported so the - * enrichment module queues behind the same five slots — a picker that grabs - * three images per selected place would otherwise sail past a cap the rest of - * the app respects. - */ -export async function withPhotoFetchSlot(fn: () => Promise): Promise { - await acquirePhotoFetchSlot(); - try { - return await fn(); - } finally { - releasePhotoFetchSlot(); - } -} - // Places near a point (#976): cached longer than the POI boxes, because the // likely caller is an import asking the same photo location again, and Google // bills every one of those. @@ -452,11 +168,9 @@ export class MapsService { private readonly googleQuota: GoogleQuotaService, private readonly googlePlaces: GooglePlacesClient, private readonly osm: OsmClient, + private readonly wiki: WikimediaClient, ) {} - /** Brand id → logo bytes, or null for "asked, has none". Insertion-ordered, so the - * oldest entry is the one evicted when it fills up. */ - private readonly brandLogoCache = new Map(); /** Amap autocomplete tips for the details fallback. Here rather than on the provider: * a provider is built per request, and a pick is two requests. */ @@ -562,74 +276,9 @@ export class MapsService { return this.resolveGoogleMapsUrl(url) as Promise; } - /** - * The logo of a brand, by its Wikidata id, as bytes. - * - * Two hops: Wikidata says which Commons file is the logo (property P154), Commons - * serves a thumbnail of it. Both answers are cached, including "this brand has no - * logo" — otherwise every map pan would ask Wikidata about the same supermarket - * chain again. Returns null whenever anything is missing or unreadable; a marker - * without a logo falls back to its category icon, which is a fine outcome. - */ - async brandLogo(wikidataId: string): Promise { - if (!WIKIDATA_ID_RE.test(wikidataId)) return null; - - const cached = this.brandLogoCache.get(wikidataId); - if (cached && Date.now() - cached.at < BRAND_LOGO_CACHE_TTL_MS) return cached.logo; - - const remember = (logo: BrandLogo | null): BrandLogo | null => { - if (this.brandLogoCache.size >= BRAND_LOGO_CACHE_MAX) { - const oldest = this.brandLogoCache.keys().next().value; - if (oldest !== undefined) this.brandLogoCache.delete(oldest); - } - this.brandLogoCache.set(wikidataId, { at: Date.now(), logo }); - return logo; - }; - - try { - const params = new URLSearchParams({ - action: 'wbgetclaims', - entity: wikidataId, - property: 'P154', - format: 'json', - }); - const claimRes = await fetch(`https://www.wikidata.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!claimRes.ok) return remember(null); - const claims = (await claimRes.json()) as { - claims?: { P154?: { mainsnak?: { datavalue?: { value?: unknown } } }[] }; - }; - const file = claims.claims?.P154?.[0]?.mainsnak?.datavalue?.value; - if (typeof file !== 'string' || !file.trim()) return remember(null); - - // Special:FilePath renders a thumbnail at the width asked for and redirects to - // the CDN, so each hop is re-checked by the guard rather than trusted. - const url = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(file)}?width=${BRAND_LOGO_WIDTH}`; - // The same six seconds the Wikidata hop above allows. Without a deadline - // this waited on undici's five-minute default, holding a request context - // and a socket per stalled logo while the pin sat on its fallback icon. - const imgRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true }); - if (!imgRes.ok) return remember(null); - - if (exceedsDeclaredLength(imgRes, BRAND_LOGO_MAX_BYTES)) { - discardBody(imgRes); - return remember(null); - } - // Streamed rather than buffered whole: a chunked answer declares no length, - // so the post-check only ever ran after the bytes were already in memory. - const { bytes, truncated } = await readCapped(imgRes, BRAND_LOGO_MAX_BYTES); - if (truncated || bytes.byteLength === 0) return remember(null); - - const contentType = imgRes.headers.get('content-type') ?? ''; - if (!contentType.startsWith('image/')) return remember(null); - - return remember(await flattenBrandLogo(bytes)); - } catch (err) { - if (err instanceof SsrfBlockedError) return remember(null); - return remember(null); - } + /** The logo of a brand, by its Wikidata id, as bytes (null: the pin keeps its category icon). */ + brandLogo(wikidataId: string): Promise { + return this.wiki.brandLogo(wikidataId); } // POI search by category within a viewport. Never calls Google. @@ -929,484 +578,6 @@ export class MapsService { return hit?.lat != null && hit?.lng != null ? { lat: hit.lat, lng: hit.lng } : null; } - // ── Wikimedia Commons photo lookup ───────────────────────────────────────── - - async fetchWikimediaPhoto( - lat: number, - lng: number, - name?: string, - ): Promise<{ photoUrl: string; attribution: string | null } | null> { - // Strategy 1: Search Wikipedia for the place name -> get the article image - if (name) { - try { - const searchParams = new URLSearchParams({ - action: 'query', - format: 'json', - titles: name, - prop: 'pageimages', - piprop: 'thumbnail', - pithumbsize: '400', - pilimit: '1', - redirects: '1', - }); - const res = await fetch(`https://en.wikipedia.org/w/api.php?${searchParams}`, { - headers: { 'User-Agent': UA }, - // This runs inside one of the few shared photo-fetch slots; a stalled - // answer would hold it for everyone. - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (res.ok) { - const data = (await res.json()) as { query?: { pages?: Record } }; - const pages = data.query?.pages; - if (pages) { - for (const page of Object.values(pages)) { - if (page.thumbnail?.source) { - return { photoUrl: page.thumbnail.source, attribution: 'Wikipedia' }; - } - } - } - } - } catch { - /* fall through to geosearch */ - } - } - - // Strategy 2: Wikimedia Commons geosearch by coordinates - const candidates = await this.fetchCommonsCandidates(lat, lng, 5); - const first = candidates[0]; - return first ? { photoUrl: first.photoUrl, attribution: first.attribution } : null; - } - - /** - * Commons images near a coordinate, licence metadata included. - * - * geosearch already returns up to `limit` files in a single request, so asking - * for a whole strip costs the same as asking for one picture. Callers that only - * want a single image (fetchWikimediaPhoto, and through it getPlacePhoto) take - * the first entry and get exactly the file they got before this existed. - */ - /** - * Anything photographed near a coordinate. The bottom rung of the picture - * ladder, and the only one with no claim on the subject at all. - * - * 60 metres rather than the 300 it used to be. Three hundred metres in a city - * centre is a whole block: the town hall, the church and the underground - * entrance are all inside it, and every one of them outranks the doner shop - * we were actually asked about. Sixty is roughly "the same building and its - * neighbours", which is the widest a picture can be taken and still plausibly - * show the place. It does not fix the real problem, which is that nobody - * photographed the shop, but it stops the wrong answer from being confident. - */ - async fetchCommonsCandidates(lat: number, lng: number, limit = 5): Promise { - const params = new URLSearchParams({ - action: 'query', - format: 'json', - generator: 'geosearch', - ggsprimary: 'all', - ggsnamespace: '6', - ggsradius: '60', - ggscoord: `${lat}|${lng}`, - // Deliberately more than the caller asked for. Around anything worth - // visiting the first few hits are survey tiles, passers-by and the - // building next door; the ranker needs a pool to reject from, and - // geosearch charges the same for one result as for twenty. - ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))), - prop: 'imageinfo', - iiprop: 'url|extmetadata|mime|size', - iiurlwidth: '400', - }); - try { - const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - // A hanging provider must not hold the whole enrichment request open; - // no pictures is a fine answer, a request that never returns is not. - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) return []; - const data = (await res.json()) as { query?: { pages?: Record } }; - // Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets - // what it always got; the enrichment column ranks before it cuts. - return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit'))); - } catch { - return []; - } - } - - /** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */ - private toCommonsCandidates( - pages: Record | undefined, - limit: number, - ): CommonsCandidate[] { - if (!pages) return []; - const out: CommonsCandidate[] = []; - // entries(), not values(): the map key is the page id, and for the queries - // that reach a file by title it is the only place the id appears. - for (const [key, page] of Object.entries(pages)) { - const info = page.imageinfo?.[0]; - // Only use actual photos (JPEG/PNG), skip SVGs and PDFs - const mime = info?.mime || ''; - if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue; - const meta = info.extmetadata; - const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null); - out.push({ - // iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it — - // info.url is the full-resolution original (multi-megapixel camera exports). - photoUrl: info.thumburl ?? info.url, - attribution: stripWikiMarkup(meta?.Artist?.value), - license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value), - licenseUrl: meta?.LicenseUrl?.value?.trim() || null, - sourceUrl: info.descriptionurl || null, - pageId: pageId && pageId > 0 ? pageId : null, - title: page.title ?? null, - width: info.width ?? null, - height: info.height ?? null, - descriptors: [ - stripWikiMarkup(meta?.ObjectName?.value), - stripWikiMarkup(meta?.ImageDescription?.value), - stripWikiMarkup(meta?.Categories?.value), - ] - .filter(Boolean) - .join(' | ') || null, - }); - if (out.length >= limit) break; - } - return out; - } - - /** - * Lead paragraph of a wiki article, from Wikivoyage first and Wikipedia after. - * - * Wikivoyage is the travel sibling: same MediaWiki API, same CC BY-SA, but it - * describes a place for someone about to go there, where Wikipedia opens with - * area in square kilometres and pronunciation. Both are resolved from the OSM - * `wikipedia` tag — guessing the article from the place name lands on the - * wrong one for every ambiguous name, so no tag means no description rather - * than a confident description of somewhere else. - */ - async fetchWikiExtract( - wikipediaTag: string | null | undefined, - ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { - const parsed = parseWikipediaTag(wikipediaTag); - if (!parsed) return null; - - // Two sentences, not three: this sits next to a form, and a fourth line of - // prose pushes the pictures out of view. - const params = new URLSearchParams({ - action: 'query', - format: 'json', - titles: parsed.title, - prop: 'extracts', - exintro: '1', - explaintext: '1', - exsentences: '2', - redirects: '1', - }); - - for (const host of ['wikivoyage', 'wikipedia'] as const) { - const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title); - if (hit) return hit; - } - return null; - } - - /** - * The lead paragraph of one named article on one named wiki. - * - * Split out from `fetchWikiExtract` because the article is not always found - * through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag - * at all (Berlin Hauptbahnhof is exactly that), and then the title comes from - * the item's sitelinks instead. - */ - async fetchWikiExtractFor( - host: 'wikivoyage' | 'wikipedia', - lang: string, - title: string, - signal?: AbortSignal, - ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { - if (!lang || !title) return null; - // Two sentences, not three: this sits next to a form, and a fourth line of - // prose pushes the pictures out of view. - const params = new URLSearchParams({ - action: 'query', - format: 'json', - titles: title, - prop: 'extracts', - exintro: '1', - explaintext: '1', - exsentences: '2', - redirects: '1', - }); - try { - const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) return null; - const data = (await res.json()) as { - query?: { pages?: Record }; - }; - for (const page of Object.values(data.query?.pages ?? {})) { - const text = page.extract?.trim(); - // A missing article comes back as a page with no extract, not a 404. - if (!text) continue; - const resolved = page.title ?? title; - return { - text, - sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`, - source: host, - }; - } - return null; - } catch { - return null; - } - } - - /** - * Which articles a Wikidata item is linked to, for the wikis we care about. - * - * The way to an article when a place has a Wikidata id but no `wikipedia` - * tag — which is most of them, because mappers add one or the other. One - * request, a few hundred bytes. - */ - async fetchWikidataSitelinks( - wikidataId: string, - sites: string[], - signal?: AbortSignal, - ): Promise> { - const qid = wikidataId.trim(); - if (!/^Q\d+$/.test(qid) || sites.length === 0) return {}; - const params = new URLSearchParams({ - action: 'wbgetentities', - props: 'sitelinks', - ids: qid, - sitefilter: sites.join('|'), - format: 'json', - }); - try { - const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS), - }); - if (!res.ok) return {}; - const data = (await res.json()) as { - entities?: Record }>; - }; - const out: Record = {}; - for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) { - if (link?.title) out[site] = link.title; - } - return out; - } catch { - return {}; - } - } - - /** - * The pictures Wikidata records for a place, best first. - * - * By far the most accurate source there is: a person chose each of these to - * represent this exact object, where a coordinate search only knows what was - * photographed nearby. Wikidata also keeps them apart by what they show, so - * asking for more than P18 buys genuine variety rather than another frame of - * the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a - * night shot, a panorama and a winter view, all curated. - * - * Two calls total whatever the item holds: one for the claims, one batch for - * the file metadata. - */ - async fetchWikidataCandidates( - wikidataId: string, - limit = 5, - ): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> { - const empty = { candidates: [], commonsCategory: null }; - const qid = wikidataId.trim(); - if (!/^Q\d+$/.test(qid)) return empty; - try { - const res = await fetch( - `https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`, - { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, - ); - if (!res.ok) return empty; - const data = (await res.json()) as { entities?: Record }; - const claims = data.entities?.[qid]?.claims; - if (!claims) return empty; - - const fileNames = wikidataImageClaims(claims, limit); - const commonsCategory = claimValue(claims.P373) ?? null; - if (fileNames.length === 0) return { candidates: [], commonsCategory }; - - const byTitle = await this.fetchCommonsFilesByName(fileNames); - // Back into the order Wikidata implied, which the batch response loses. - const candidates = fileNames.map((name) => byTitle.get(normalizeFileTitle(name))).filter((c): c is CommonsCandidate => !!c); - return { candidates, commonsCategory }; - } catch { - return empty; - } - } - - /** - * Metadata for a list of Commons files, in one request. - * - * `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia - * lead image often names a file that has since been renamed, and without it - * the API answers with a `missing` page and the picture disappears silently. - * Keyed by normalised title so callers can restore their own ordering. - */ - async fetchCommonsFilesByName(fileNames: string[]): Promise> { - const out = new Map(); - const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`)); - if (titles.length === 0) return out; - - const params = new URLSearchParams({ - action: 'query', - format: 'json', - titles: titles.join('|'), - redirects: '1', - prop: 'imageinfo', - iiprop: 'url|extmetadata|mime|size', - iiurlwidth: '400', - }); - try { - const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) return out; - const data = (await res.json()) as { - query?: { - pages?: Record; - normalized?: { from: string; to: string }[]; - redirects?: { from: string; to: string }[]; - }; - }; - // The API renames titles twice on the way in (normalisation, then - // redirects), so walk the chain back to what the caller asked for. - const aliases = new Map(); - for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) { - aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from)); - } - const resolveOriginal = (title: string): string => { - let key = normalizeFileTitle(title); - for (let hop = 0; hop < 4; hop++) { - const previous = aliases.get(key); - if (!previous || previous === key) break; - key = previous; - } - return key; - }; - - for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) { - if (!candidate.title) continue; - out.set(resolveOriginal(candidate.title), candidate); - // Also reachable under its own name, for callers that already resolved. - out.set(normalizeFileTitle(candidate.title), candidate); - } - return out; - } catch { - return out; - } - } - - /** - * The lead image a wiki article picked for a place. - * - * Only the file NAME is taken from here; the bytes and the licence come from - * the same Commons batch as everything else. The thumbnail URL the API offers - * alongside it carries no attribution, and a picture we cannot credit is a - * picture we cannot show. - */ - async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise { - const parsed = parseWikipediaTag(wikipediaTag); - if (!parsed) return null; - const params = new URLSearchParams({ - action: 'query', - format: 'json', - titles: parsed.title, - prop: 'pageimages', - piprop: 'name', - redirects: '1', - }); - for (const host of ['wikivoyage', 'wikipedia'] as const) { - try { - const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) continue; - const data = (await res.json()) as { - query?: { pages?: Record }; - }; - for (const page of Object.values(data.query?.pages ?? {})) { - if (page.pageimage) return page.pageimage; - } - } catch { - /* try the next wiki */ - } - } - return null; - } - - /** - * Commons images from a category, which is the set of pictures OF a place. - * - * Preferred over the coordinate search wherever a place carries a - * `wikimedia_commons` tag: geosearch around a city centre returns statues and - * passers-by, while the category of a restaurant returns the restaurant. - */ - async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise { - const name = normalizeCategoryName(category); - if (!name) return []; - // Overfetch: the ranker throws away survey imagery, diagrams and repeats, - // and it can only do that from a pool bigger than the strip. - const poolSize = String(Math.max(1, Math.min(limit * 3, 20))); - - // `generator=search` first. `categorymembers` orders by sort key, i.e. - // alphabetically by file name, which is not a quality signal in any - // direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation, - // a marathon photo and six near-identical press shots, and - // "Category:Hamburg Airport" with a noise map and a terminal layout. The - // search index at least ranks by how well a file matches its category. - const search = new URLSearchParams({ - action: 'query', - format: 'json', - generator: 'search', - gsrsearch: `incategory:"${name}" filetype:bitmap`, - gsrnamespace: '6', - gsrlimit: poolSize, - prop: 'imageinfo', - iiprop: 'url|extmetadata|mime|size', - iiurlwidth: '400', - }); - const members = new URLSearchParams({ - action: 'query', - format: 'json', - generator: 'categorymembers', - gcmtitle: `Category:${name}`, - gcmtype: 'file', - gcmlimit: poolSize, - prop: 'imageinfo', - iiprop: 'url|extmetadata|mime|size', - iiurlwidth: '400', - }); - - for (const params of [search, members]) { - try { - const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) continue; - const data = (await res.json()) as { query?: { pages?: Record } }; - const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize)); - if (hits.length) return hits; - } catch { - /* fall through to the second strategy */ - } - } - return []; - } - // ── Search places (Google or Nominatim fallback) ─────────────────────────── async searchPlaces( @@ -1971,8 +1142,7 @@ export class MapsService { let providerFailed = false; const fetchPromise = (async (): Promise<{ attribution: string | null } | null> => { - await acquirePhotoFetchSlot(); - try { + return withPhotoFetchSlot(async () => { const apiKey = await this.getMapsKey(userId); // Coordinate-based Wikipedia/Wikimedia lookup. Used for coordinate-only @@ -1980,30 +1150,11 @@ export class MapsService { // so a place added via search still gets a marker image when Google returns // nothing. Returns null (without marking an error) so the caller decides. const fetchWikimediaFallback = async (): Promise<{ attribution: string | null } | null> => { - if (Number.isNaN(lat) || Number.isNaN(lng)) return null; + const outcome = await this.wiki.downloadPhoto(lat, lng, name); + if (outcome.kind === 'failed') providerFailed = true; + if (outcome.kind !== 'photo') return null; try { - const wiki = await this.fetchWikimediaPhoto(lat, lng, name); - if (!wiki) return null; - // Follow redirects manually so each hop (the image URL can 3xx to a CDN - // host) is re-validated against the SSRF guard, not just the first URL. - const imgRes = await safeFetchFollow( - wiki.photoUrl, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true }, - ); - if (!imgRes.ok) { - providerFailed = true; - return null; - } - if (exceedsDeclaredLength(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES)) { - discardBody(imgRes); - providerFailed = true; - return null; - } - const { bytes, truncated } = await readCapped(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES); - if (truncated || bytes.byteLength === 0) { - providerFailed = true; - return null; - } - const cached = await this.photoCache.put(placeId, bytes, wiki.attribution); + const cached = await this.photoCache.put(placeId, outcome.bytes, outcome.attribution); return { attribution: cached.attribution }; } catch { providerFailed = true; @@ -2046,9 +1197,7 @@ export class MapsService { await this.photoCache.markError(placeId, providerFailed ? 'provider-error' : 'no-photo'); return null; - } finally { - releasePhotoFetchSlot(); - } + }); })(); this.photoCache.setInFlight(placeId, fetchPromise); diff --git a/server/src/nest/maps/photo-fetch-slot.ts b/server/src/nest/maps/photo-fetch-slot.ts new file mode 100644 index 000000000..21f3ac4e9 --- /dev/null +++ b/server/src/nest/maps/photo-fetch-slot.ts @@ -0,0 +1,41 @@ +// ── Concurrency limiter for outbound photo fetches ─────────────────────────── +// Caps simultaneous Wikimedia/Google photo requests so a bulk import of hundreds +// of places cannot monopolise the event loop or trigger external API rate limits. +// Module-scoped ON PURPOSE (permissions-cache precedent): every consumer — the +// marker photo in MapsService and the enrichment column — must share one limiter. + +const MAX_CONCURRENT_PHOTO_FETCHES = 5; +let photoFetchActive = 0; +const photoFetchQueue: Array<() => void> = []; + +function acquirePhotoFetchSlot(): Promise { + if (photoFetchActive < MAX_CONCURRENT_PHOTO_FETCHES) { + photoFetchActive++; + return Promise.resolve(); + } + return new Promise((resolve) => photoFetchQueue.push(resolve)); +} + +function releasePhotoFetchSlot(): void { + const next = photoFetchQueue.shift(); + if (next) { + next(); + } else { + photoFetchActive--; + } +} + +/** + * Runs an outbound photo fetch under the shared slot limit. Exported so the + * enrichment module queues behind the same five slots — a picker that grabs + * three images per selected place would otherwise sail past a cap the rest of + * the app respects. + */ +export async function withPhotoFetchSlot(fn: () => Promise): Promise { + await acquirePhotoFetchSlot(); + try { + return await fn(); + } finally { + releasePhotoFetchSlot(); + } +} diff --git a/server/src/nest/maps/providers/wikimedia.client.ts b/server/src/nest/maps/providers/wikimedia.client.ts new file mode 100644 index 000000000..c0c42ac89 --- /dev/null +++ b/server/src/nest/maps/providers/wikimedia.client.ts @@ -0,0 +1,830 @@ +/** + * Wikipedia, Wikivoyage, Wikidata and Wikimedia Commons: descriptions, + * curated pictures and brand logos for a place. + * + * Everything here is free and keyless, and every call carries TREK's user + * agent and a deadline, because the enrichment column and the marker photo + * answer a live dialog: a slow wiki is dropped rather than waited out. Commons + * images are mostly CC BY / CC BY-SA, so a picture is only ever handed back + * with the metadata needed to credit it. + * + * MapsService (the marker photo, the brand logo route) and the enrichment + * column both ask through this client. It never imports MapsService: + * providers sit below the orchestrator (lint:boundaries holds that). + */ +import { Injectable } from '@nestjs/common'; +import { Jimp } from 'jimp'; +import { safeFetchFollow } from '../../../utils/ssrfGuard'; +import { discardBody, exceedsDeclaredLength, readCapped } from '../../../utils/cappedFetch'; +import { UA, stripWikiMarkup, parseWikipediaTag } from '../maps.helpers'; + +interface WikidataSnak { + mainsnak?: { datavalue?: { value?: string } }; + rank?: 'preferred' | 'normal' | 'deprecated'; +} +type WikidataClaims = Record; + +/** + * Wikidata properties that name a picture of a place, in the order we want them. + * + * P18 is the representative image. The rest exist because a station or a + * monument is not one view: asking for the interior, the night shot, the + * panorama and the aerial gives a picker four genuinely different pictures + * instead of four frames of the same façade. + */ +const WIKIDATA_IMAGE_PROPERTIES = [ + 'P18', // image + 'P5775', // interior view + 'P3451', // night view + 'P8592', // aerial view + 'P4291', // panoramic view + 'P5252', // winter view + 'P948', // Wikivoyage banner +] as const; + +/** First non-empty string value of a claim list. */ +function claimValue(snaks: WikidataSnak[] | undefined): string | null { + for (const snak of snaks ?? []) { + const value = snak.mainsnak?.datavalue?.value; + if (typeof value === 'string' && value.trim()) return value.trim(); + } + return null; +} + +/** + * File names from an item's picture properties, best first. + * + * Within P18 the statement rank decides: an item with several images marks one + * `preferred`, and that is the one an editor considers representative. The API + * returns statements in edit order, not rank order, so taking `[0]` picks + * whichever was added first — for the Brandenburg Gate that is a coin toss + * between the morning shot and a wide overview. + */ +function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] { + const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const; + const names: string[] = []; + const seen = new Set(); + + for (const property of WIKIDATA_IMAGE_PROPERTIES) { + const snaks = [...(claims[property] ?? [])] + .filter((snak) => snak.rank !== 'deprecated') + .sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1)); + for (const snak of snaks) { + const value = snak.mainsnak?.datavalue?.value; + if (typeof value !== 'string' || !value.trim()) continue; + const key = normalizeFileTitle(value); + if (seen.has(key)) continue; + seen.add(key); + names.push(value.trim()); + if (names.length >= limit) return names; + } + } + return names; +} + +/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */ +function normalizeFileTitle(title: string): string { + return title.replace(/^File:/i, '').replaceAll('_', ' ').trim().toLowerCase(); +} + +/** + * A bare Commons category name out of whatever an OSM `wikimedia_commons` tag + * holds. + * + * The tag is free text and mappers put three different things in it: a bare + * name, a prefixed `Category:…`, or — against the wiki's own advice — a single + * `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`, + * which matches nothing and fell through to the coordinate search without a + * word. Localised prefixes (`Kategorie:`) appear too. + */ +function normalizeCategoryName(raw: string): string | null { + const value = raw.trim(); + if (!value) return null; + // A file name is not a category, and there is nothing sensible to derive. + if (/^(file|datei|image|bild)\s*:/i.test(value)) return null; + return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null; +} + +interface WikiCommonsPage { + pageid?: number; + title?: string; + imageinfo?: { + url?: string; + thumburl?: string; + mime?: string; + width?: number; + height?: number; + /** The file description page — where the full licence terms live. */ + descriptionurl?: string; + extmetadata?: { + Artist?: { value?: string }; + LicenseShortName?: { value?: string }; + LicenseUrl?: { value?: string }; + UsageTerms?: { value?: string }; + /** Used to spot survey imagery and diagrams, which are not pictures of a place. */ + Categories?: { value?: string }; + ObjectName?: { value?: string }; + ImageDescription?: { value?: string }; + }; + }[]; +} + +/** + * One Commons image with everything needed to credit it. Commons is mostly + * CC BY / CC BY-SA, so a candidate that cannot be attributed is not usable in a + * picker — the fields are nullable because Commons metadata is user-maintained + * and genuinely incomplete on some files, not because they are optional to show. + */ +export interface CommonsCandidate { + photoUrl: string; + attribution: string | null; + license: string | null; + licenseUrl: string | null; + sourceUrl: string | null; + /** + * Commons page id — the only stable identity a file has across the four ways + * we reach it. The thumbnail URL is not: the same file comes back from + * commons.wikimedia.org and from a language Wikipedia with different query + * strings, so deduplicating on the URL silently lets the same picture through + * twice. It is also what keys the cached bytes, so it must survive. + */ + pageId: number | null; + /** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */ + title: string | null; + width: number | null; + height: number | null; + /** Free text used to reject survey imagery, floor plans and logos. */ + descriptors: string | null; +} + +/** Wikimedia is normally well under a second, but a cold TLS handshake from a + * fresh container has been seen at eight. Enrichment answers a live dialog, so + * a slow provider is dropped rather than waited out. */ +const WIKI_TIMEOUT_MS = 6000; + +/** + * The sitelinks call sits at the front of the description chain (identity, + * then sitelinks, then the extract), so it gets the same tight deadline the + * OSM identity lookup in front of it has. + */ +const SITELINKS_TIMEOUT_MS = 2500; + +// ── Brand logos ────────────────────────────────────────────────────────────── +// +// A road trip corridor is mostly chains — Shell, Aral, JET — and the brand is the +// fastest thing to recognise on a map. OSM carries `brand:wikidata` on most of them, +// Wikidata carries the logo (P154), and Commons serves it. +// +// The bytes are proxied rather than linked so the browser never talks to Wikimedia: +// one self-hosted instance asking for a handful of logos is a very different egress +// profile from every visitor's browser announcing which petrol stations they are +// looking at. The cache is in memory on purpose — a GET must not write to the DB, and +// there are only so many fuel brands. +const BRAND_LOGO_CACHE_TTL_MS = 24 * 60 * 60 * 1000; +const BRAND_LOGO_CACHE_MAX = 300; +/** Well past any logo; a Commons original can be a multi-megabyte SVG or print-res PNG. */ +const BRAND_LOGO_MAX_BYTES = 512 * 1024; +/** A place photo is a full-size image rather than a 128px mark, so its own ceiling. */ +const WIKIMEDIA_PHOTO_MAX_BYTES = 8 * 1024 * 1024; +const BRAND_LOGO_WIDTH = 128; +/** The square the logo is centred in, and the breathing room around it. */ +const BRAND_LOGO_CANVAS = 96; +const BRAND_LOGO_PADDING = 8; +const WIKIDATA_ID_RE = /^Q[1-9][0-9]{0,11}$/; + +export interface BrandLogo { + bytes: Buffer; + contentType: string; +} + +/** What a marker-photo download from Wikimedia came back with. */ +export type WikimediaPhotoOutcome = + | { kind: 'photo'; bytes: Buffer; attribution: string | null } + /** Nothing photographed here (or no coordinates to look with): a plain miss. */ + | { kind: 'none' } + /** Wikimedia refused, timed out or sent something unusable: remembered only briefly. */ + | { kind: 'failed' }; + +/** + * Puts a logo on a background it can actually be seen against. + * + * Half the fuel brands ship a white wordmark with a transparent background — + * TotalEnergies, Esso and JET among them — and on the white pill the marker used to + * draw they were invisible. Which background is right is a property of the image, not + * of the brand, so it is measured: the mean brightness of the pixels that are actually + * opaque decides between a white and a near-black backdrop. + * + * Returns the flattened PNG, or null when the bytes cannot be read at all — the pin + * then keeps its category icon, which is a fine outcome. + */ +async function flattenBrandLogo(bytes: Buffer): Promise { + try { + const image = await Jimp.read(bytes); + const width = image.bitmap.width; + const height = image.bitmap.height; + if (!width || !height) return null; + + let sum = 0; + let opaque = 0; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const idx = (y * width + x) * 4; + const alpha = image.bitmap.data[idx + 3]; + // Half-transparent pixels are anti-aliasing, not the mark itself. + if (alpha < 128) continue; + const r = image.bitmap.data[idx]; + const g = image.bitmap.data[idx + 1]; + const b = image.bitmap.data[idx + 2]; + sum += 0.2126 * r + 0.7152 * g + 0.0722 * b; + opaque++; + } + } + // A logo with nothing opaque in it is not a logo. + if (opaque === 0) return null; + + // Square, with the logo scaled to fit inside it. Most brand marks are wide wordmarks + // — TotalEnergies is three times as wide as it is tall — and a round pin showing the + // middle of one is unreadable. Fitting it into a square means the pin can crop to a + // circle without ever cutting the mark itself. + const scale = Math.min( + (BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / width, + (BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / height, + ); + // Never upscale: a 16-pixel favicon blown up to 96 looks worse than a small one. + if (scale < 1) image.scale(scale); + + const light = sum / opaque > 150; + const canvas = new Jimp({ + width: BRAND_LOGO_CANVAS, + height: BRAND_LOGO_CANVAS, + color: light ? 0x111827ff : 0xffffffff, + }); + canvas.composite( + image, + Math.round((BRAND_LOGO_CANVAS - image.bitmap.width) / 2), + Math.round((BRAND_LOGO_CANVAS - image.bitmap.height) / 2), + ); + const out = await canvas.getBuffer('image/png'); + return { bytes: Buffer.from(out), contentType: 'image/png' }; + } catch { + return null; + } +} + +@Injectable() +export class WikimediaClient { + /** Brand id → logo bytes, or null for "asked, has none". Insertion-ordered, so the + * oldest entry is the one evicted when it fills up. */ + private readonly brandLogoCache = new Map(); + + /** + * The logo of a brand, by its Wikidata id, as bytes. + * + * Two hops: Wikidata says which Commons file is the logo (property P154), Commons + * serves a thumbnail of it. Both answers are cached, including "this brand has no + * logo" — otherwise every map pan would ask Wikidata about the same supermarket + * chain again. Returns null whenever anything is missing or unreadable; a marker + * without a logo falls back to its category icon, which is a fine outcome. + */ + async brandLogo(wikidataId: string): Promise { + if (!WIKIDATA_ID_RE.test(wikidataId)) return null; + + const cached = this.brandLogoCache.get(wikidataId); + if (cached && Date.now() - cached.at < BRAND_LOGO_CACHE_TTL_MS) return cached.logo; + + const remember = (logo: BrandLogo | null): BrandLogo | null => { + if (this.brandLogoCache.size >= BRAND_LOGO_CACHE_MAX) { + const oldest = this.brandLogoCache.keys().next().value; + if (oldest !== undefined) this.brandLogoCache.delete(oldest); + } + this.brandLogoCache.set(wikidataId, { at: Date.now(), logo }); + return logo; + }; + + try { + const params = new URLSearchParams({ + action: 'wbgetclaims', + entity: wikidataId, + property: 'P154', + format: 'json', + }); + const claimRes = await fetch(`https://www.wikidata.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!claimRes.ok) return remember(null); + const claims = (await claimRes.json()) as { + claims?: { P154?: { mainsnak?: { datavalue?: { value?: unknown } } }[] }; + }; + const file = claims.claims?.P154?.[0]?.mainsnak?.datavalue?.value; + if (typeof file !== 'string' || !file.trim()) return remember(null); + + // Special:FilePath renders a thumbnail at the width asked for and redirects to + // the CDN, so each hop is re-checked by the guard rather than trusted. + const url = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(file)}?width=${BRAND_LOGO_WIDTH}`; + // The same six seconds the Wikidata hop above allows. Without a deadline + // this waited on undici's five-minute default, holding a request context + // and a socket per stalled logo while the pin sat on its fallback icon. + const imgRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true }); + if (!imgRes.ok) return remember(null); + + if (exceedsDeclaredLength(imgRes, BRAND_LOGO_MAX_BYTES)) { + discardBody(imgRes); + return remember(null); + } + // Streamed rather than buffered whole: a chunked answer declares no length, + // so the post-check only ever ran after the bytes were already in memory. + const { bytes, truncated } = await readCapped(imgRes, BRAND_LOGO_MAX_BYTES); + if (truncated || bytes.byteLength === 0) return remember(null); + + const contentType = imgRes.headers.get('content-type') ?? ''; + if (!contentType.startsWith('image/')) return remember(null); + + return remember(await flattenBrandLogo(bytes)); + } catch { + // An SSRF refusal on a redirect hop and a network failure end the same + // way: no logo, remembered, and the pin keeps its category icon. + return remember(null); + } + } + + /** + * The marker photo for a place without a Google one, as bytes and a credit. + * + * Used for coordinate-only (right-click) places and as the fallback when a + * Google place yields no photo. The image URL can 3xx to a CDN host, so the + * download follows redirects through the SSRF guard, every hop re-validated. + */ + async downloadPhoto(lat: number, lng: number, name?: string): Promise { + if (Number.isNaN(lat) || Number.isNaN(lng)) return { kind: 'none' }; + try { + const wiki = await this.fetchWikimediaPhoto(lat, lng, name); + if (!wiki) return { kind: 'none' }; + const imgRes = await safeFetchFollow(wiki.photoUrl, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true }); + if (!imgRes.ok) return { kind: 'failed' }; + if (exceedsDeclaredLength(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES)) { + discardBody(imgRes); + return { kind: 'failed' }; + } + const { bytes, truncated } = await readCapped(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES); + if (truncated || bytes.byteLength === 0) return { kind: 'failed' }; + return { kind: 'photo', bytes, attribution: wiki.attribution }; + } catch { + return { kind: 'failed' }; + } + } + + // ── Wikimedia Commons photo lookup ───────────────────────────────────────── + + async fetchWikimediaPhoto(lat: number, lng: number, name?: string): Promise<{ photoUrl: string; attribution: string | null } | null> { + // Strategy 1: Search Wikipedia for the place name -> get the article image + if (name) { + try { + const searchParams = new URLSearchParams({ + action: 'query', + format: 'json', + titles: name, + prop: 'pageimages', + piprop: 'thumbnail', + pithumbsize: '400', + pilimit: '1', + redirects: '1', + }); + const res = await fetch(`https://en.wikipedia.org/w/api.php?${searchParams}`, { + headers: { 'User-Agent': UA }, + // This runs inside one of the few shared photo-fetch slots; a stalled + // answer would hold it for everyone. + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (res.ok) { + const data = (await res.json()) as { query?: { pages?: Record } }; + const pages = data.query?.pages; + if (pages) { + for (const page of Object.values(pages)) { + if (page.thumbnail?.source) { + return { photoUrl: page.thumbnail.source, attribution: 'Wikipedia' }; + } + } + } + } + } catch { + /* fall through to geosearch */ + } + } + + // Strategy 2: Wikimedia Commons geosearch by coordinates + const candidates = await this.fetchCommonsCandidates(lat, lng, 5); + const first = candidates[0]; + return first ? { photoUrl: first.photoUrl, attribution: first.attribution } : null; + } + + /** + * Anything photographed near a coordinate, licence metadata included. The + * bottom rung of the picture ladder, and the only one with no claim on the + * subject at all. + * + * geosearch already returns up to `limit` files in a single request, so asking + * for a whole strip costs the same as asking for one picture. Callers that only + * want a single image (fetchWikimediaPhoto, and through it the marker photo) + * take the first entry. + * + * 60 metres rather than the 300 it used to be. Three hundred metres in a city + * centre is a whole block: the town hall, the church and the underground + * entrance are all inside it, and every one of them outranks the doner shop + * we were actually asked about. Sixty is roughly "the same building and its + * neighbours", which is the widest a picture can be taken and still plausibly + * show the place. It does not fix the real problem, which is that nobody + * photographed the shop, but it stops the wrong answer from being confident. + */ + async fetchCommonsCandidates(lat: number, lng: number, limit = 5): Promise { + const params = new URLSearchParams({ + action: 'query', + format: 'json', + generator: 'geosearch', + ggsprimary: 'all', + ggsnamespace: '6', + ggsradius: '60', + ggscoord: `${lat}|${lng}`, + // Deliberately more than the caller asked for. Around anything worth + // visiting the first few hits are survey tiles, passers-by and the + // building next door; the ranker needs a pool to reject from, and + // geosearch charges the same for one result as for twenty. + ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))), + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + try { + const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + // A hanging provider must not hold the whole enrichment request open; + // no pictures is a fine answer, a request that never returns is not. + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) return []; + const data = (await res.json()) as { query?: { pages?: Record } }; + // Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets + // what it always got; the enrichment column ranks before it cuts. + return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit'))); + } catch { + return []; + } + } + + /** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */ + private toCommonsCandidates(pages: Record | undefined, limit: number): CommonsCandidate[] { + if (!pages) return []; + const out: CommonsCandidate[] = []; + // entries(), not values(): the map key is the page id, and for the queries + // that reach a file by title it is the only place the id appears. + for (const [key, page] of Object.entries(pages)) { + const info = page.imageinfo?.[0]; + // Only use actual photos (JPEG/PNG), skip SVGs and PDFs + const mime = info?.mime || ''; + if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue; + const meta = info.extmetadata; + const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null); + out.push({ + // iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it — + // info.url is the full-resolution original (multi-megapixel camera exports). + photoUrl: info.thumburl ?? info.url, + attribution: stripWikiMarkup(meta?.Artist?.value), + license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value), + licenseUrl: meta?.LicenseUrl?.value?.trim() || null, + sourceUrl: info.descriptionurl || null, + pageId: pageId && pageId > 0 ? pageId : null, + title: page.title ?? null, + width: info.width ?? null, + height: info.height ?? null, + descriptors: + [ + stripWikiMarkup(meta?.ObjectName?.value), + stripWikiMarkup(meta?.ImageDescription?.value), + stripWikiMarkup(meta?.Categories?.value), + ] + .filter(Boolean) + .join(' | ') || null, + }); + if (out.length >= limit) break; + } + return out; + } + + /** + * Lead paragraph of a wiki article, from Wikivoyage first and Wikipedia after. + * + * Wikivoyage is the travel sibling: same MediaWiki API, same CC BY-SA, but it + * describes a place for someone about to go there, where Wikipedia opens with + * area in square kilometres and pronunciation. Both are resolved from the OSM + * `wikipedia` tag — guessing the article from the place name lands on the + * wrong one for every ambiguous name, so no tag means no description rather + * than a confident description of somewhere else. + */ + async fetchWikiExtract( + wikipediaTag: string | null | undefined, + ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { + const parsed = parseWikipediaTag(wikipediaTag); + if (!parsed) return null; + for (const host of ['wikivoyage', 'wikipedia'] as const) { + const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title); + if (hit) return hit; + } + return null; + } + + /** + * The lead paragraph of one named article on one named wiki. + * + * Split out from `fetchWikiExtract` because the article is not always found + * through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag + * at all (Berlin Hauptbahnhof is exactly that), and then the title comes from + * the item's sitelinks instead. + */ + async fetchWikiExtractFor( + host: 'wikivoyage' | 'wikipedia', + lang: string, + title: string, + signal?: AbortSignal, + ): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> { + if (!lang || !title) return null; + // Two sentences, not three: this sits next to a form, and a fourth line of + // prose pushes the pictures out of view. + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: title, + prop: 'extracts', + exintro: '1', + explaintext: '1', + exsentences: '2', + redirects: '1', + }); + try { + const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) return null; + const data = (await res.json()) as { + query?: { pages?: Record }; + }; + for (const page of Object.values(data.query?.pages ?? {})) { + const text = page.extract?.trim(); + // A missing article comes back as a page with no extract, not a 404. + if (!text) continue; + const resolved = page.title ?? title; + return { + text, + sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`, + source: host, + }; + } + return null; + } catch { + return null; + } + } + + /** + * Which articles a Wikidata item is linked to, for the wikis we care about. + * + * The way to an article when a place has a Wikidata id but no `wikipedia` + * tag — which is most of them, because mappers add one or the other. One + * request, a few hundred bytes. + */ + async fetchWikidataSitelinks(wikidataId: string, sites: string[], signal?: AbortSignal): Promise> { + const qid = wikidataId.trim(); + if (!/^Q\d+$/.test(qid) || sites.length === 0) return {}; + const params = new URLSearchParams({ + action: 'wbgetentities', + props: 'sitelinks', + ids: qid, + sitefilter: sites.join('|'), + format: 'json', + }); + try { + const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: signal ?? AbortSignal.timeout(SITELINKS_TIMEOUT_MS), + }); + if (!res.ok) return {}; + const data = (await res.json()) as { + entities?: Record }>; + }; + const out: Record = {}; + for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) { + if (link?.title) out[site] = link.title; + } + return out; + } catch { + return {}; + } + } + + /** + * The pictures Wikidata records for a place, best first. + * + * By far the most accurate source there is: a person chose each of these to + * represent this exact object, where a coordinate search only knows what was + * photographed nearby. Wikidata also keeps them apart by what they show, so + * asking for more than P18 buys genuine variety rather than another frame of + * the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a + * night shot, a panorama and a winter view, all curated. + * + * Two calls total whatever the item holds: one for the claims, one batch for + * the file metadata. + */ + async fetchWikidataCandidates( + wikidataId: string, + limit = 5, + ): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> { + const empty = { candidates: [], commonsCategory: null }; + const qid = wikidataId.trim(); + if (!/^Q\d+$/.test(qid)) return empty; + try { + const res = await fetch(`https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) return empty; + const data = (await res.json()) as { entities?: Record }; + const claims = data.entities?.[qid]?.claims; + if (!claims) return empty; + + const fileNames = wikidataImageClaims(claims, limit); + const commonsCategory = claimValue(claims.P373) ?? null; + if (fileNames.length === 0) return { candidates: [], commonsCategory }; + + const byTitle = await this.fetchCommonsFilesByName(fileNames); + // Back into the order Wikidata implied, which the batch response loses. + const candidates = fileNames + .map((name) => byTitle.get(normalizeFileTitle(name))) + .filter((c): c is CommonsCandidate => !!c); + return { candidates, commonsCategory }; + } catch { + return empty; + } + } + + /** + * Metadata for a list of Commons files, in one request. + * + * `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia + * lead image often names a file that has since been renamed, and without it + * the API answers with a `missing` page and the picture disappears silently. + * Keyed by normalised title so callers can restore their own ordering. + */ + async fetchCommonsFilesByName(fileNames: string[]): Promise> { + const out = new Map(); + const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`)); + if (titles.length === 0) return out; + + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: titles.join('|'), + redirects: '1', + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + try { + const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) return out; + const data = (await res.json()) as { + query?: { + pages?: Record; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + }; + }; + // The API renames titles twice on the way in (normalisation, then + // redirects), so walk the chain back to what the caller asked for. + const aliases = new Map(); + for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) { + aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from)); + } + const resolveOriginal = (title: string): string => { + let key = normalizeFileTitle(title); + for (let hop = 0; hop < 4; hop++) { + const previous = aliases.get(key); + if (!previous || previous === key) break; + key = previous; + } + return key; + }; + + for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) { + if (!candidate.title) continue; + out.set(resolveOriginal(candidate.title), candidate); + // Also reachable under its own name, for callers that already resolved. + out.set(normalizeFileTitle(candidate.title), candidate); + } + return out; + } catch { + return out; + } + } + + /** + * The lead image a wiki article picked for a place. + * + * Only the file NAME is taken from here; the bytes and the licence come from + * the same Commons batch as everything else. The thumbnail URL the API offers + * alongside it carries no attribution, and a picture we cannot credit is a + * picture we cannot show. + */ + async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise { + const parsed = parseWikipediaTag(wikipediaTag); + if (!parsed) return null; + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: parsed.title, + prop: 'pageimages', + piprop: 'name', + redirects: '1', + }); + for (const host of ['wikivoyage', 'wikipedia'] as const) { + try { + const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) continue; + const data = (await res.json()) as { + query?: { pages?: Record }; + }; + for (const page of Object.values(data.query?.pages ?? {})) { + if (page.pageimage) return page.pageimage; + } + } catch { + /* try the next wiki */ + } + } + return null; + } + + /** + * Commons images from a category, which is the set of pictures OF a place. + * + * Preferred over the coordinate search wherever a place carries a + * `wikimedia_commons` tag: geosearch around a city centre returns statues and + * passers-by, while the category of a restaurant returns the restaurant. + */ + async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise { + const name = normalizeCategoryName(category); + if (!name) return []; + // Overfetch: the ranker throws away survey imagery, diagrams and repeats, + // and it can only do that from a pool bigger than the strip. + const poolSize = String(Math.max(1, Math.min(limit * 3, 20))); + + // `generator=search` first. `categorymembers` orders by sort key, i.e. + // alphabetically by file name, which is not a quality signal in any + // direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation, + // a marathon photo and six near-identical press shots, and + // "Category:Hamburg Airport" with a noise map and a terminal layout. The + // search index at least ranks by how well a file matches its category. + const search = new URLSearchParams({ + action: 'query', + format: 'json', + generator: 'search', + gsrsearch: `incategory:"${name}" filetype:bitmap`, + gsrnamespace: '6', + gsrlimit: poolSize, + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + const members = new URLSearchParams({ + action: 'query', + format: 'json', + generator: 'categorymembers', + gcmtitle: `Category:${name}`, + gcmtype: 'file', + gcmlimit: poolSize, + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + + for (const params of [search, members]) { + try { + const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) continue; + const data = (await res.json()) as { query?: { pages?: Record } }; + const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize)); + if (hits.length) return hits; + } catch { + /* fall through to the second strategy */ + } + } + return []; + } +} diff --git a/server/src/nest/place-enrichment/place-enrichment.service.ts b/server/src/nest/place-enrichment/place-enrichment.service.ts index 31309d21b..8706db1b9 100644 --- a/server/src/nest/place-enrichment/place-enrichment.service.ts +++ b/server/src/nest/place-enrichment/place-enrichment.service.ts @@ -29,6 +29,7 @@ import { buildOsmDetails, isGooglePlaceId, parseWikipediaTag, rankCommonsCandida import { trekPlacesById } from '../maps/trek-places.client'; import { GooglePlacesClient } from '../maps/providers/google-places.provider'; import { OsmClient } from '../maps/providers/osm.client'; +import { WikimediaClient } from '../maps/providers/wikimedia.client'; import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service'; import { readAppSetting } from '../common/app-settings.registry'; @@ -327,6 +328,7 @@ export class PlaceEnrichmentService { private readonly photoCache: PlacePhotoCacheService, private readonly googlePlaces: GooglePlacesClient, private readonly osm: OsmClient, + private readonly wiki: WikimediaClient, ) {} /** @@ -490,21 +492,21 @@ export class PlaceEnrichmentService { let categoryName = identity.wikimedia_commons; if (wikidata) { - const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP); + const fromWikidata = await this.wiki.fetchWikidataCandidates(wikidata, COMMONS_CAP); push(commonsPool, fromWikidata.candidates, 'wikidata'); categoryName ??= fromWikidata.commonsCategory; } if (commonsPool.length < COMMONS_CAP && wikipedia) { - const leadName = await this.maps.fetchWikiLeadImageName(wikipedia); + const leadName = await this.wiki.fetchWikiLeadImageName(wikipedia); if (leadName) { - const byName = await this.maps.fetchCommonsFilesByName([leadName]); + const byName = await this.wiki.fetchCommonsFilesByName([leadName]); push(commonsPool, [...byName.values()], 'wikipedia'); } } if (commonsPool.length < COMMONS_CAP && categoryName) { - push(commonsPool, await this.maps.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category'); + push(commonsPool, await this.wiki.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category'); } // Two is the bar: one curated picture plus the nearby noise reads worse @@ -521,7 +523,7 @@ export class PlaceEnrichmentService { const skipNearby = nearbyWouldMislead(details) || nearbyWouldMislead(identity.osmTags); const nearbyPending = curated < 2 && !skipNearby - ? this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP) + ? this.wiki.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP) : Promise.resolve([]); const googleRefs = await googlePending; @@ -787,21 +789,21 @@ export class PlaceEnrichmentService { { site: 'enwikivoyage', host: 'wikivoyage', lang: 'en' }, { site: 'enwiki', host: 'wikipedia', lang: 'en' }, ]; - const sitelinks = await this.maps.fetchWikidataSitelinks( + const sitelinks = await this.wiki.fetchWikidataSitelinks( identity.wikidata, wanted.map((w) => w.site), ); for (const { site, host, lang: hostLang } of wanted) { const title = sitelinks[site]; if (!title) continue; - const hit = await this.maps.fetchWikiExtractFor(host, hostLang, title); + const hit = await this.wiki.fetchWikiExtractFor(host, hostLang, title); if (hit) return hit; } } // No Wikidata id, or its sitelinks led nowhere: fall back to the tag, which // names an article directly. - return identity.wikipedia ? this.maps.fetchWikiExtract(identity.wikipedia) : null; + return identity.wikipedia ? this.wiki.fetchWikiExtract(identity.wikipedia) : null; } /** diff --git a/server/tests/e2e/place-enrichment.e2e.test.ts b/server/tests/e2e/place-enrichment.e2e.test.ts index 3b210b17e..3ded2be9d 100644 --- a/server/tests/e2e/place-enrichment.e2e.test.ts +++ b/server/tests/e2e/place-enrichment.e2e.test.ts @@ -33,6 +33,7 @@ import { PlaceEnrichmentModule } from '../../src/nest/place-enrichment/place-enr import { candidateKey } from '../../src/nest/place-enrichment/place-enrichment.service'; import { MapsService } from '../../src/nest/maps/maps.service'; import { OsmClient } from '../../src/nest/maps/providers/osm.client'; +import { WikimediaClient } from '../../src/nest/maps/providers/wikimedia.client'; import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import { RateLimitService } from '../../src/nest/common/rate-limit.service'; import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; @@ -51,6 +52,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => let app: Awaited>; let maps: MapsService; let osm: OsmClient; + let wiki: WikimediaClient; let orm: TestOrm; async function build() { @@ -73,6 +75,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => maps = app.get(MapsService); osm = app.get(OsmClient); + wiki = app.get(WikimediaClient); const photoCache = app.get(PlacePhotoCacheService); vi.spyOn(photoCache, 'get').mockReturnValue(null); vi.spyOn(photoCache, 'put').mockImplementation(async (key: string, _b: Buffer, attribution: string | null) => ({ @@ -93,19 +96,19 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => await app.get(RateLimitService).reset('place_enrichment'); // spyOn hands back the same spy on a second call, so the call counts carry // over between tests unless they are cleared explicitly. - vi.spyOn(maps, 'fetchCommonsCandidates').mockClear().mockResolvedValue([]); - vi.spyOn(maps, 'fetchWikiExtract').mockClear().mockResolvedValue(null); - vi.spyOn(maps, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]); + vi.spyOn(wiki, 'fetchCommonsCandidates').mockClear().mockResolvedValue([]); + vi.spyOn(wiki, 'fetchWikiExtract').mockClear().mockResolvedValue(null); + vi.spyOn(wiki, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]); vi.spyOn(maps, 'details').mockClear().mockResolvedValue({ place: null }); // Every provider the service can reach has to be stubbed here, not just the // ones a given case cares about: anything left open goes out over the real // network from CI, which is both slow and rude to the provider. vi.spyOn(osm, 'resolveOsmIdentity').mockClear().mockResolvedValue(null); - vi.spyOn(maps, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({}); - vi.spyOn(maps, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null); - vi.spyOn(maps, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null }); - vi.spyOn(maps, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null); - vi.spyOn(maps, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map()); + vi.spyOn(wiki, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({}); + vi.spyOn(wiki, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null); + vi.spyOn(wiki, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null }); + vi.spyOn(wiki, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null); + vi.spyOn(wiki, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map()); }); it('401 without a session cookie', async () => { @@ -137,7 +140,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => }); it('200 with a Commons candidate carrying its licence', async () => { - vi.spyOn(maps, 'fetchCommonsCandidates').mockResolvedValue([ + vi.spyOn(wiki, 'fetchCommonsCandidates').mockResolvedValue([ { photoUrl: 'https://commons.org/thumb.jpg', attribution: 'Alice', @@ -175,13 +178,13 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => expect(res.status).toBe(200); expect(res.body).toEqual({ photos: [], description: null, facts: [], disabled: true }); - expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled(); + expect(wiki.fetchCommonsCandidates).not.toHaveBeenCalled(); }); it('stays enabled on an instance that has never seen the setting', async () => { const res = await request(server).post('/api/maps/enrichment').set('Cookie', sessionCookie(1)).send(BODY); expect(res.body.disabled).toBeUndefined(); - expect(maps.fetchCommonsCandidates).toHaveBeenCalled(); + expect(wiki.fetchCommonsCandidates).toHaveBeenCalled(); }); it('serves the second request for the same place from the row cache', async () => { diff --git a/server/tests/helpers/maps-service.ts b/server/tests/helpers/maps-service.ts index ba460a756..988e5d8bd 100644 --- a/server/tests/helpers/maps-service.ts +++ b/server/tests/helpers/maps-service.ts @@ -1,6 +1,7 @@ import { MapsService } from '../../src/nest/maps/maps.service'; import { GooglePlacesClient } from '../../src/nest/maps/providers/google-places.provider'; import { OsmClient } from '../../src/nest/maps/providers/osm.client'; +import { WikimediaClient } from '../../src/nest/maps/providers/wikimedia.client'; import type { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import type { AppSettingsRepository } from '../../src/db/repositories/AppSettings.repository'; import type { UsersRepository } from '../../src/db/repositories/Users.repository'; @@ -14,6 +15,7 @@ export interface MapsParts { svc: MapsService; google: GooglePlacesClient; osm: OsmClient; + wiki: WikimediaClient; } /** @@ -32,8 +34,9 @@ export function buildMapsParts( ): MapsParts { const google = new GooglePlacesClient(googleQuota); const osm = new OsmClient(); - const svc = new MapsService(photoCache, appSettings, users, placeDetailsCache, places, googleQuota, google, osm); - return { svc, google, osm }; + const wiki = new WikimediaClient(); + const svc = new MapsService(photoCache, appSettings, users, placeDetailsCache, places, googleQuota, google, osm, wiki); + return { svc, google, osm, wiki }; } export function buildMapsService( diff --git a/server/tests/unit/nest/maps.curated-images.test.ts b/server/tests/unit/nest/maps.curated-images.test.ts index 3993025ff..63cbabb2e 100644 --- a/server/tests/unit/nest/maps.curated-images.test.ts +++ b/server/tests/unit/nest/maps.curated-images.test.ts @@ -23,10 +23,9 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({ SsrfBlockedError: class extends Error {}, })); -import { buildMapsService } from '../../helpers/maps-service'; -import { noGoogleQuota } from '../../helpers/google-quota'; +import { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; -const svcOf = () => buildMapsService({} as never, {} as never, {} as never, {} as never, {} as never, noGoogleQuota); +const svcOf = () => new WikimediaClient(); const filePage = (over: Record = {}) => ({ pageid: 4711, diff --git a/server/tests/unit/nest/maps.identity.test.ts b/server/tests/unit/nest/maps.identity.test.ts index e5a0a7e25..7581529f7 100644 --- a/server/tests/unit/nest/maps.identity.test.ts +++ b/server/tests/unit/nest/maps.identity.test.ts @@ -20,11 +20,10 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({ })); import { OsmClient } from '../../../src/nest/maps/providers/osm.client'; -import { buildMapsService } from '../../helpers/maps-service'; -import { noGoogleQuota } from '../../helpers/google-quota'; +import { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; import { toWikiLang, haversineMetres, namesOverlap } from '../../../src/nest/maps/maps.helpers'; -const svcOf = () => buildMapsService({} as never, {} as never, {} as never, {} as never, {} as never, noGoogleQuota); +const wikiOf = () => new WikimediaClient(); const osmOf = () => new OsmClient(); // The Brandenburg Gate and the underground station named after it, 250m apart. @@ -201,7 +200,7 @@ describe('fetchWikidataSitelinks', () => { }); vi.stubGlobal('fetch', fetchMock); - const out = await svcOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']); + const out = await wikiOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']); expect(decodeURIComponent(String(fetchMock.mock.calls[0][0]))).toContain('sitefilter=dewikivoyage|dewiki|enwiki'); expect(out).toEqual({ dewiki: 'Berlin Hauptbahnhof' }); @@ -211,16 +210,16 @@ describe('fetchWikidataSitelinks', () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - expect(await svcOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({}); - expect(await svcOf().fetchWikidataSitelinks('Q1', [])).toEqual({}); + expect(await wikiOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({}); + expect(await wikiOf().fetchWikidataSitelinks('Q1', [])).toEqual({}); expect(fetchMock).not.toHaveBeenCalled(); }); it('MAPS-173: yields an empty map on a bad response or a throw', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); - expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); + expect(await wikiOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); - expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); + expect(await wikiOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({}); }); }); diff --git a/server/tests/unit/nest/maps.service.test.ts b/server/tests/unit/nest/maps.service.test.ts index 245e7389e..4a73329ef 100644 --- a/server/tests/unit/nest/maps.service.test.ts +++ b/server/tests/unit/nest/maps.service.test.ts @@ -232,7 +232,7 @@ const placesStub = { // through a repository stub that flows into the SAME mockDbGet/mockDbRun/ // mockInstanceGet/mockProviderGet functions, so they keep firing exactly as // they did for the legacy module. -const { svc, google, osm } = buildMapsParts(photoCacheStub, appSettingsStub, usersStub, placeDetailsCacheStub, placesStub, noGoogleQuota); +const { svc, google, osm, wiki } = buildMapsParts(photoCacheStub, appSettingsStub, usersStub, placeDetailsCacheStub, placesStub, noGoogleQuota); /** * Switch the TREK Places index off for one case. @@ -1094,7 +1094,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Eiffel Tower'); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Eiffel Tower'); expect(result).toBeDefined(); expect(result!.photoUrl).toBe('https://example.com/thumb.jpg'); expect(result!.attribution).toBe('Wikipedia'); @@ -1117,7 +1117,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }; vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiResponse).mockResolvedValueOnce(commonsResponse)); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); expect(result).toBeDefined(); expect(result!.photoUrl).toBe('https://commons.org/img.jpg'); expect(result!.attribution).toBe('Alice'); @@ -1145,7 +1145,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }; vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiResponse).mockResolvedValueOnce(commonsResponse)); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); expect(result).toBeDefined(); expect(result!.photoUrl).toBe('https://commons.org/thumb-400.jpg'); expect(result!.attribution).toBe('Alice'); @@ -1159,7 +1159,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { json: async () => ({ query: { pages: {} } }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result).toBeNull(); }); @@ -1170,7 +1170,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { json: async () => ({ query: { pages: {} } }), }); vi.stubGlobal('fetch', fetchMock); - await svc.fetchWikimediaPhoto(48.8, 2.3); + await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(fetchMock).toHaveBeenCalledTimes(1); }); @@ -1191,7 +1191,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { 'fetch', vi.fn().mockRejectedValueOnce(new Error('Wikipedia network error')).mockResolvedValueOnce(commonsResponse), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); expect(result).toBeDefined(); expect(result!.photoUrl).toBe('https://commons.org/fallback.jpg'); // no Artist in extmetadata -> attribution null @@ -1219,7 +1219,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }; vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiNotOk).mockResolvedValueOnce(commonsResponse)); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place'); expect(result).toBeDefined(); // HTML tags stripped from attribution expect(result!.attribution).toBe('Bob'); @@ -1227,7 +1227,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { it('MAPS-037e: returns null when Commons geosearch returns not ok', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false })); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result).toBeNull(); }); @@ -1239,13 +1239,13 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { json: async () => ({ query: {} }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result).toBeNull(); }); it('MAPS-037g: returns null when Commons fetch throws', async () => { vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('Commons network error'))); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result).toBeNull(); }); @@ -1265,7 +1265,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result).toBeNull(); }); @@ -1291,7 +1291,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result!.photoUrl).toBe('https://commons.org/photo.png'); expect(result!.attribution).toBe('Carol'); }); @@ -1312,7 +1312,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => { }), }), ); - const result = await svc.fetchWikimediaPhoto(48.8, 2.3); + const result = await wiki.fetchWikimediaPhoto(48.8, 2.3); expect(result!.attribution).toBeNull(); }); }); @@ -2946,7 +2946,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { json: async () => ({ query: { pages: { '1': page(), '2': page({ thumburl: 'https://commons.org/t2.jpg' }) } } }), }), ); - const out = await svc.fetchCommonsCandidates(48.8, 2.3, 5); + const out = await wiki.fetchCommonsCandidates(48.8, 2.3, 5); expect(out).toHaveLength(2); expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/thumb.jpg', @@ -2976,7 +2976,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { }), }), ); - const out = await svc.fetchCommonsCandidates(48.8, 2.3); + const out = await wiki.fetchCommonsCandidates(48.8, 2.3); expect(out[0].license).toBe('Public domain'); expect(out[0].attribution).toBeNull(); expect(out[0].licenseUrl).toBeNull(); @@ -3000,7 +3000,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { }), }), ); - const out = await svc.fetchCommonsCandidates(48.8, 2.3); + const out = await wiki.fetchCommonsCandidates(48.8, 2.3); expect(out).toHaveLength(1); expect(out[0].photoUrl).toBe('https://commons.org/original.jpg'); }); @@ -3016,25 +3016,25 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { // survey tiles and the building next door, and the ranker can only reject // from a pool. Geosearch bills the same for one result as for twenty, so // the whole pool comes back and gets cut after ranking, not before. - expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3); + expect(await wiki.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3); expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=8'); - await svc.fetchCommonsCandidates(48.8, 2.3, 0); + await wiki.fetchCommonsCandidates(48.8, 2.3, 0); expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=8'); - await svc.fetchCommonsCandidates(48.8, 2.3, 99); + await wiki.fetchCommonsCandidates(48.8, 2.3, 99); expect(String(fetchMock.mock.calls[2][0])).toContain('ggslimit=20'); }); it('MAPS-121: returns an empty list on a bad response, missing pages or a throw', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); - expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]); + expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]); vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: {} }) })); - expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]); + expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]); vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); - expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]); + expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]); }); }); @@ -3049,7 +3049,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => { const fetchMock = vi.fn().mockResolvedValue(page('Museum Ludwig', ' Ein Museum in Köln. ')); vi.stubGlobal('fetch', fetchMock); - const out = await svc.fetchWikiExtract('de:Museum Ludwig'); + const out = await wiki.fetchWikiExtract('de:Museum Ludwig'); expect(out).toEqual({ text: 'Ein Museum in Köln.', sourceUrl: 'https://de.wikivoyage.org/wiki/Museum%20Ludwig', @@ -3067,7 +3067,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => { .mockResolvedValueOnce(page('Museum Ludwig', 'Das Museum Ludwig ist ein Museum.')); vi.stubGlobal('fetch', fetchMock); - const out = await svc.fetchWikiExtract('de:Museum Ludwig'); + const out = await wiki.fetchWikiExtract('de:Museum Ludwig'); expect(out).toMatchObject({ source: 'wikipedia' }); expect(String(fetchMock.mock.calls[1][0])).toContain('https://de.wikipedia.org/w/api.php'); }); @@ -3076,36 +3076,36 @@ describe('fetchWikiExtract (fetch stubbed)', () => { const fetchMock = vi.fn().mockResolvedValue(page('X', 'Kurz.')); vi.stubGlobal('fetch', fetchMock); - await svc.fetchWikiExtract('de:X'); + await wiki.fetchWikiExtract('de:X'); expect(String(fetchMock.mock.calls[0][0])).toContain('exsentences=2'); }); it('MAPS-123: prefers the resolved title so a redirect links to where it landed', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue(page('Eiffel Tower', 'A tower.'))); - const out = await svc.fetchWikiExtract('en:Eiffelturm'); + const out = await wiki.fetchWikiExtract('en:Eiffelturm'); expect(out!.sourceUrl).toBe('https://en.wikivoyage.org/wiki/Eiffel%20Tower'); }); it('MAPS-124: returns null without calling out when the tag has no language', async () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - expect(await svc.fetchWikiExtract('Museum Ludwig')).toBeNull(); - expect(await svc.fetchWikiExtract(null)).toBeNull(); + expect(await wiki.fetchWikiExtract('Museum Ludwig')).toBeNull(); + expect(await wiki.fetchWikiExtract(null)).toBeNull(); expect(fetchMock).not.toHaveBeenCalled(); }); it('MAPS-125: treats a miss on both wikis as no description', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue(noArticle)); - expect(await svc.fetchWikiExtract('de:X')).toBeNull(); + expect(await wiki.fetchWikiExtract('de:X')).toBeNull(); vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({}) })); - expect(await svc.fetchWikiExtract('de:X')).toBeNull(); + expect(await wiki.fetchWikiExtract('de:X')).toBeNull(); vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); - expect(await svc.fetchWikiExtract('de:X')).toBeNull(); + expect(await wiki.fetchWikiExtract('de:X')).toBeNull(); vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); - expect(await svc.fetchWikiExtract('de:X')).toBeNull(); + expect(await wiki.fetchWikiExtract('de:X')).toBeNull(); }); it('MAPS-125b: still tries Wikipedia after Wikivoyage threw', async () => { @@ -3115,7 +3115,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => { .mockResolvedValueOnce(page('X', 'Ein Ort.')); vi.stubGlobal('fetch', fetchMock); - expect(await svc.fetchWikiExtract('de:X')).toMatchObject({ source: 'wikipedia' }); + expect(await wiki.fetchWikiExtract('de:X')).toMatchObject({ source: 'wikipedia' }); }); }); @@ -3143,7 +3143,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => { }); vi.stubGlobal('fetch', fetchMock); - const out = await svc.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3); + const out = await wiki.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3); expect(out).toHaveLength(1); expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/t.jpg', attribution: 'Alice', license: 'CC BY 4.0' }); // Ranked search, not the category listing: `categorymembers` orders by sort @@ -3157,7 +3157,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) }); vi.stubGlobal('fetch', fetchMock); - await svc.fetchCommonsCategoryCandidates('Museum Ludwig'); + await wiki.fetchCommonsCategoryCandidates('Museum Ludwig'); expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22'); // Empty search falls through to the category listing as a second chance. expect(String(fetchMock.mock.calls[1][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig'); @@ -3170,7 +3170,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - expect(await svc.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]); + expect(await wiki.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]); expect(fetchMock).not.toHaveBeenCalled(); }); @@ -3178,16 +3178,16 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) }); vi.stubGlobal('fetch', fetchMock); - await svc.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig'); + await wiki.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig'); expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22'); }); it('MAPS-125e: yields nothing on an error response or a throw', async () => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); - expect(await svc.fetchCommonsCategoryCandidates('Category:X')).toEqual([]); + expect(await wiki.fetchCommonsCategoryCandidates('Category:X')).toEqual([]); vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); - expect(await svc.fetchCommonsCategoryCandidates('Category:X')).toEqual([]); + expect(await wiki.fetchCommonsCategoryCandidates('Category:X')).toEqual([]); }); }); diff --git a/server/tests/unit/nest/place-enrichment.brand.test.ts b/server/tests/unit/nest/place-enrichment.brand.test.ts index 992f6c6a3..a51901b90 100644 --- a/server/tests/unit/nest/place-enrichment.brand.test.ts +++ b/server/tests/unit/nest/place-enrichment.brand.test.ts @@ -34,10 +34,11 @@ import { readBrandIdentity } from '../../../src/nest/maps/maps.service'; import type { MapsService } from '../../../src/nest/maps/maps.service'; import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider'; import type { OsmClient } from '../../../src/nest/maps/providers/osm.client'; +import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; /** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */ type Seams = { [K in keyof T]: T[K] }; -type EnrichmentSeams = Seams & Seams & Seams; +type EnrichmentSeams = Seams & Seams & Seams & Seams; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository'; import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository'; @@ -90,7 +91,7 @@ function cacheRepoStub(): PlaceDetailsCacheRepository { return { findEntry: mockFindEntry, upsertEntry: mockUpsertEntry } as unknown as PlaceDetailsCacheRepository; } -const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient); +const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient); /** A place whose own identity is empty but that belongs to a chain. */ const branchOfAChain = (over: Partial> = {}) => diff --git a/server/tests/unit/nest/place-enrichment.description.test.ts b/server/tests/unit/nest/place-enrichment.description.test.ts index 771e38b90..7aa236774 100644 --- a/server/tests/unit/nest/place-enrichment.description.test.ts +++ b/server/tests/unit/nest/place-enrichment.description.test.ts @@ -38,10 +38,11 @@ import { PlaceEnrichmentService } from '../../../src/nest/place-enrichment/place import type { MapsService } from '../../../src/nest/maps/maps.service'; import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider'; import type { OsmClient } from '../../../src/nest/maps/providers/osm.client'; +import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; /** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */ type Seams = { [K in keyof T]: T[K] }; -type EnrichmentSeams = Seams & Seams & Seams; +type EnrichmentSeams = Seams & Seams & Seams & Seams; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository'; import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository'; @@ -89,7 +90,7 @@ function cacheRepoStub(): PlaceDetailsCacheRepository { return { findEntry: mockFindEntry, upsertEntry: mockUpsertEntry } as unknown as PlaceDetailsCacheRepository; } -const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient); +const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient); beforeEach(() => { mockGetValue.mockReset(); diff --git a/server/tests/unit/nest/place-enrichment.service.test.ts b/server/tests/unit/nest/place-enrichment.service.test.ts index 18024eb12..770d82fc9 100644 --- a/server/tests/unit/nest/place-enrichment.service.test.ts +++ b/server/tests/unit/nest/place-enrichment.service.test.ts @@ -53,10 +53,11 @@ import { import type { MapsService } from '../../../src/nest/maps/maps.service'; import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider'; import type { OsmClient } from '../../../src/nest/maps/providers/osm.client'; +import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client'; /** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */ type Seams = { [K in keyof T]: T[K] }; -type EnrichmentSeams = Seams & Seams & Seams; +type EnrichmentSeams = Seams & Seams & Seams & Seams; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository'; import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository'; @@ -109,7 +110,7 @@ function cacheStub(over: Record = {}) { } function make(maps: EnrichmentSeams, cache: PlacePhotoCacheService) { - return new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cache, maps as unknown as GooglePlacesClient, maps as unknown as OsmClient); + return new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cache, maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient); } let candidateSeq = 0;