diff --git a/client/src/components/Planner/PlaceDetailsColumn.tsx b/client/src/components/Planner/PlaceDetailsColumn.tsx index 31ae4b2d8..64e2a45a8 100644 --- a/client/src/components/Planner/PlaceDetailsColumn.tsx +++ b/client/src/components/Planner/PlaceDetailsColumn.tsx @@ -72,9 +72,16 @@ function writeSession(key: string, value: MapsPlaceEnrichmentResult): void { } } +/** + * Bumped with the server's CACHE_VERSION. sessionStorage outlives a deploy, so + * without it the tab that was open while the fix shipped keeps replaying the + * answer the fix was about — and reports it as still broken. + */ +const ENRICH_CACHE_V = 3 + function cacheKeyFor(selection: PlaceDetailsSelection, language: string): string { const id = selection.placeId || `coords:${selection.lat}:${selection.lng}` - return `enrich_${id}_${language}` + return `enrich_v${ENRICH_CACHE_V}_${id}_${language}` } type LoadState = 'idle' | 'loading' | 'ready' | 'error' diff --git a/server/src/nest/maps/maps.helpers.ts b/server/src/nest/maps/maps.helpers.ts index 4a0ef2727..b9ea5a1fa 100644 --- a/server/src/nest/maps/maps.helpers.ts +++ b/server/src/nest/maps/maps.helpers.ts @@ -405,3 +405,121 @@ export const OSM_PLACE_ID = /^(?:node|way|relation):/i; export function isGooglePlaceId(placeId: string): boolean { return !NON_GOOGLE_PLACE_ID.test(placeId); } + +// ── Ranking Commons candidates ─────────────────────────────────────────────── + +/** + * A candidate as the ranker needs to see it. Structurally the subset of + * `CommonsCandidate` it reads, kept local so the pure function does not have to + * import from the service that imports this file. + */ +export interface RankableCommonsCandidate { + pageId: number | null; + title: string | null; + attribution: string | null; + width: number | null; + height: number | null; + descriptors: string | null; + photoUrl: string; +} + +/** + * Commons files that are near a place but are not a picture of it. + * + * Orthophotos are the worst offender by volume: German states upload their + * aerial survey tiles to Commons with coordinates, so a geosearch around any + * airport returns the runway as seen from a plane at 3000m — four of them for + * Hamburg, one per survey year. The rest are documents that happen to live in a + * place's category: noise maps, floor plans, terminal layouts, logos, coats of + * arms. + */ +const NOT_A_PHOTO_OF_THE_PLACE = + /orthophoto|orthofoto|sommerbefliegung|luftbildkarte|dop\d+|l[äa]rmkarte|noise map|floor ?plan|grundriss|lageplan|layout|diagram|schematic|blueprint|coat of arms|wappen|\blogo\b|flag of/i; + +/** + * Strips what makes two frames of the same burst look like different files, so + * they collapse onto one stem: trailing counters, camera dumps, dates, and the + * `- 17` / `(2)` suffixes press sets use. + */ +function seriesStem(title: string): string { + return title + .replace(/^File:/i, '') + .replace(/\.[a-z0-9]+$/i, '') + .toLowerCase() + // 20260614 100717648 HDR — a camera dump, all from the same minute + .replace(/\b\d{8}[ _-]\d{6,9}\b/g, ' ') + .replace(/\b(19|20)\d{2}\b/g, ' ') + .replace(/[ _-]+\(?\d{1,4}\)?$/g, '') + .replace(/[^a-z]+/g, ' ') + .trim(); +} + +/** + * Picks the pictures worth showing out of whatever the providers returned. + * + * Everything here was written against a real failing case. Deduplication is by + * page id because the same file reaches us under different thumbnail URLs. The + * series rule exists because Commons categories are full of bursts — the four + * `Hauptbahnhof Berlin interior 0807..0810` shots that filled the picker were + * one person walking across a concourse. The author cap is the same problem + * seen from the other side: one contributor who documented a station thoroughly + * should not supply the whole strip. + * + * `perAuthor` is the one knob: coordinate search gets 1, because there being no + * evidence the pictures are even of the right subject makes variety the only + * defence. Curated sources get 2 — someone already vouched that these depict + * the place. + */ +export function rankCommonsCandidates( + candidates: T[], + limit: number, + opts: { perAuthor?: number } = {}, +): T[] { + const perAuthor = opts.perAuthor ?? 2; + const seenPages = new Set(); + const seenUrls = new Set(); + const seenStems = new Set(); + const authorCounts = new Map(); + const out: T[] = []; + + for (const candidate of candidates) { + if (out.length >= limit) break; + + if (candidate.pageId != null) { + if (seenPages.has(candidate.pageId)) continue; + } else if (seenUrls.has(candidate.photoUrl)) { + // No page id (an older payload, or a provider that does not report one): + // the URL is a weaker key but better than letting an exact repeat through. + continue; + } + + const haystack = `${candidate.title ?? ''} ${candidate.descriptors ?? ''}`; + if (NOT_A_PHOTO_OF_THE_PLACE.test(haystack)) continue; + + const { width, height } = candidate; + if (width && height) { + // Survey tiles are square and enormous; nothing photographed by hand is. + if (width === height && width >= 3000) continue; + const ratio = Math.max(width / height, height / width); + // Panorama strips and tall banners crop to nothing in a square tile. + if (ratio > 4) continue; + } + + const stem = candidate.title ? seriesStem(candidate.title) : ''; + if (stem && seenStems.has(stem)) continue; + + const author = (candidate.attribution ?? '').trim().toLowerCase(); + if (author) { + const used = authorCounts.get(author) ?? 0; + if (used >= perAuthor) continue; + authorCounts.set(author, used + 1); + } + + if (candidate.pageId != null) seenPages.add(candidate.pageId); + seenUrls.add(candidate.photoUrl); + if (stem) seenStems.add(stem); + out.push(candidate); + } + + return out; +} diff --git a/server/src/nest/maps/maps.service.ts b/server/src/nest/maps/maps.service.ts index e0355a7a0..2aa581357 100644 --- a/server/src/nest/maps/maps.service.ts +++ b/server/src/nest/maps/maps.service.ts @@ -54,6 +54,124 @@ interface NominatimResult { display_name?: string; lat: string; lon: string; + extratags?: Record | null; +} + +/** + * The keys that say which encyclopaedia entry, Wikidata item and Commons + * category describe a place. Everything the enrichment column shows beyond + * coordinates hangs off one of these. + * + * Bare keys only. OSM also carries `brand:wikidata` / `brand:wikipedia`, and + * following those means a branch of a chain gets the chain's article and the + * chain's logo — for "L'Osteria Rostock" you would confidently describe + * L'Osteria the company. That is the exact failure the tag-only rule was + * written to avoid, so do not "improve" this by falling back to brand:*. + */ +const WIKI_IDENTITY_TAGS = ['wikipedia', 'wikidata', 'wikimedia_commons'] as const; + +export interface WikiIdentity { + wikipedia: string | null; + wikidata: string | null; + wikimedia_commons: string | null; +} + +interface WikidataSnak { + mainsnak?: { datavalue?: { value?: string } }; + rank?: 'preferred' | 'normal' | 'deprecated'; +} +type WikidataClaims = Record; + +/** + * Wikidata properties that name a picture of a place, in the order we want them. + * + * P18 is the representative image. The rest exist because a station or a + * monument is not one view: asking for the interior, the night shot, the + * panorama and the aerial gives a picker four genuinely different pictures + * instead of four frames of the same façade. + */ +const WIKIDATA_IMAGE_PROPERTIES = [ + 'P18', // image + 'P5775', // interior view + 'P3451', // night view + 'P8592', // aerial view + 'P4291', // panoramic view + 'P5252', // winter view + 'P948', // Wikivoyage banner +] as const; + +/** First non-empty string value of a claim list. */ +function claimValue(snaks: WikidataSnak[] | undefined): string | null { + for (const snak of snaks ?? []) { + const value = snak.mainsnak?.datavalue?.value; + if (typeof value === 'string' && value.trim()) return value.trim(); + } + return null; +} + +/** + * File names from an item's picture properties, best first. + * + * Within P18 the statement rank decides: an item with several images marks one + * `preferred`, and that is the one an editor considers representative. The API + * returns statements in edit order, not rank order, so taking `[0]` picks + * whichever was added first — for the Brandenburg Gate that is a coin toss + * between the morning shot and a wide overview. + */ +function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] { + const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const; + const names: string[] = []; + const seen = new Set(); + + for (const property of WIKIDATA_IMAGE_PROPERTIES) { + const snaks = [...(claims[property] ?? [])] + .filter((snak) => snak.rank !== 'deprecated') + .sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1)); + for (const snak of snaks) { + const value = snak.mainsnak?.datavalue?.value; + if (typeof value !== 'string' || !value.trim()) continue; + const key = normalizeFileTitle(value); + if (seen.has(key)) continue; + seen.add(key); + names.push(value.trim()); + if (names.length >= limit) return names; + } + } + return names; +} + +/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */ +function normalizeFileTitle(title: string): string { + return title.replace(/^File:/i, '').replace(/_/g, ' ').trim().toLowerCase(); +} + +/** + * A bare Commons category name out of whatever an OSM `wikimedia_commons` tag + * holds. + * + * The tag is free text and mappers put three different things in it: a bare + * name, a prefixed `Category:…`, or — against the wiki's own advice — a single + * `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`, + * which matches nothing and fell through to the coordinate search without a + * word. Localised prefixes (`Kategorie:`) appear too. + */ +function normalizeCategoryName(raw: string): string | null { + const value = raw.trim(); + if (!value) return null; + // A file name is not a category, and there is nothing sensible to derive. + if (/^(file|datei|image|bild)\s*:/i.test(value)) return null; + return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null; +} + +/** Picks the three identity tags out of a Nominatim `extratags` blob. */ +export function readWikiIdentity(extratags: Record | null | undefined): WikiIdentity { + const out: WikiIdentity = { wikipedia: null, wikidata: null, wikimedia_commons: null }; + if (!extratags) return out; + for (const tag of WIKI_IDENTITY_TAGS) { + const value = extratags[tag]; + if (typeof value === 'string' && value.trim()) out[tag] = value.trim(); + } + return out; } interface OverpassElement { @@ -61,10 +179,14 @@ interface OverpassElement { } interface WikiCommonsPage { + pageid?: number; + title?: string; imageinfo?: { url?: string; thumburl?: string; mime?: string; + width?: number; + height?: number; /** The file description page — where the full licence terms live. */ descriptionurl?: string; extmetadata?: { @@ -72,6 +194,10 @@ interface WikiCommonsPage { LicenseShortName?: { value?: string }; LicenseUrl?: { value?: string }; UsageTerms?: { value?: string }; + /** Used to spot survey imagery and diagrams, which are not pictures of a place. */ + Categories?: { value?: string }; + ObjectName?: { value?: string }; + ImageDescription?: { value?: string }; }; }[]; } @@ -88,6 +214,20 @@ export interface CommonsCandidate { license: string | null; licenseUrl: string | null; sourceUrl: string | null; + /** + * Commons page id — the only stable identity a file has across the four ways + * we reach it. The thumbnail URL is not: the same file comes back from + * commons.wikimedia.org and from a language Wikipedia with different query + * strings, so deduplicating on the URL silently lets the same picture through + * twice. It is also what keys the cached bytes, so it must survive. + */ + pageId: number | null; + /** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */ + title: string | null; + width: number | null; + height: number | null; + /** Free text used to reject survey imagery, floor plans and logos. */ + descriptors: string | null; } interface GooglePlaceResult { @@ -372,6 +512,11 @@ export class MapsService { q: query, format: 'json', addressdetails: '1', + // Free, same request: this is where a place's wikidata/wikipedia/commons + // tags live. Without them the enrichment column can only fall back to + // "photos taken within 300m", which around a city centre is passers-by + // and the neighbouring building. + extratags: '1', limit: '10', 'accept-language': toApiLang(lang), }); @@ -402,6 +547,7 @@ export class MapsService { website: null, phone: null, source: 'openstreetmap', + ...readWikiIdentity(item.extratags), }; }); } @@ -417,11 +563,15 @@ export class MapsService { address: string; lat: number | null; lng: number | null; + extratags: Record | null; } | null> { const typePrefix = osmType.charAt(0).toUpperCase(); // N, W, R const params = new URLSearchParams({ osm_ids: `${typePrefix}${osmId}`, format: 'json', + // Overpass is the richer source but it is also the one that times out; + // whatever Nominatim already knows costs nothing extra here. + extratags: '1', 'accept-language': toApiLang(lang), }); try { @@ -439,6 +589,7 @@ export class MapsService { address: item.display_name || '', lat: Number.isFinite(lat) ? lat : null, lng: Number.isFinite(lng) ? lng : null, + extratags: item.extratags ?? null, }; } catch { return null; @@ -623,9 +774,13 @@ export class MapsService { ggsnamespace: '6', ggsradius: '300', ggscoord: `${lat}|${lng}`, - ggslimit: String(Math.max(1, Math.min(limit, 20))), + // Deliberately more than the caller asked for. Around anything worth + // visiting the first few hits are survey tiles, passers-by and the + // building next door; the ranker needs a pool to reject from, and + // geosearch charges the same for one result as for twenty. + ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))), prop: 'imageinfo', - iiprop: 'url|extmetadata|mime', + iiprop: 'url|extmetadata|mime|size', iiurlwidth: '400', }); try { @@ -637,25 +792,30 @@ export class MapsService { }); if (!res.ok) return []; const data = (await res.json()) as { query?: { pages?: Record } }; - return this.toCommonsCandidates(data.query?.pages, limit); + // Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets + // what it always got; the enrichment column ranks before it cuts. + return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit'))); } catch { return []; } } - /** Shared shaping for both Commons queries (by coordinate and by category). */ + /** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */ private toCommonsCandidates( pages: Record | undefined, limit: number, ): CommonsCandidate[] { if (!pages) return []; const out: CommonsCandidate[] = []; - for (const page of Object.values(pages)) { + // entries(), not values(): the map key is the page id, and for the queries + // that reach a file by title it is the only place the id appears. + for (const [key, page] of Object.entries(pages)) { const info = page.imageinfo?.[0]; // Only use actual photos (JPEG/PNG), skip SVGs and PDFs const mime = info?.mime || ''; if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue; const meta = info.extmetadata; + const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null); out.push({ // iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it — // info.url is the full-resolution original (multi-megapixel camera exports). @@ -664,6 +824,17 @@ export class MapsService { license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value), licenseUrl: meta?.LicenseUrl?.value?.trim() || null, sourceUrl: info.descriptionurl || null, + pageId: pageId && pageId > 0 ? pageId : null, + title: page.title ?? null, + width: info.width ?? null, + height: info.height ?? null, + descriptors: [ + stripWikiMarkup(meta?.ObjectName?.value), + stripWikiMarkup(meta?.ImageDescription?.value), + stripWikiMarkup(meta?.Categories?.value), + ] + .filter(Boolean) + .join(' | ') || null, }); if (out.length >= limit) break; } @@ -730,50 +901,150 @@ export class MapsService { } /** - * The one picture Wikidata records for a place (property P18). + * The pictures Wikidata records for a place, best first. * - * By far the most accurate source there is: someone chose this image to + * By far the most accurate source there is: a person chose each of these to * represent this exact object, where a coordinate search only knows what was - * photographed nearby. Restaurants and small venues carry a `wikidata` tag far - * more often than a Commons category, so this is also the source that reaches - * the places the others miss. + * photographed nearby. Wikidata also keeps them apart by what they show, so + * asking for more than P18 buys genuine variety rather than another frame of + * the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a + * night shot, a panorama and a winter view, all curated. + * + * Two calls total whatever the item holds: one for the claims, one batch for + * the file metadata. */ - async fetchWikidataImage(wikidataId: string): Promise { - if (!/^Q\d+$/.test(wikidataId.trim())) return null; + async fetchWikidataCandidates( + wikidataId: string, + limit = 5, + ): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> { + const empty = { candidates: [], commonsCategory: null }; + const qid = wikidataId.trim(); + if (!/^Q\d+$/.test(qid)) return empty; try { - const claims = await fetch( - `https://www.wikidata.org/w/api.php?action=wbgetclaims&entity=${wikidataId.trim()}&property=P18&format=json`, + const res = await fetch( + `https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, ); - if (!claims.ok) return null; - const data = (await claims.json()) as { - claims?: { P18?: { mainsnak?: { datavalue?: { value?: string } } }[] }; - }; - const fileName = data.claims?.P18?.[0]?.mainsnak?.datavalue?.value; - if (!fileName) return null; + if (!res.ok) return empty; + const data = (await res.json()) as { entities?: Record }; + const claims = data.entities?.[qid]?.claims; + if (!claims) return empty; - // Second hop for the URL and, more importantly, the licence — the claim - // itself carries only a file name. - const params = new URLSearchParams({ - action: 'query', - format: 'json', - titles: `File:${fileName}`, - prop: 'imageinfo', - iiprop: 'url|extmetadata|mime', - iiurlwidth: '400', - }); - const file = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { + const fileNames = wikidataImageClaims(claims, limit); + const commonsCategory = claimValue(claims.P373) ?? null; + if (fileNames.length === 0) return { candidates: [], commonsCategory }; + + const byTitle = await this.fetchCommonsFilesByName(fileNames); + // Back into the order Wikidata implied, which the batch response loses. + const candidates = fileNames.map((name) => byTitle.get(normalizeFileTitle(name))).filter((c): c is CommonsCandidate => !!c); + return { candidates, commonsCategory }; + } catch { + return empty; + } + } + + /** + * Metadata for a list of Commons files, in one request. + * + * `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia + * lead image often names a file that has since been renamed, and without it + * the API answers with a `missing` page and the picture disappears silently. + * Keyed by normalised title so callers can restore their own ordering. + */ + async fetchCommonsFilesByName(fileNames: string[]): Promise> { + const out = new Map(); + const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`)); + if (titles.length === 0) return out; + + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: titles.join('|'), + redirects: '1', + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + try { + const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), }); - if (!file.ok) return null; - const fileData = (await file.json()) as { query?: { pages?: Record } }; - return this.toCommonsCandidates(fileData.query?.pages, 1)[0] ?? null; + if (!res.ok) return out; + const data = (await res.json()) as { + query?: { + pages?: Record; + normalized?: { from: string; to: string }[]; + redirects?: { from: string; to: string }[]; + }; + }; + // The API renames titles twice on the way in (normalisation, then + // redirects), so walk the chain back to what the caller asked for. + const aliases = new Map(); + for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) { + aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from)); + } + const resolveOriginal = (title: string): string => { + let key = normalizeFileTitle(title); + for (let hop = 0; hop < 4; hop++) { + const previous = aliases.get(key); + if (!previous || previous === key) break; + key = previous; + } + return key; + }; + + for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) { + if (!candidate.title) continue; + out.set(resolveOriginal(candidate.title), candidate); + // Also reachable under its own name, for callers that already resolved. + out.set(normalizeFileTitle(candidate.title), candidate); + } + return out; } catch { - return null; + return out; } } + /** + * The lead image a wiki article picked for a place. + * + * Only the file NAME is taken from here; the bytes and the licence come from + * the same Commons batch as everything else. The thumbnail URL the API offers + * alongside it carries no attribution, and a picture we cannot credit is a + * picture we cannot show. + */ + async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise { + const parsed = parseWikipediaTag(wikipediaTag); + if (!parsed) return null; + const params = new URLSearchParams({ + action: 'query', + format: 'json', + titles: parsed.title, + prop: 'pageimages', + piprop: 'name', + redirects: '1', + }); + for (const host of ['wikivoyage', 'wikipedia'] as const) { + try { + const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) continue; + const data = (await res.json()) as { + query?: { pages?: Record }; + }; + for (const page of Object.values(data.query?.pages ?? {})) { + if (page.pageimage) return page.pageimage; + } + } catch { + /* try the next wiki */ + } + } + return null; + } + /** * Commons images from a category, which is the set of pictures OF a place. * @@ -782,29 +1053,56 @@ export class MapsService { * passers-by, while the category of a restaurant returns the restaurant. */ async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise { - const title = category.startsWith('Category:') ? category : `Category:${category}`; - const params = new URLSearchParams({ + const name = normalizeCategoryName(category); + if (!name) return []; + // Overfetch: the ranker throws away survey imagery, diagrams and repeats, + // and it can only do that from a pool bigger than the strip. + const poolSize = String(Math.max(1, Math.min(limit * 3, 20))); + + // `generator=search` first. `categorymembers` orders by sort key, i.e. + // alphabetically by file name, which is not a quality signal in any + // direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation, + // a marathon photo and six near-identical press shots, and + // "Category:Hamburg Airport" with a noise map and a terminal layout. The + // search index at least ranks by how well a file matches its category. + const search = new URLSearchParams({ + action: 'query', + format: 'json', + generator: 'search', + gsrsearch: `incategory:"${name}" filetype:bitmap`, + gsrnamespace: '6', + gsrlimit: poolSize, + prop: 'imageinfo', + iiprop: 'url|extmetadata|mime|size', + iiurlwidth: '400', + }); + const members = new URLSearchParams({ action: 'query', format: 'json', generator: 'categorymembers', - gcmtitle: title, + gcmtitle: `Category:${name}`, gcmtype: 'file', - gcmlimit: String(Math.max(1, Math.min(limit * 2, 20))), + gcmlimit: poolSize, prop: 'imageinfo', - iiprop: 'url|extmetadata|mime', + iiprop: 'url|extmetadata|mime|size', iiurlwidth: '400', }); - try { - const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { - headers: { 'User-Agent': UA }, - signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), - }); - if (!res.ok) return []; - const data = (await res.json()) as { query?: { pages?: Record } }; - return this.toCommonsCandidates(data.query?.pages, limit); - } catch { - return []; + + for (const params of [search, members]) { + try { + const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, { + headers: { 'User-Agent': UA }, + signal: AbortSignal.timeout(WIKI_TIMEOUT_MS), + }); + if (!res.ok) continue; + const data = (await res.json()) as { query?: { pages?: Record } }; + const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize)); + if (hits.length) return hits; + } catch { + /* fall through to the second strategy */ + } } + return []; } /** @@ -1033,12 +1331,22 @@ export class MapsService { // OSM details: placeId is "node:123456" or "way:123456" etc. if (placeId.includes(':')) { const [osmType, osmId] = placeId.split(':'); - const element = await this.fetchOverpassDetails(osmType, osmId); - const details = buildOsmDetails(element?.tags || {}, osmType, osmId); - // buildOsmDetails never yields name/address/coordinates — Nominatim is // always the source for those (Overpass contributes the tag-derived rest). - const nominatim = await this.lookupNominatim(osmType, osmId, lang); + const [element, nominatim] = await Promise.all([ + this.fetchOverpassDetails(osmType, osmId), + this.lookupNominatim(osmType, osmId, lang), + ]); + // Overpass has the fuller tag set and wins where both answer, but it is + // also the one that goes down — overpass-api.de is regularly overloaded. + // Nominatim's extratags carry the wikidata/wikipedia/commons tags too, so + // a place keeps its pictures and its description when Overpass times out + // instead of falling back to "photographed within 300m". + const details = buildOsmDetails( + { ...(nominatim?.extratags ?? {}), ...(element?.tags ?? {}) }, + osmType, + osmId, + ); return { place: { diff --git a/server/src/nest/place-enrichment/place-enrichment.service.ts b/server/src/nest/place-enrichment/place-enrichment.service.ts index eafb4216a..1f04923dd 100644 --- a/server/src/nest/place-enrichment/place-enrichment.service.ts +++ b/server/src/nest/place-enrichment/place-enrichment.service.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { Injectable } from '@nestjs/common'; import type { MapsPlaceEnrichmentRequest, @@ -8,8 +9,8 @@ import type { } from '@trek/shared'; import { safeFetchFollow } from '../../utils/ssrfGuard'; import { DatabaseService } from '../database/database.service'; -import { MapsService, withPhotoFetchSlot } from '../maps/maps.service'; -import { isGooglePlaceId } from '../maps/maps.helpers'; +import { MapsService, withPhotoFetchSlot, type CommonsCandidate } from '../maps/maps.service'; +import { isGooglePlaceId, rankCommonsCandidates } from '../maps/maps.helpers'; import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service'; /** @@ -37,11 +38,40 @@ const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; * from before the change — which is exactly what happened when the facts and * Wikivoyage landed. */ -const CACHE_VERSION = 2; +const CACHE_VERSION = 3; -/** Cache key for one candidate picture. Deliberately not a bare place id. */ -function candidateKey(placeId: string, index: number): string { - return `${placeId}~p${index}`; +/** + * Cache key for one candidate picture. Deliberately not a bare place id. + * + * Keyed by which picture it is, not by where it sat in the strip. Position was + * the obvious choice and it was wrong: the ladder below returns different + * numbers of candidates depending on which providers answered, so a slot that + * held a Wikidata photo one minute held a category photo the next — and the + * cache hands back whatever bytes it stored for that slot while the response + * carries the new candidate's author. Under CC BY that is not a stale + * thumbnail, it is the wrong person credited for someone's work. + * + * Hashed because the identity can be a Google resource path with slashes, and + * this ends up in a URL. + */ +export function candidateKey(placeId: string, identity: string): string { + const digest = createHash('sha1').update(identity).digest('hex').slice(0, 12); + return `${placeId}~p${digest}`; +} + +/** A ranked Commons pick, tagged with the rung of the ladder it came from. */ +type CommonsPick = CommonsCandidate & { rung: 'wikidata' | 'wikipedia' | 'category' | 'nearby' }; + +interface PendingCandidate { + /** Stable identity of the underlying file, for the cache key. */ + identity: string; + candidate: PlacePhotoCandidate; + fetchBytes: () => Promise; +} + +/** Appends candidates to the pool, remembering which rung produced them. */ +function push(pool: CommonsPick[], candidates: CommonsCandidate[], rung: CommonsPick['rung']): void { + for (const candidate of candidates) pool.push({ ...candidate, rung }); } /** @@ -207,53 +237,95 @@ export class PlaceEnrichmentService { const apiKey = this.maps.getMapsKey(userId); const wantsGoogle = !!apiKey && !this.maps.photosDisabled() && isGooglePlaceId(placeId); - // Ask both providers at once — one listing call each, and they know nothing - // about one another. - // Accuracy, best first. Wikidata's P18 is one picture somebody chose to - // represent this exact object. A Commons category is pictures of it. The - // coordinate search is only pictures NEAR it, which around a city centre - // means statues and passers-by — it is the last resort, not the plan. - const category = typeof details?.wikimedia_commons === 'string' ? details.wikimedia_commons : null; - const wikidata = typeof details?.wikidata === 'string' ? details.wikidata : null; + const tag = (key: string): string | null => (typeof details?.[key] === 'string' ? (details[key] as string) : null); + const wikidata = tag('wikidata'); + const wikipedia = tag('wikipedia'); - const [googleRefs, wikidataImage, commons] = await Promise.all([ - wantsGoogle ? this.maps.fetchGooglePhotoRefs(placeId, apiKey!, GOOGLE_CAP) : Promise.resolve([]), - wikidata ? this.maps.fetchWikidataImage(wikidata) : Promise.resolve(null), - category - ? this.maps - .fetchCommonsCategoryCandidates(category, COMMONS_CAP) - .then((hits) => (hits.length ? hits : this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP))) - : this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP), - ]); + // Google's listing is one call for the whole strip and knows nothing about + // the free sources, so it runs alongside them rather than in the ladder. + const googlePending = wantsGoogle + ? this.maps.fetchGooglePhotoRefs(placeId, apiKey!, GOOGLE_CAP) + : Promise.resolve([] as { name: string; attribution: string | null }[]); - // The Wikidata picture leads, and is dropped from the rest of the list when - // a category happens to contain it too. - const commonsList = wikidataImage - ? [wikidataImage, ...commons.filter((c) => c.photoUrl !== wikidataImage.photoUrl)].slice(0, COMMONS_CAP) - : commons; + // The free ladder, in order of how much anyone vouched that the picture + // shows THIS place: + // Wikidata — a person attached this file to this exact object. + // Wiki lead — the article about it opens with this picture. + // Category — the set of pictures of it. + // Nearby — anything photographed within 300m. + // Only the last one has no claim on the subject at all, which is how an + // airport ended up represented by aerial survey tiles of its runway and a + // gate by the underground station beneath it. It is the fallback, and it + // only runs when the ladder above came up short. + const commonsPool: CommonsPick[] = []; + let categoryName = tag('wikimedia_commons'); - // Google first — its pictures are of the place itself, while Commons - // geosearch finds whatever happens to have been photographed nearby. - const pending: PlacePhotoCandidate[] = [ + if (wikidata) { + const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP); + push(commonsPool, fromWikidata.candidates, 'wikidata'); + categoryName ??= fromWikidata.commonsCategory; + } + + if (commonsPool.length < COMMONS_CAP && wikipedia) { + const leadName = await this.maps.fetchWikiLeadImageName(wikipedia); + if (leadName) { + const byName = await this.maps.fetchCommonsFilesByName([leadName]); + push(commonsPool, [...byName.values()], 'wikipedia'); + } + } + + if (commonsPool.length < COMMONS_CAP && categoryName) { + push(commonsPool, await this.maps.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category'); + } + + // Two is the bar: one curated picture plus the nearby noise reads worse + // than one curated picture on its own. + const curated = commonsPool.length; + // Started here, decided below. Waiting for Google's listing first would put + // two round trips end to end, and that is precisely the sequencing that + // used to push this endpoint past the client's timeout. Geosearch is free + // and unmetered, so a speculative call that gets discarded costs nothing + // anyone pays for. + const nearbyPending = + curated < 2 ? this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP) : Promise.resolve([]); + + const googleRefs = await googlePending; + if (curated < 2 && googleRefs.length < GOOGLE_CAP) { + push(commonsPool, await nearbyPending, 'nearby'); + } + + // One ranking pass over everything: the same file reaches us from several + // rungs and only the page id catches that. + const ranked = rankCommonsCandidates(commonsPool, COMMONS_CAP, { perAuthor: curated > 0 ? 2 : 1 }); + // Google leads the strip — its pictures are of the business itself. + const pending: PendingCandidate[] = [ ...googleRefs.map((ref) => ({ - key: '', - url: '', - attribution: ref.attribution, - // Google grants no reusable licence for these; the author line it hands - // back is all we may show, so the rest stays honestly empty. - license: null, - licenseUrl: null, - sourceUrl: null, - source: 'google' as const, + identity: `google:${ref.name}`, + candidate: { + key: '', + url: '', + attribution: ref.attribution, + // Google grants no reusable licence for these; the author line it + // hands back is all we may show, so the rest stays honestly empty. + license: null, + licenseUrl: null, + sourceUrl: null, + source: 'google' as const, + }, + fetchBytes: () => this.maps.fetchGooglePhotoBytes(ref.name, apiKey!), })), - ...commonsList.map((candidate) => ({ - key: '', - url: '', - attribution: candidate.attribution, - license: candidate.license, - licenseUrl: candidate.licenseUrl, - sourceUrl: candidate.sourceUrl, - source: 'wikimedia' as const, + ...ranked.map((pick) => ({ + identity: `commons:${pick.pageId ?? pick.photoUrl}`, + candidate: { + key: '', + url: '', + attribution: pick.attribution, + license: pick.license, + licenseUrl: pick.licenseUrl, + sourceUrl: pick.sourceUrl, + source: (pick.rung === 'wikipedia' ? 'wikipedia' : 'wikimedia') as 'wikipedia' | 'wikimedia', + }, + fetchBytes: () => this.fetchRemoteBytes(pick.photoUrl), })), ]; @@ -262,16 +334,9 @@ export class PlaceEnrichmentService { // which pushed the whole request past the client's request timeout. The // shared slot limiter still caps how many run at once. const results = await Promise.all( - pending.map(async (candidate, index) => { - const key = candidateKey(placeId, index); - const fetchBytes = - candidate.source === 'google' - ? () => this.maps.fetchGooglePhotoBytes(googleRefs[index].name, apiKey!) - : () => this.fetchRemoteBytes(commonsList[index - googleRefs.length].photoUrl); + pending.map(async ({ identity, candidate, fetchBytes }) => { + const key = candidateKey(placeId, identity); const url = await this.storeCandidate(key, creditLine(candidate.attribution, candidate.license), fetchBytes); - // The index is fixed before any download, so a picture that fails to - // load leaves a gap rather than renumbering the ones after it — the key - // has to keep pointing at the same file across requests. return url ? { ...candidate, key, url } : null; }), ); diff --git a/server/tests/e2e/place-enrichment.e2e.test.ts b/server/tests/e2e/place-enrichment.e2e.test.ts index 456272280..a0e75ff91 100644 --- a/server/tests/e2e/place-enrichment.e2e.test.ts +++ b/server/tests/e2e/place-enrichment.e2e.test.ts @@ -30,6 +30,7 @@ const { db } = vi.hoisted(() => { vi.mock('../../src/db/database', () => ({ db, closeDb: () => {}, reinitialize: () => {} })); import { PlaceEnrichmentModule } from '../../src/nest/place-enrichment/place-enrichment.module'; +import { candidateKey } from '../../src/nest/place-enrichment/place-enrichment.service'; import { MapsService } from '../../src/nest/maps/maps.service'; import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service'; import { DatabaseModule } from '../../src/nest/database/database.module'; @@ -123,6 +124,11 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => license: 'CC BY-SA 4.0', licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/', sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg', + pageId: 4711, + title: 'File:X.jpg', + width: 1600, + height: 1200, + descriptors: null, }, ]); // The bytes download goes through the SSRF-guarded fetch — stub the network. @@ -133,7 +139,8 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () => expect(res.status).toBe(200); expect(res.body.photos).toHaveLength(1); expect(res.body.photos[0]).toMatchObject({ - key: 'way:12345~p0', + // Keyed by the file's Commons page id, not by its slot in the strip. + key: candidateKey('way:12345', 'commons:4711'), attribution: 'Alice', license: 'CC BY-SA 4.0', source: 'wikimedia', diff --git a/server/tests/unit/nest/maps.curated-images.test.ts b/server/tests/unit/nest/maps.curated-images.test.ts new file mode 100644 index 000000000..f73e11c5d --- /dev/null +++ b/server/tests/unit/nest/maps.curated-images.test.ts @@ -0,0 +1,254 @@ +/** + * MAPS-147..156 — the curated picture sources behind the place detail column. + * + * These are the rungs above "photographed within 300m": the Wikidata item, the + * article's lead image, and the Commons batch that turns a file name into + * something creditable. Payload shapes are taken from live responses for Q1097 + * (Berlin Hauptbahnhof), Q27706 (Hamburg Airport) and Q82425 (Brandenburg + * Gate) — the three places whose pictures were wrong. + * + * fetch is stubbed; the DB is mocked the same way maps.service.test.ts does it. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +vi.mock('../../../src/db/database', () => ({ + db: { prepare: () => ({ get: () => undefined, run: () => undefined, all: () => [] }) }, +})); + +vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) })); + +vi.mock('../../../src/utils/ssrfGuard', () => ({ + safeFetchFollow: vi.fn(), + checkSsrf: vi.fn(async () => ({ allowed: true })), + SsrfBlockedError: class extends Error {}, +})); + +import { db } from '../../../src/db/database'; +import { DatabaseService } from '../../../src/nest/database/database.service'; +import { MapsService } from '../../../src/nest/maps/maps.service'; + +const svcOf = () => new MapsService(new DatabaseService(db as never), {} as never); + +const filePage = (over: Record = {}) => ({ + pageid: 4711, + title: 'File:Outside.jpg', + imageinfo: [ + { + url: 'https://commons.org/full.jpg', + thumburl: 'https://commons.org/thumb.jpg', + mime: 'image/jpeg', + width: 1600, + height: 1200, + descriptionurl: 'https://commons.wikimedia.org/wiki/File:X.jpg', + extmetadata: { Artist: { value: 'Alice' }, LicenseShortName: { value: 'CC BY-SA 4.0' } }, + }, + ], + ...over, +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('fetchWikidataCandidates', () => { + it('MAPS-147: refuses anything that is not a Q-id without calling out', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + expect(await svcOf().fetchWikidataCandidates('not-a-qid')).toEqual({ candidates: [], commonsCategory: null }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('MAPS-148: puts the preferred P18 statement ahead of a normal one', async () => { + // The API returns statements in edit order, not rank order, so reading the + // first one is a coin toss between the picture an editor chose to represent + // the place and whatever happened to be added first. + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + entities: { + Q82425: { + claims: { + P18: [ + { mainsnak: { datavalue: { value: 'Berlin-Brandenburg Gate overview.jpg' } }, rank: 'normal' }, + { mainsnak: { datavalue: { value: 'Brandenburger Tor morgens.jpg' } }, rank: 'preferred' }, + ], + P373: [{ mainsnak: { datavalue: { value: 'Brandenburg Gate' } } }], + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: {} } }) }); + vi.stubGlobal('fetch', fetchMock); + + const out = await svcOf().fetchWikidataCandidates('Q82425'); + + const batchUrl = decodeURIComponent(String(fetchMock.mock.calls[1][0])); + expect(batchUrl.indexOf('Brandenburger Tor morgens')).toBeLessThan(batchUrl.indexOf('overview')); + // P373 comes back too, so a place with no wikimedia_commons tag still finds + // its category — Berlin Hauptbahnhof is exactly that case. + expect(out.commonsCategory).toBe('Brandenburg Gate'); + }); + + it('MAPS-149: ignores a deprecated statement', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce({ + ok: true, + json: async () => ({ + entities: { Q1: { claims: { P18: [{ mainsnak: { datavalue: { value: 'Wrong.jpg' } }, rank: 'deprecated' }] } } }, + }), + }); + vi.stubGlobal('fetch', fetchMock); + + expect((await svcOf().fetchWikidataCandidates('Q1')).candidates).toEqual([]); + // Nothing worth asking Commons about, so Commons is not asked. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('MAPS-150: gathers the other view properties, not just P18', async () => { + // A station is not one view. P5775 (interior), P3451 (night) and P8592 + // (aerial) are what make the strip genuinely different pictures rather than + // four takes on the same facade. + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + entities: { + Q1097: { + claims: { + P18: [{ mainsnak: { datavalue: { value: 'Outside.jpg' } } }], + P5775: [{ mainsnak: { datavalue: { value: 'Inside.jpg' } } }], + P3451: [{ mainsnak: { datavalue: { value: 'AtNight.jpg' } } }], + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: {} } }) }); + vi.stubGlobal('fetch', fetchMock); + + await svcOf().fetchWikidataCandidates('Q1097'); + + const batchUrl = decodeURIComponent(String(fetchMock.mock.calls[1][0])); + expect(batchUrl).toContain('File:Outside.jpg|File:Inside.jpg|File:AtNight.jpg'); + }); + + it('MAPS-151: hands the files back in the order the claims implied', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + entities: { + Q1: { + claims: { + P18: [{ mainsnak: { datavalue: { value: 'First.jpg' } } }], + P5775: [{ mainsnak: { datavalue: { value: 'Second.jpg' } } }], + }, + }, + }, + }), + }) + .mockResolvedValueOnce({ + ok: true, + // Deliberately the other way round: a batch response does not preserve + // the order the titles went in. + json: async () => ({ + query: { + pages: { + '2': filePage({ pageid: 2, title: 'File:Second.jpg' }), + '1': filePage({ pageid: 1, title: 'File:First.jpg' }), + }, + }, + }), + }); + vi.stubGlobal('fetch', fetchMock); + + const out = await svcOf().fetchWikidataCandidates('Q1'); + expect(out.candidates.map((c) => c.title)).toEqual(['File:First.jpg', 'File:Second.jpg']); + }); + + it('MAPS-152: survives a bad response and a throw', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); + expect(await svcOf().fetchWikidataCandidates('Q1')).toEqual({ candidates: [], commonsCategory: null }); + + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); + expect(await svcOf().fetchWikidataCandidates('Q1')).toEqual({ candidates: [], commonsCategory: null }); + }); +}); + +describe('fetchCommonsFilesByName', () => { + it('MAPS-153: follows a rename back to the name the caller asked for', async () => { + // Wikidata claims and article lead images routinely name files that have + // since been renamed. Without redirects the API answers with a `missing` + // page and the picture disappears without a word. + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + query: { + normalized: [{ from: 'File:old_name.jpg', to: 'File:Old name.jpg' }], + redirects: [{ from: 'File:Old name.jpg', to: 'File:New name.jpg' }], + pages: { '9': filePage({ pageid: 9, title: 'File:New name.jpg' }) }, + }, + }), + }); + vi.stubGlobal('fetch', fetchMock); + + const out = await svcOf().fetchCommonsFilesByName(['old_name.jpg']); + + expect(String(fetchMock.mock.calls[0][0])).toContain('redirects=1'); + expect(out.get('old name.jpg')?.pageId).toBe(9); + // And under its current name, for callers that already resolved it. + expect(out.get('new name.jpg')?.pageId).toBe(9); + }); + + it('MAPS-154: asks nothing for an empty list', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + expect((await svcOf().fetchCommonsFilesByName([])).size).toBe(0); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('MAPS-154b: returns an empty map on a bad response or a throw', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) })); + expect((await svcOf().fetchCommonsFilesByName(['X.jpg'])).size).toBe(0); + + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); + expect((await svcOf().fetchCommonsFilesByName(['X.jpg'])).size).toBe(0); + }); +}); + +describe('fetchWikiLeadImageName', () => { + it('MAPS-155: asks Wikivoyage before Wikipedia and returns only the file name', async () => { + // Only the name. The thumbnail URL the API offers alongside carries no + // author, and a picture that cannot be credited cannot be shown. + const fetchMock = vi + .fn() + .mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: { '1': {} } } }) }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ query: { pages: { '2': { pageimage: 'Hamburg airport terminals.jpg' } } } }), + }); + vi.stubGlobal('fetch', fetchMock); + + const name = await svcOf().fetchWikiLeadImageName('de:Flughafen Hamburg'); + + expect(name).toBe('Hamburg airport terminals.jpg'); + expect(String(fetchMock.mock.calls[0][0])).toContain('de.wikivoyage.org'); + expect(String(fetchMock.mock.calls[1][0])).toContain('de.wikipedia.org'); + }); + + it('MAPS-156: yields null without a tag and survives a throw', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + expect(await svcOf().fetchWikiLeadImageName(null)).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network'))); + expect(await svcOf().fetchWikiLeadImageName('de:X')).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/maps.service.test.ts b/server/tests/unit/nest/maps.service.test.ts index 8552ec61e..5cca903b3 100644 --- a/server/tests/unit/nest/maps.service.test.ts +++ b/server/tests/unit/nest/maps.service.test.ts @@ -22,6 +22,8 @@ import { resolveOverpassTimeoutMs, stripWikiMarkup, parseWikipediaTag, + rankCommonsCandidates, + type RankableCommonsCandidate, } from '../../../src/nest/maps/maps.helpers'; import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest'; @@ -104,7 +106,7 @@ const photoCacheStub = { import { db } from '../../../src/db/database'; import { DatabaseService } from '../../../src/nest/database/database.service'; -import { MapsService, withPhotoFetchSlot } from '../../../src/nest/maps/maps.service'; +import { MapsService, withPhotoFetchSlot, readWikiIdentity } from '../../../src/nest/maps/maps.service'; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; // The service under test, constructed over the mocked db stub — DatabaseService @@ -2343,13 +2345,16 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { ); const out = await svc.fetchCommonsCandidates(48.8, 2.3, 5); expect(out).toHaveLength(2); - expect(out[0]).toEqual({ + expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/thumb.jpg', attribution: 'Alice', license: 'CC BY-SA 4.0', licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/', sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg', }); + // The page id travels with the candidate: it is what identifies the file + // across the four routes that can reach it, and what keys its cached bytes. + expect(out[0].pageId).toBe(1); }); it('MAPS-118: falls back to UsageTerms when there is no short licence name', async () => { @@ -2397,18 +2402,22 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => { expect(out[0].photoUrl).toBe('https://commons.org/original.jpg'); }); - it('MAPS-120: honours the limit and clamps it into the range Commons accepts', async () => { + it('MAPS-120: asks for more than the strip needs and clamps to what Commons accepts', async () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: { '1': page(), '2': page(), '3': page() } } }), }); vi.stubGlobal('fetch', fetchMock); - expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(2); - expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=2'); + // Overfetch on purpose: the first hits around anything worth visiting are + // survey tiles and the building next door, and the ranker can only reject + // from a pool. Geosearch bills the same for one result as for twenty, so + // the whole pool comes back and gets cut after ranking, not before. + expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3); + expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=8'); await svc.fetchCommonsCandidates(48.8, 2.3, 0); - expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=1'); + expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=8'); await svc.fetchCommonsCandidates(48.8, 2.3, 99); expect(String(fetchMock.mock.calls[2][0])).toContain('ggslimit=20'); @@ -2534,15 +2543,40 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => { const out = await svc.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3); expect(out).toHaveLength(1); expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/t.jpg', attribution: 'Alice', license: 'CC BY 4.0' }); - expect(String(fetchMock.mock.calls[0][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig'); + // Ranked search, not the category listing: `categorymembers` orders by sort + // key, i.e. alphabetically by file name, and that put an .ogg pronunciation + // and six near-identical press shots at the head of the Brandenburg Gate + // category. The search index at least ranks by how well a file matches. + expect(String(fetchMock.mock.calls[0][0])).toContain('gsrsearch=incategory%3A%22Museum+Ludwig%22'); }); - it('MAPS-125d: adds the Category prefix when the tag omits it', async () => { + it('MAPS-125d: takes the bare category name whether or not the tag is prefixed', async () => { const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) }); vi.stubGlobal('fetch', fetchMock); await svc.fetchCommonsCategoryCandidates('Museum Ludwig'); - expect(String(fetchMock.mock.calls[0][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig'); + expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22'); + // Empty search falls through to the category listing as a second chance. + expect(String(fetchMock.mock.calls[1][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig'); + }); + + it('MAPS-125f: refuses a tag that names a file rather than a category', async () => { + // Mappers do put `File:…` in wikimedia_commons. Prefixing it blindly asked + // Commons for `Category:File:X.jpg`, which matches nothing and fell through + // to the coordinate search without a word. + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + expect(await svc.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('MAPS-125g: strips a localised category prefix', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) }); + vi.stubGlobal('fetch', fetchMock); + + await svc.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig'); + expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22'); }); it('MAPS-125e: yields nothing on an error response or a throw', async () => { @@ -2706,3 +2740,139 @@ describe('withPhotoFetchSlot', () => { await expect(withPhotoFetchSlot(async () => 'free')).resolves.toBe('free'); }); }); + +/** + * MAPS-135..146 — the Commons ranker and the identity tags. + * + * Every fixture here is a real payload recorded while working out why the + * picker showed runways and train front-ends for Hamburg Airport, Berlin + * Hauptbahnhof and the Brandenburg Gate. + */ +describe('rankCommonsCandidates', () => { + const pic = (over: Partial & { pageId: number }): RankableCommonsCandidate => ({ + title: `File:pic ${over.pageId}.jpg`, + attribution: 'Someone', + width: 1600, + height: 1200, + descriptors: null, + photoUrl: `https://commons/thumb/${over.pageId}.jpg`, + ...over, + }); + + it('MAPS-135: drops a file that reached us twice under different thumbnail urls', () => { + const first = pic({ pageId: 47529341, photoUrl: 'https://commons/x.jpg?utm_source=commons.wikimedia.org' }); + const again = { ...first, photoUrl: 'https://commons/x.jpg?utm_source=de.wikipedia.org' }; + expect(rankCommonsCandidates([first, again], 5)).toHaveLength(1); + }); + + it('MAPS-136: rejects the aerial survey tiles that fill an airport geosearch', () => { + const tiles = [2013, 2015, 2016, 2019].map((year, i) => + pic({ + pageId: 100 + i, + title: `File:Dop20rgb 32565 5943 ${year}.jpg`, + attribution: 'Landesbetrieb Geoinformation Hamburg', + width: 5000, + height: 5000, + descriptors: `Orthophoto Sommerbefliegung ${year}`, + }), + ); + expect(rankCommonsCandidates(tiles, 5)).toEqual([]); + }); + + it('MAPS-137: rejects noise maps and terminal layouts from a category', () => { + const docs = [ + pic({ pageId: 200, title: 'File:Lärmkarte Flughafen Hamburg.png' }), + pic({ pageId: 201, title: 'File:EDDH HAM Layout.png' }), + pic({ pageId: 202, title: 'File:Grundriss Terminal 1.jpg' }), + ]; + expect(rankCommonsCandidates(docs, 5)).toEqual([]); + }); + + it('MAPS-138: keeps one frame of a camera burst, not four', () => { + // The four shots that filled the Brandenburg Gate picker were one person + // walking through a station concourse. + const burst = [807, 808, 809, 810].map((n) => + pic({ pageId: 300 + n, title: `File:Hauptbahnhof Berlin interior 0${n}.jpg`, attribution: 'Dosseman' }), + ); + const kept = rankCommonsCandidates(burst, 5); + expect(kept).toHaveLength(1); + expect(kept[0].title).toBe('File:Hauptbahnhof Berlin interior 0807.jpg'); + }); + + it('MAPS-139: collapses a press set numbered with trailing counters', () => { + const set = [16, 17, 18, 19, 20, 21].map((n) => + pic({ pageId: 400 + n, title: `File:Alexander Schallenberg in Berlin on 7 June 2024 - ${n}.jpg` }), + ); + expect(rankCommonsCandidates(set, 5)).toHaveLength(1); + }); + + it('MAPS-140: caps how much of the strip one author may supply', () => { + // Unrelated titles, same photographer — the series rule cannot see these, + // only the author cap can. On the coordinate rung nothing vouches that the + // pictures are even of the right subject, so variety is the only defence + // and one contributor gets one slot; a curated rung gets two. + const same = ['Rear entrance', 'Platform at dusk', 'Ticket hall'].map((subject, i) => + pic({ pageId: 500 + i, title: `File:${subject}.jpg`, attribution: 'Bahnthaler' }), + ); + expect(rankCommonsCandidates(same, 5, { perAuthor: 1 })).toHaveLength(1); + expect(rankCommonsCandidates(same, 5, { perAuthor: 2 })).toHaveLength(2); + }); + + it('MAPS-140b: the series rule catches a numbered pair before the author cap does', () => { + // "Berlin-Hamburg-Express 1/2.JPG", both by Bahnthaler — two files, one + // subject. This is the pair that filled the Berlin Hauptbahnhof picker. + const pair = [1, 2].map((n) => + pic({ pageId: 510 + n, title: `File:Berlin-Hamburg-Express ${n}.JPG`, attribution: 'Bahnthaler' }), + ); + expect(rankCommonsCandidates(pair, 5, { perAuthor: 2 })).toHaveLength(1); + }); + + it('MAPS-141: rejects panorama strips that would crop to nothing in a square tile', () => { + expect(rankCommonsCandidates([pic({ pageId: 600, width: 9000, height: 1200 })], 5)).toEqual([]); + }); + + it('MAPS-142: keeps a large square photo that is not a survey tile', () => { + const square = pic({ pageId: 601, width: 1200, height: 1200 }); + expect(rankCommonsCandidates([square], 5)).toHaveLength(1); + }); + + it('MAPS-143: falls back to the url when a candidate carries no page id', () => { + const noId = { ...pic({ pageId: 0 }), pageId: null }; + expect(rankCommonsCandidates([noId, { ...noId }], 5)).toHaveLength(1); + }); + + it('MAPS-144: stops at the limit', () => { + const many = Array.from({ length: 9 }, (_, i) => + pic({ pageId: 700 + i, title: `File:Distinct subject ${String.fromCharCode(97 + i)}.jpg`, attribution: `Author ${i}` }), + ); + expect(rankCommonsCandidates(many, 3)).toHaveLength(3); + }); +}); + +describe('readWikiIdentity', () => { + it('MAPS-145: reads the three identity tags and nothing else', () => { + expect( + readWikiIdentity({ + wikidata: 'Q82425', + wikipedia: 'de:Brandenburger Tor', + wikimedia_commons: 'Category:Brandenburg Gate', + cuisine: 'italian', + }), + ).toEqual({ + wikidata: 'Q82425', + wikipedia: 'de:Brandenburger Tor', + wikimedia_commons: 'Category:Brandenburg Gate', + }); + }); + + it('MAPS-146: ignores brand tags, which describe the chain and not the branch', () => { + // Real payload for "L'Osteria Rostock": following brand:wikidata would + // describe (and illustrate) L'Osteria the company. + const identity = readWikiIdentity({ + 'brand:wikidata': 'Q17323478', + 'brand:wikipedia': 'de:L’Osteria', + cuisine: 'pizza', + }); + expect(identity).toEqual({ wikidata: null, wikipedia: null, wikimedia_commons: null }); + }); +}); diff --git a/server/tests/unit/nest/place-enrichment.service.test.ts b/server/tests/unit/nest/place-enrichment.service.test.ts index 46dff0930..0fca91f8b 100644 --- a/server/tests/unit/nest/place-enrichment.service.test.ts +++ b/server/tests/unit/nest/place-enrichment.service.test.ts @@ -36,7 +36,7 @@ vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KE import { db } from '../../../src/db/database'; import { DatabaseService } from '../../../src/nest/database/database.service'; -import { PlaceEnrichmentService, creditLine, collectFacts } from '../../../src/nest/place-enrichment/place-enrichment.service'; +import { candidateKey, PlaceEnrichmentService, creditLine, collectFacts } from '../../../src/nest/place-enrichment/place-enrichment.service'; import type { MapsService } from '../../../src/nest/maps/maps.service'; import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; @@ -55,6 +55,9 @@ function mapsStub(over: Partial> = {}) { fetchEditorialSummary: vi.fn(async () => null as string | null), fetchWikiExtract: vi.fn(async () => null as { text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null), fetchCommonsCategoryCandidates: vi.fn(async () => [] as any[]), + fetchWikidataCandidates: vi.fn(async () => ({ candidates: [] as any[], commonsCategory: null as string | null })), + fetchCommonsFilesByName: vi.fn(async () => new Map()), + fetchWikiLeadImageName: vi.fn(async () => null as string | null), details: vi.fn(async () => ({ place: null })), ...over, } as unknown as MapsService; @@ -76,16 +79,31 @@ function make(maps: MapsService, cache: PlacePhotoCacheService) { return new PlaceEnrichmentService(new DatabaseService(db as never), maps, cache); } -const commonsCandidate = (over: Record = {}) => ({ - photoUrl: 'https://commons.org/thumb.jpg', - attribution: 'Alice', - license: 'CC BY-SA 4.0', - licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/', - sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg', - ...over, -}); +let candidateSeq = 0; +const commonsCandidate = (over: Record = {}) => { + // Distinct by default: the ranker deduplicates by page id and collapses + // same-stem titles, so fixtures that share either would silently collapse + // into one and make a test about ordering pass for the wrong reason. + const n = ++candidateSeq; + return { + photoUrl: 'https://commons.org/thumb.jpg', + attribution: 'Alice', + license: 'CC BY-SA 4.0', + licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/', + sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg', + pageId: 1000 + n, + title: `File:Subject ${String.fromCharCode(96 + n)}.jpg`, + width: 1600, + height: 1200, + descriptors: null, + ...over, + }; +}; beforeEach(() => { + // Per test, so a fixture's page id is a property of the case rather than of + // where the case sits in the file. + candidateSeq = 0; mockDbGet.mockReset(); mockDbGet.mockReturnValue(undefined); mockDbRun.mockReset(); @@ -129,10 +147,12 @@ describe('collectPhotos', () => { const out = await make(maps, cacheStub()).enrich(1, REQ); + // Keyed by which file it is, not by where it landed in the strip. + const key = candidateKey('ChIJmuseum', 'commons:1001'); expect(out.photos).toEqual([ { - key: 'ChIJmuseum~p0', - url: '/api/maps/place-photo/ChIJmuseum~p0/bytes', + key, + url: `/api/maps/place-photo/${encodeURIComponent(key)}/bytes`, attribution: 'Alice', license: 'CC BY-SA 4.0', licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/', @@ -166,14 +186,14 @@ describe('collectPhotos', () => { expect(out.photos.map((p) => p.source)).toEqual(['google', 'wikimedia']); expect(out.photos[0]).toMatchObject({ - key: 'ChIJmuseum~p0', + key: candidateKey('ChIJmuseum', 'google:places/p/photos/a'), attribution: 'Bob', // Google grants no reusable licence — the fields stay empty rather than invented. license: null, licenseUrl: null, sourceUrl: null, }); - expect(out.photos[1].key).toBe('ChIJmuseum~p1'); + expect(out.photos[1].key).toBe(candidateKey('ChIJmuseum', 'commons:1001')); }); it('ENRICH-006: respects the existing photos kill switch for the Google half', async () => { @@ -212,7 +232,10 @@ describe('collectPhotos', () => { it('ENRICH-009: skips a candidate whose bytes will not download', async () => { const maps = mapsStub({ - fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ photoUrl: 'https://commons.org/gone.jpg' }), commonsCandidate()]), + fetchCommonsCandidates: vi.fn(async () => [ + commonsCandidate({ photoUrl: 'https://commons.org/gone.jpg', attribution: 'Alice' }), + commonsCandidate({ attribution: 'Bob' }), + ]), }); mockSafeFetchFollow .mockResolvedValueOnce({ ok: false, arrayBuffer: async () => new ArrayBuffer(0) }) @@ -220,11 +243,11 @@ describe('collectPhotos', () => { const out = await make(maps, cacheStub()).enrich(1, REQ); - // The survivor keeps index 1 — keys number the provider's list, so a picture - // that fails leaves a gap instead of renumbering the ones behind it. Keys - // have to point at the same file on the next request. + // The survivor keeps its own key. That used to depend on the failed + // picture leaving a numbered gap behind it; now the key is derived from + // which file it is, so nothing in front of it can shift it. expect(out.photos).toHaveLength(1); - expect(out.photos[0].key).toBe('ChIJmuseum~p1'); + expect(out.photos[0].key).toBe(candidateKey('ChIJmuseum', 'commons:1002')); }); it('ENRICH-010: treats an empty body and a throwing download as a miss', async () => { @@ -277,7 +300,10 @@ describe('collectPhotos', () => { const out = await make(maps, cacheStub()).enrich(1, REQ); expect(out.photos).toHaveLength(1); - expect(out.photos[0]).toMatchObject({ key: 'ChIJmuseum~p1', attribution: 'Cara' }); + expect(out.photos[0]).toMatchObject({ + key: candidateKey('ChIJmuseum', 'google:places/p/photos/ok'), + attribution: 'Cara', + }); }); it('ENRICH-012c: downloads the pictures concurrently, not one after another', async () => { @@ -286,7 +312,9 @@ describe('collectPhotos', () => { let active = 0; let peak = 0; const maps = mapsStub({ - fetchCommonsCandidates: vi.fn(async () => [commonsCandidate(), commonsCandidate(), commonsCandidate()]), + fetchCommonsCandidates: vi.fn(async () => + ['Alice', 'Bob', 'Cara'].map((attribution) => commonsCandidate({ attribution })), + ), }); mockSafeFetchFollow.mockImplementation(async () => { active++; @@ -329,7 +357,7 @@ describe('collectPhotos', () => { const out = await make(maps, cacheStub()).enrich(1, { lat: 50.9, lng: 6.96, name: 'Bench' }); - expect(out.photos[0].key).toBe('coords:50.9:6.96~p0'); + expect(out.photos[0].key).toBe(candidateKey('coords:50.9:6.96', 'commons:1001')); expect(maps.fetchGooglePhotoRefs).not.toHaveBeenCalled(); }); }); @@ -473,7 +501,7 @@ describe('collectDescription', () => { describe('result cache', () => { // v is the cache format version; an entry written before a release that // changes the shape or the sources is discarded rather than served. - const cachedRow = (photos: unknown[], fetchedAt = Date.now(), v: number | undefined = 2) => ({ + const cachedRow = (photos: unknown[], fetchedAt = Date.now(), v: number | undefined = 3) => ({ payload_json: JSON.stringify({ photos, description: null, facts: [], v }), fetched_at: fetchedAt, }); @@ -610,7 +638,11 @@ describe('creditLine', () => { await make(maps, cache).enrich(1, REQ); - expect(cache.put).toHaveBeenCalledWith('ChIJmuseum~p0', expect.any(Buffer), 'Alice · CC BY-SA 4.0'); + expect(cache.put).toHaveBeenCalledWith( + candidateKey('ChIJmuseum', 'commons:1001'), + expect.any(Buffer), + 'Alice · CC BY-SA 4.0', + ); }); it('ENRICH-029: reads a stored credit back by cache key', () => { @@ -702,14 +734,17 @@ describe('picture source selection', () => { // category of a place returns the place. const maps = mapsStub({ details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikimedia_commons: 'Category:Museum Ludwig' } })), - fetchCommonsCategoryCandidates: vi.fn(async () => [commonsCandidate()]), + fetchCommonsCategoryCandidates: vi.fn(async () => + ['Alice', 'Bob'].map((attribution) => commonsCandidate({ attribution })), + ), }); const out = await make(maps, cacheStub()).enrich(1, OSM_REQ); expect(maps.fetchCommonsCategoryCandidates).toHaveBeenCalledWith('Category:Museum Ludwig', 5); + // Two pictures of the place is a strip; the coordinate search is not asked. expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled(); - expect(out.photos).toHaveLength(1); + expect(out.photos).toHaveLength(2); }); it('ENRICH-056: falls back to the coordinate search when the category is empty', async () => { @@ -740,3 +775,175 @@ describe('picture source selection', () => { expect(out.description).toMatchObject({ source: 'wikivoyage', license: 'CC BY-SA 4.0' }); }); }); + +/** + * ENRICH-060..069 — the source ladder. + * + * The picker used to show aerial survey tiles for Hamburg Airport and the + * underground station for the Brandenburg Gate, because the only rung that ever + * ran was "photographed within 300m". These cases pin the order it climbs and + * when it is allowed to fall back that far. + */ +describe('picture source ladder', () => { + const withTags = (tags: Record) => + vi.fn(async () => ({ place: { source: 'openstreetmap', ...tags } })); + + it('ENRICH-060: asks Wikidata first when the place carries a wikidata tag', async () => { + const maps = mapsStub({ + details: withTags({ wikidata: 'Q1097' }), + fetchWikidataCandidates: vi.fn(async () => ({ + candidates: ['Alice', 'Bob', 'Cara', 'Dana', 'Erin'].map((attribution) => commonsCandidate({ attribution })), + commonsCategory: 'Berlin Hauptbahnhof', + })), + }); + + const out = await make(maps, cacheStub()).enrich(1, OSM_REQ); + + expect(maps.fetchWikidataCandidates).toHaveBeenCalledWith('Q1097', 5); + // A full strip of pictures somebody attached to this exact object — no rung + // below it is asked, least of all the coordinate search. + expect(maps.fetchCommonsCategoryCandidates).not.toHaveBeenCalled(); + expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled(); + expect(out.photos).toHaveLength(5); + }); + + it('ENRICH-061: takes the Commons category Wikidata names when the tag omits it', async () => { + // Berlin Hauptbahnhof carries wikidata but no wikimedia_commons in OSM; the + // item's P373 knows the category anyway. + const maps = mapsStub({ + details: withTags({ wikidata: 'Q1097' }), + fetchWikidataCandidates: vi.fn(async () => ({ + candidates: [commonsCandidate()], + commonsCategory: 'Berlin Hauptbahnhof', + })), + fetchCommonsCategoryCandidates: vi.fn(async () => [commonsCandidate({ attribution: 'Bob' })]), + }); + + await make(maps, cacheStub()).enrich(1, OSM_REQ); + + expect(maps.fetchCommonsCategoryCandidates).toHaveBeenCalledWith('Berlin Hauptbahnhof', 5); + }); + + it('ENRICH-062: reaches for the article lead image when there is no wikidata tag', async () => { + const maps = mapsStub({ + details: withTags({ wikipedia: 'de:Flughafen Hamburg' }), + fetchWikiLeadImageName: vi.fn(async () => 'Hamburg airport terminals.jpg'), + fetchCommonsFilesByName: vi.fn(async () => + new Map([['hamburg airport terminals.jpg', commonsCandidate({ attribution: 'Alice' })]]), + ), + }); + + const out = await make(maps, cacheStub()).enrich(1, OSM_REQ); + + expect(maps.fetchWikiLeadImageName).toHaveBeenCalledWith('de:Flughafen Hamburg'); + expect(maps.fetchCommonsFilesByName).toHaveBeenCalledWith(['Hamburg airport terminals.jpg']); + // Reported as a Wikipedia picture, because that is where it was chosen. + expect(out.photos[0].source).toBe('wikipedia'); + }); + + it('ENRICH-063: only falls back to what was photographed nearby when the rest came up thin', async () => { + const maps = mapsStub({ + details: withTags({ wikidata: 'Q1097' }), + fetchWikidataCandidates: vi.fn(async () => ({ candidates: [commonsCandidate()], commonsCategory: null })), + fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ attribution: 'Bob' })]), + }); + + await make(maps, cacheStub()).enrich(1, OSM_REQ); + + // One curated picture is not a strip, so the coordinate search tops it up. + expect(maps.fetchCommonsCandidates).toHaveBeenCalled(); + }); + + it('ENRICH-064: does not run the coordinate search behind a full Google strip', async () => { + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + fetchGooglePhotoRefs: vi.fn(async () => [ + { name: 'places/p/photos/a', attribution: 'Bob' }, + { name: 'places/p/photos/b', attribution: 'Cara' }, + { name: 'places/p/photos/c', attribution: 'Dana' }, + ]), + fetchGooglePhotoBytes: vi.fn(async () => Buffer.from([1, 2, 3])), + }); + + const out = await make(maps, cacheStub()).enrich(1, REQ); + + expect(out.photos).toHaveLength(3); + expect(out.photos.every((p) => p.source === 'google')).toBe(true); + }); + + it('ENRICH-065: keeps the coordinate search off the critical path of Google\'s listing', async () => { + // Waiting for the listing before deciding put two round trips end to end, + // which is what pushed this endpoint past the client timeout once before. + const order: string[] = []; + const maps = mapsStub({ + getMapsKey: vi.fn(() => 'key'), + fetchGooglePhotoRefs: vi.fn(async () => { + order.push('google:start'); + await new Promise((resolve) => setTimeout(resolve, 10)); + order.push('google:end'); + return []; + }), + fetchCommonsCandidates: vi.fn(async () => { + order.push('commons:start'); + return []; + }), + }); + + await make(maps, cacheStub()).enrich(1, REQ); + + expect(order.indexOf('commons:start')).toBeLessThan(order.indexOf('google:end')); + }); + + it('ENRICH-066: drops a picture that two rungs both produced', async () => { + const shared = commonsCandidate({ pageId: 4242, attribution: 'Alice' }); + const maps = mapsStub({ + details: withTags({ wikidata: 'Q1097', wikimedia_commons: 'Category:X' }), + fetchWikidataCandidates: vi.fn(async () => ({ candidates: [shared], commonsCategory: null })), + fetchCommonsCategoryCandidates: vi.fn(async () => [{ ...shared, photoUrl: 'https://commons.org/other-url.jpg' }]), + }); + + const out = await make(maps, cacheStub()).enrich(1, OSM_REQ); + + // Same page id, different thumbnail url — the url alone would let it through. + expect(out.photos).toHaveLength(1); + }); + + it('ENRICH-067: throws away the survey tiles a geosearch around an airport returns', async () => { + const maps = mapsStub({ + fetchCommonsCandidates: vi.fn(async () => + [2013, 2015, 2016, 2019].map((year) => + commonsCandidate({ + title: `File:Dop20rgb 32565 5943 ${year}.jpg`, + attribution: 'Landesbetrieb Geoinformation Hamburg', + width: 5000, + height: 5000, + descriptors: 'Orthophoto Sommerbefliegung', + }), + ), + ), + }); + + const out = await make(maps, cacheStub()).enrich(1, REQ); + + expect(out.photos).toEqual([]); + }); + + it('ENRICH-068: keys a picture by which file it is, not by where it sat', async () => { + // The strip length changes with which providers answered. Keyed by position, + // a slot that held a Wikidata photo one minute held a category photo the + // next while the cache kept serving the first one's bytes — under CC BY + // that credits the wrong person, not just the wrong thumbnail. + const one = mapsStub({ fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ pageId: 777 })]) }); + const first = (await make(one, cacheStub()).enrich(1, REQ)).photos[0].key; + + const two = mapsStub({ + fetchCommonsCandidates: vi.fn(async () => [ + commonsCandidate({ pageId: 111, attribution: 'Zoe' }), + commonsCandidate({ pageId: 777 }), + ]), + }); + const shifted = (await make(two, cacheStub()).enrich(1, REQ)).photos.find((p) => p.attribution === 'Alice'); + + expect(shifted?.key).toBe(first); + }); +});