mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
Ask the sources that know the place before the ones that know the area
This commit is contained in:
Maurice
@@ -72,9 +72,16 @@ function writeSession(key: string, value: MapsPlaceEnrichmentResult): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bumped with the server's CACHE_VERSION. sessionStorage outlives a deploy, so
|
||||
* without it the tab that was open while the fix shipped keeps replaying the
|
||||
* answer the fix was about — and reports it as still broken.
|
||||
*/
|
||||
const ENRICH_CACHE_V = 3
|
||||
|
||||
function cacheKeyFor(selection: PlaceDetailsSelection, language: string): string {
|
||||
const id = selection.placeId || `coords:${selection.lat}:${selection.lng}`
|
||||
return `enrich_${id}_${language}`
|
||||
return `enrich_v${ENRICH_CACHE_V}_${id}_${language}`
|
||||
}
|
||||
|
||||
type LoadState = 'idle' | 'loading' | 'ready' | 'error'
|
||||
|
||||
@@ -405,3 +405,121 @@ export const OSM_PLACE_ID = /^(?:node|way|relation):/i;
|
||||
export function isGooglePlaceId(placeId: string): boolean {
|
||||
return !NON_GOOGLE_PLACE_ID.test(placeId);
|
||||
}
|
||||
|
||||
// ── Ranking Commons candidates ───────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A candidate as the ranker needs to see it. Structurally the subset of
|
||||
* `CommonsCandidate` it reads, kept local so the pure function does not have to
|
||||
* import from the service that imports this file.
|
||||
*/
|
||||
export interface RankableCommonsCandidate {
|
||||
pageId: number | null;
|
||||
title: string | null;
|
||||
attribution: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
descriptors: string | null;
|
||||
photoUrl: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commons files that are near a place but are not a picture of it.
|
||||
*
|
||||
* Orthophotos are the worst offender by volume: German states upload their
|
||||
* aerial survey tiles to Commons with coordinates, so a geosearch around any
|
||||
* airport returns the runway as seen from a plane at 3000m — four of them for
|
||||
* Hamburg, one per survey year. The rest are documents that happen to live in a
|
||||
* place's category: noise maps, floor plans, terminal layouts, logos, coats of
|
||||
* arms.
|
||||
*/
|
||||
const NOT_A_PHOTO_OF_THE_PLACE =
|
||||
/orthophoto|orthofoto|sommerbefliegung|luftbildkarte|dop\d+|l[äa]rmkarte|noise map|floor ?plan|grundriss|lageplan|layout|diagram|schematic|blueprint|coat of arms|wappen|\blogo\b|flag of/i;
|
||||
|
||||
/**
|
||||
* Strips what makes two frames of the same burst look like different files, so
|
||||
* they collapse onto one stem: trailing counters, camera dumps, dates, and the
|
||||
* `- 17` / `(2)` suffixes press sets use.
|
||||
*/
|
||||
function seriesStem(title: string): string {
|
||||
return title
|
||||
.replace(/^File:/i, '')
|
||||
.replace(/\.[a-z0-9]+$/i, '')
|
||||
.toLowerCase()
|
||||
// 20260614 100717648 HDR — a camera dump, all from the same minute
|
||||
.replace(/\b\d{8}[ _-]\d{6,9}\b/g, ' ')
|
||||
.replace(/\b(19|20)\d{2}\b/g, ' ')
|
||||
.replace(/[ _-]+\(?\d{1,4}\)?$/g, '')
|
||||
.replace(/[^a-z]+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks the pictures worth showing out of whatever the providers returned.
|
||||
*
|
||||
* Everything here was written against a real failing case. Deduplication is by
|
||||
* page id because the same file reaches us under different thumbnail URLs. The
|
||||
* series rule exists because Commons categories are full of bursts — the four
|
||||
* `Hauptbahnhof Berlin interior 0807..0810` shots that filled the picker were
|
||||
* one person walking across a concourse. The author cap is the same problem
|
||||
* seen from the other side: one contributor who documented a station thoroughly
|
||||
* should not supply the whole strip.
|
||||
*
|
||||
* `perAuthor` is the one knob: coordinate search gets 1, because there being no
|
||||
* evidence the pictures are even of the right subject makes variety the only
|
||||
* defence. Curated sources get 2 — someone already vouched that these depict
|
||||
* the place.
|
||||
*/
|
||||
export function rankCommonsCandidates<T extends RankableCommonsCandidate>(
|
||||
candidates: T[],
|
||||
limit: number,
|
||||
opts: { perAuthor?: number } = {},
|
||||
): T[] {
|
||||
const perAuthor = opts.perAuthor ?? 2;
|
||||
const seenPages = new Set<number>();
|
||||
const seenUrls = new Set<string>();
|
||||
const seenStems = new Set<string>();
|
||||
const authorCounts = new Map<string, number>();
|
||||
const out: T[] = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (out.length >= limit) break;
|
||||
|
||||
if (candidate.pageId != null) {
|
||||
if (seenPages.has(candidate.pageId)) continue;
|
||||
} else if (seenUrls.has(candidate.photoUrl)) {
|
||||
// No page id (an older payload, or a provider that does not report one):
|
||||
// the URL is a weaker key but better than letting an exact repeat through.
|
||||
continue;
|
||||
}
|
||||
|
||||
const haystack = `${candidate.title ?? ''} ${candidate.descriptors ?? ''}`;
|
||||
if (NOT_A_PHOTO_OF_THE_PLACE.test(haystack)) continue;
|
||||
|
||||
const { width, height } = candidate;
|
||||
if (width && height) {
|
||||
// Survey tiles are square and enormous; nothing photographed by hand is.
|
||||
if (width === height && width >= 3000) continue;
|
||||
const ratio = Math.max(width / height, height / width);
|
||||
// Panorama strips and tall banners crop to nothing in a square tile.
|
||||
if (ratio > 4) continue;
|
||||
}
|
||||
|
||||
const stem = candidate.title ? seriesStem(candidate.title) : '';
|
||||
if (stem && seenStems.has(stem)) continue;
|
||||
|
||||
const author = (candidate.attribution ?? '').trim().toLowerCase();
|
||||
if (author) {
|
||||
const used = authorCounts.get(author) ?? 0;
|
||||
if (used >= perAuthor) continue;
|
||||
authorCounts.set(author, used + 1);
|
||||
}
|
||||
|
||||
if (candidate.pageId != null) seenPages.add(candidate.pageId);
|
||||
seenUrls.add(candidate.photoUrl);
|
||||
if (stem) seenStems.add(stem);
|
||||
out.push(candidate);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -54,6 +54,124 @@ interface NominatimResult {
|
||||
display_name?: string;
|
||||
lat: string;
|
||||
lon: string;
|
||||
extratags?: Record<string, string> | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The keys that say which encyclopaedia entry, Wikidata item and Commons
|
||||
* category describe a place. Everything the enrichment column shows beyond
|
||||
* coordinates hangs off one of these.
|
||||
*
|
||||
* Bare keys only. OSM also carries `brand:wikidata` / `brand:wikipedia`, and
|
||||
* following those means a branch of a chain gets the chain's article and the
|
||||
* chain's logo — for "L'Osteria Rostock" you would confidently describe
|
||||
* L'Osteria the company. That is the exact failure the tag-only rule was
|
||||
* written to avoid, so do not "improve" this by falling back to brand:*.
|
||||
*/
|
||||
const WIKI_IDENTITY_TAGS = ['wikipedia', 'wikidata', 'wikimedia_commons'] as const;
|
||||
|
||||
export interface WikiIdentity {
|
||||
wikipedia: string | null;
|
||||
wikidata: string | null;
|
||||
wikimedia_commons: string | null;
|
||||
}
|
||||
|
||||
interface WikidataSnak {
|
||||
mainsnak?: { datavalue?: { value?: string } };
|
||||
rank?: 'preferred' | 'normal' | 'deprecated';
|
||||
}
|
||||
type WikidataClaims = Record<string, WikidataSnak[] | undefined>;
|
||||
|
||||
/**
|
||||
* Wikidata properties that name a picture of a place, in the order we want them.
|
||||
*
|
||||
* P18 is the representative image. The rest exist because a station or a
|
||||
* monument is not one view: asking for the interior, the night shot, the
|
||||
* panorama and the aerial gives a picker four genuinely different pictures
|
||||
* instead of four frames of the same façade.
|
||||
*/
|
||||
const WIKIDATA_IMAGE_PROPERTIES = [
|
||||
'P18', // image
|
||||
'P5775', // interior view
|
||||
'P3451', // night view
|
||||
'P8592', // aerial view
|
||||
'P4291', // panoramic view
|
||||
'P5252', // winter view
|
||||
'P948', // Wikivoyage banner
|
||||
] as const;
|
||||
|
||||
/** First non-empty string value of a claim list. */
|
||||
function claimValue(snaks: WikidataSnak[] | undefined): string | null {
|
||||
for (const snak of snaks ?? []) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value === 'string' && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* File names from an item's picture properties, best first.
|
||||
*
|
||||
* Within P18 the statement rank decides: an item with several images marks one
|
||||
* `preferred`, and that is the one an editor considers representative. The API
|
||||
* returns statements in edit order, not rank order, so taking `[0]` picks
|
||||
* whichever was added first — for the Brandenburg Gate that is a coin toss
|
||||
* between the morning shot and a wide overview.
|
||||
*/
|
||||
function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] {
|
||||
const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const;
|
||||
const names: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const property of WIKIDATA_IMAGE_PROPERTIES) {
|
||||
const snaks = [...(claims[property] ?? [])]
|
||||
.filter((snak) => snak.rank !== 'deprecated')
|
||||
.sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1));
|
||||
for (const snak of snaks) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value !== 'string' || !value.trim()) continue;
|
||||
const key = normalizeFileTitle(value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
names.push(value.trim());
|
||||
if (names.length >= limit) return names;
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */
|
||||
function normalizeFileTitle(title: string): string {
|
||||
return title.replace(/^File:/i, '').replace(/_/g, ' ').trim().toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* A bare Commons category name out of whatever an OSM `wikimedia_commons` tag
|
||||
* holds.
|
||||
*
|
||||
* The tag is free text and mappers put three different things in it: a bare
|
||||
* name, a prefixed `Category:…`, or — against the wiki's own advice — a single
|
||||
* `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`,
|
||||
* which matches nothing and fell through to the coordinate search without a
|
||||
* word. Localised prefixes (`Kategorie:`) appear too.
|
||||
*/
|
||||
function normalizeCategoryName(raw: string): string | null {
|
||||
const value = raw.trim();
|
||||
if (!value) return null;
|
||||
// A file name is not a category, and there is nothing sensible to derive.
|
||||
if (/^(file|datei|image|bild)\s*:/i.test(value)) return null;
|
||||
return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null;
|
||||
}
|
||||
|
||||
/** Picks the three identity tags out of a Nominatim `extratags` blob. */
|
||||
export function readWikiIdentity(extratags: Record<string, string> | null | undefined): WikiIdentity {
|
||||
const out: WikiIdentity = { wikipedia: null, wikidata: null, wikimedia_commons: null };
|
||||
if (!extratags) return out;
|
||||
for (const tag of WIKI_IDENTITY_TAGS) {
|
||||
const value = extratags[tag];
|
||||
if (typeof value === 'string' && value.trim()) out[tag] = value.trim();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface OverpassElement {
|
||||
@@ -61,10 +179,14 @@ interface OverpassElement {
|
||||
}
|
||||
|
||||
interface WikiCommonsPage {
|
||||
pageid?: number;
|
||||
title?: string;
|
||||
imageinfo?: {
|
||||
url?: string;
|
||||
thumburl?: string;
|
||||
mime?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
/** The file description page — where the full licence terms live. */
|
||||
descriptionurl?: string;
|
||||
extmetadata?: {
|
||||
@@ -72,6 +194,10 @@ interface WikiCommonsPage {
|
||||
LicenseShortName?: { value?: string };
|
||||
LicenseUrl?: { value?: string };
|
||||
UsageTerms?: { value?: string };
|
||||
/** Used to spot survey imagery and diagrams, which are not pictures of a place. */
|
||||
Categories?: { value?: string };
|
||||
ObjectName?: { value?: string };
|
||||
ImageDescription?: { value?: string };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
@@ -88,6 +214,20 @@ export interface CommonsCandidate {
|
||||
license: string | null;
|
||||
licenseUrl: string | null;
|
||||
sourceUrl: string | null;
|
||||
/**
|
||||
* Commons page id — the only stable identity a file has across the four ways
|
||||
* we reach it. The thumbnail URL is not: the same file comes back from
|
||||
* commons.wikimedia.org and from a language Wikipedia with different query
|
||||
* strings, so deduplicating on the URL silently lets the same picture through
|
||||
* twice. It is also what keys the cached bytes, so it must survive.
|
||||
*/
|
||||
pageId: number | null;
|
||||
/** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */
|
||||
title: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
/** Free text used to reject survey imagery, floor plans and logos. */
|
||||
descriptors: string | null;
|
||||
}
|
||||
|
||||
interface GooglePlaceResult {
|
||||
@@ -372,6 +512,11 @@ export class MapsService {
|
||||
q: query,
|
||||
format: 'json',
|
||||
addressdetails: '1',
|
||||
// Free, same request: this is where a place's wikidata/wikipedia/commons
|
||||
// tags live. Without them the enrichment column can only fall back to
|
||||
// "photos taken within 300m", which around a city centre is passers-by
|
||||
// and the neighbouring building.
|
||||
extratags: '1',
|
||||
limit: '10',
|
||||
'accept-language': toApiLang(lang),
|
||||
});
|
||||
@@ -402,6 +547,7 @@ export class MapsService {
|
||||
website: null,
|
||||
phone: null,
|
||||
source: 'openstreetmap',
|
||||
...readWikiIdentity(item.extratags),
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -417,11 +563,15 @@ export class MapsService {
|
||||
address: string;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
extratags: Record<string, string> | null;
|
||||
} | null> {
|
||||
const typePrefix = osmType.charAt(0).toUpperCase(); // N, W, R
|
||||
const params = new URLSearchParams({
|
||||
osm_ids: `${typePrefix}${osmId}`,
|
||||
format: 'json',
|
||||
// Overpass is the richer source but it is also the one that times out;
|
||||
// whatever Nominatim already knows costs nothing extra here.
|
||||
extratags: '1',
|
||||
'accept-language': toApiLang(lang),
|
||||
});
|
||||
try {
|
||||
@@ -439,6 +589,7 @@ export class MapsService {
|
||||
address: item.display_name || '',
|
||||
lat: Number.isFinite(lat) ? lat : null,
|
||||
lng: Number.isFinite(lng) ? lng : null,
|
||||
extratags: item.extratags ?? null,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
@@ -623,9 +774,13 @@ export class MapsService {
|
||||
ggsnamespace: '6',
|
||||
ggsradius: '300',
|
||||
ggscoord: `${lat}|${lng}`,
|
||||
ggslimit: String(Math.max(1, Math.min(limit, 20))),
|
||||
// Deliberately more than the caller asked for. Around anything worth
|
||||
// visiting the first few hits are survey tiles, passers-by and the
|
||||
// building next door; the ranker needs a pool to reject from, and
|
||||
// geosearch charges the same for one result as for twenty.
|
||||
ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))),
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
@@ -637,25 +792,30 @@ export class MapsService {
|
||||
});
|
||||
if (!res.ok) return [];
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
return this.toCommonsCandidates(data.query?.pages, limit);
|
||||
// Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets
|
||||
// what it always got; the enrichment column ranks before it cuts.
|
||||
return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit')));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared shaping for both Commons queries (by coordinate and by category). */
|
||||
/** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */
|
||||
private toCommonsCandidates(
|
||||
pages: Record<string, WikiCommonsPage> | undefined,
|
||||
limit: number,
|
||||
): CommonsCandidate[] {
|
||||
if (!pages) return [];
|
||||
const out: CommonsCandidate[] = [];
|
||||
for (const page of Object.values(pages)) {
|
||||
// entries(), not values(): the map key is the page id, and for the queries
|
||||
// that reach a file by title it is the only place the id appears.
|
||||
for (const [key, page] of Object.entries(pages)) {
|
||||
const info = page.imageinfo?.[0];
|
||||
// Only use actual photos (JPEG/PNG), skip SVGs and PDFs
|
||||
const mime = info?.mime || '';
|
||||
if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue;
|
||||
const meta = info.extmetadata;
|
||||
const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null);
|
||||
out.push({
|
||||
// iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it —
|
||||
// info.url is the full-resolution original (multi-megapixel camera exports).
|
||||
@@ -664,6 +824,17 @@ export class MapsService {
|
||||
license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value),
|
||||
licenseUrl: meta?.LicenseUrl?.value?.trim() || null,
|
||||
sourceUrl: info.descriptionurl || null,
|
||||
pageId: pageId && pageId > 0 ? pageId : null,
|
||||
title: page.title ?? null,
|
||||
width: info.width ?? null,
|
||||
height: info.height ?? null,
|
||||
descriptors: [
|
||||
stripWikiMarkup(meta?.ObjectName?.value),
|
||||
stripWikiMarkup(meta?.ImageDescription?.value),
|
||||
stripWikiMarkup(meta?.Categories?.value),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' | ') || null,
|
||||
});
|
||||
if (out.length >= limit) break;
|
||||
}
|
||||
@@ -730,50 +901,150 @@ export class MapsService {
|
||||
}
|
||||
|
||||
/**
|
||||
* The one picture Wikidata records for a place (property P18).
|
||||
* The pictures Wikidata records for a place, best first.
|
||||
*
|
||||
* By far the most accurate source there is: someone chose this image to
|
||||
* By far the most accurate source there is: a person chose each of these to
|
||||
* represent this exact object, where a coordinate search only knows what was
|
||||
* photographed nearby. Restaurants and small venues carry a `wikidata` tag far
|
||||
* more often than a Commons category, so this is also the source that reaches
|
||||
* the places the others miss.
|
||||
* photographed nearby. Wikidata also keeps them apart by what they show, so
|
||||
* asking for more than P18 buys genuine variety rather than another frame of
|
||||
* the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a
|
||||
* night shot, a panorama and a winter view, all curated.
|
||||
*
|
||||
* Two calls total whatever the item holds: one for the claims, one batch for
|
||||
* the file metadata.
|
||||
*/
|
||||
async fetchWikidataImage(wikidataId: string): Promise<CommonsCandidate | null> {
|
||||
if (!/^Q\d+$/.test(wikidataId.trim())) return null;
|
||||
async fetchWikidataCandidates(
|
||||
wikidataId: string,
|
||||
limit = 5,
|
||||
): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> {
|
||||
const empty = { candidates: [], commonsCategory: null };
|
||||
const qid = wikidataId.trim();
|
||||
if (!/^Q\d+$/.test(qid)) return empty;
|
||||
try {
|
||||
const claims = await fetch(
|
||||
`https://www.wikidata.org/w/api.php?action=wbgetclaims&entity=${wikidataId.trim()}&property=P18&format=json`,
|
||||
const res = await fetch(
|
||||
`https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`,
|
||||
{ headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) },
|
||||
);
|
||||
if (!claims.ok) return null;
|
||||
const data = (await claims.json()) as {
|
||||
claims?: { P18?: { mainsnak?: { datavalue?: { value?: string } } }[] };
|
||||
};
|
||||
const fileName = data.claims?.P18?.[0]?.mainsnak?.datavalue?.value;
|
||||
if (!fileName) return null;
|
||||
if (!res.ok) return empty;
|
||||
const data = (await res.json()) as { entities?: Record<string, { claims?: WikidataClaims }> };
|
||||
const claims = data.entities?.[qid]?.claims;
|
||||
if (!claims) return empty;
|
||||
|
||||
// Second hop for the URL and, more importantly, the licence — the claim
|
||||
// itself carries only a file name.
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: `File:${fileName}`,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
const file = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
const fileNames = wikidataImageClaims(claims, limit);
|
||||
const commonsCategory = claimValue(claims.P373) ?? null;
|
||||
if (fileNames.length === 0) return { candidates: [], commonsCategory };
|
||||
|
||||
const byTitle = await this.fetchCommonsFilesByName(fileNames);
|
||||
// Back into the order Wikidata implied, which the batch response loses.
|
||||
const candidates = fileNames.map((name) => byTitle.get(normalizeFileTitle(name))).filter((c): c is CommonsCandidate => !!c);
|
||||
return { candidates, commonsCategory };
|
||||
} catch {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for a list of Commons files, in one request.
|
||||
*
|
||||
* `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia
|
||||
* lead image often names a file that has since been renamed, and without it
|
||||
* the API answers with a `missing` page and the picture disappears silently.
|
||||
* Keyed by normalised title so callers can restore their own ordering.
|
||||
*/
|
||||
async fetchCommonsFilesByName(fileNames: string[]): Promise<Map<string, CommonsCandidate>> {
|
||||
const out = new Map<string, CommonsCandidate>();
|
||||
const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`));
|
||||
if (titles.length === 0) return out;
|
||||
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: titles.join('|'),
|
||||
redirects: '1',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!file.ok) return null;
|
||||
const fileData = (await file.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
return this.toCommonsCandidates(fileData.query?.pages, 1)[0] ?? null;
|
||||
if (!res.ok) return out;
|
||||
const data = (await res.json()) as {
|
||||
query?: {
|
||||
pages?: Record<string, WikiCommonsPage>;
|
||||
normalized?: { from: string; to: string }[];
|
||||
redirects?: { from: string; to: string }[];
|
||||
};
|
||||
};
|
||||
// The API renames titles twice on the way in (normalisation, then
|
||||
// redirects), so walk the chain back to what the caller asked for.
|
||||
const aliases = new Map<string, string>();
|
||||
for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) {
|
||||
aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from));
|
||||
}
|
||||
const resolveOriginal = (title: string): string => {
|
||||
let key = normalizeFileTitle(title);
|
||||
for (let hop = 0; hop < 4; hop++) {
|
||||
const previous = aliases.get(key);
|
||||
if (!previous || previous === key) break;
|
||||
key = previous;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) {
|
||||
if (!candidate.title) continue;
|
||||
out.set(resolveOriginal(candidate.title), candidate);
|
||||
// Also reachable under its own name, for callers that already resolved.
|
||||
out.set(normalizeFileTitle(candidate.title), candidate);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return null;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The lead image a wiki article picked for a place.
|
||||
*
|
||||
* Only the file NAME is taken from here; the bytes and the licence come from
|
||||
* the same Commons batch as everything else. The thumbnail URL the API offers
|
||||
* alongside it carries no attribution, and a picture we cannot credit is a
|
||||
* picture we cannot show.
|
||||
*/
|
||||
async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise<string | null> {
|
||||
const parsed = parseWikipediaTag(wikipediaTag);
|
||||
if (!parsed) return null;
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: parsed.title,
|
||||
prop: 'pageimages',
|
||||
piprop: 'name',
|
||||
redirects: '1',
|
||||
});
|
||||
for (const host of ['wikivoyage', 'wikipedia'] as const) {
|
||||
try {
|
||||
const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { pageimage?: string }> };
|
||||
};
|
||||
for (const page of Object.values(data.query?.pages ?? {})) {
|
||||
if (page.pageimage) return page.pageimage;
|
||||
}
|
||||
} catch {
|
||||
/* try the next wiki */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commons images from a category, which is the set of pictures OF a place.
|
||||
*
|
||||
@@ -782,29 +1053,56 @@ export class MapsService {
|
||||
* passers-by, while the category of a restaurant returns the restaurant.
|
||||
*/
|
||||
async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise<CommonsCandidate[]> {
|
||||
const title = category.startsWith('Category:') ? category : `Category:${category}`;
|
||||
const params = new URLSearchParams({
|
||||
const name = normalizeCategoryName(category);
|
||||
if (!name) return [];
|
||||
// Overfetch: the ranker throws away survey imagery, diagrams and repeats,
|
||||
// and it can only do that from a pool bigger than the strip.
|
||||
const poolSize = String(Math.max(1, Math.min(limit * 3, 20)));
|
||||
|
||||
// `generator=search` first. `categorymembers` orders by sort key, i.e.
|
||||
// alphabetically by file name, which is not a quality signal in any
|
||||
// direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation,
|
||||
// a marathon photo and six near-identical press shots, and
|
||||
// "Category:Hamburg Airport" with a noise map and a terminal layout. The
|
||||
// search index at least ranks by how well a file matches its category.
|
||||
const search = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'search',
|
||||
gsrsearch: `incategory:"${name}" filetype:bitmap`,
|
||||
gsrnamespace: '6',
|
||||
gsrlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
const members = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'categorymembers',
|
||||
gcmtitle: title,
|
||||
gcmtitle: `Category:${name}`,
|
||||
gcmtype: 'file',
|
||||
gcmlimit: String(Math.max(1, Math.min(limit * 2, 20))),
|
||||
gcmlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return [];
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
return this.toCommonsCandidates(data.query?.pages, limit);
|
||||
} catch {
|
||||
return [];
|
||||
|
||||
for (const params of [search, members]) {
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize));
|
||||
if (hits.length) return hits;
|
||||
} catch {
|
||||
/* fall through to the second strategy */
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1033,12 +1331,22 @@ export class MapsService {
|
||||
// OSM details: placeId is "node:123456" or "way:123456" etc.
|
||||
if (placeId.includes(':')) {
|
||||
const [osmType, osmId] = placeId.split(':');
|
||||
const element = await this.fetchOverpassDetails(osmType, osmId);
|
||||
const details = buildOsmDetails(element?.tags || {}, osmType, osmId);
|
||||
|
||||
// buildOsmDetails never yields name/address/coordinates — Nominatim is
|
||||
// always the source for those (Overpass contributes the tag-derived rest).
|
||||
const nominatim = await this.lookupNominatim(osmType, osmId, lang);
|
||||
const [element, nominatim] = await Promise.all([
|
||||
this.fetchOverpassDetails(osmType, osmId),
|
||||
this.lookupNominatim(osmType, osmId, lang),
|
||||
]);
|
||||
// Overpass has the fuller tag set and wins where both answer, but it is
|
||||
// also the one that goes down — overpass-api.de is regularly overloaded.
|
||||
// Nominatim's extratags carry the wikidata/wikipedia/commons tags too, so
|
||||
// a place keeps its pictures and its description when Overpass times out
|
||||
// instead of falling back to "photographed within 300m".
|
||||
const details = buildOsmDetails(
|
||||
{ ...(nominatim?.extratags ?? {}), ...(element?.tags ?? {}) },
|
||||
osmType,
|
||||
osmId,
|
||||
);
|
||||
|
||||
return {
|
||||
place: {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type {
|
||||
MapsPlaceEnrichmentRequest,
|
||||
@@ -8,8 +9,8 @@ import type {
|
||||
} from '@trek/shared';
|
||||
import { safeFetchFollow } from '../../utils/ssrfGuard';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { MapsService, withPhotoFetchSlot } from '../maps/maps.service';
|
||||
import { isGooglePlaceId } from '../maps/maps.helpers';
|
||||
import { MapsService, withPhotoFetchSlot, type CommonsCandidate } from '../maps/maps.service';
|
||||
import { isGooglePlaceId, rankCommonsCandidates } from '../maps/maps.helpers';
|
||||
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
|
||||
|
||||
/**
|
||||
@@ -37,11 +38,40 @@ const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
* from before the change — which is exactly what happened when the facts and
|
||||
* Wikivoyage landed.
|
||||
*/
|
||||
const CACHE_VERSION = 2;
|
||||
const CACHE_VERSION = 3;
|
||||
|
||||
/** Cache key for one candidate picture. Deliberately not a bare place id. */
|
||||
function candidateKey(placeId: string, index: number): string {
|
||||
return `${placeId}~p${index}`;
|
||||
/**
|
||||
* Cache key for one candidate picture. Deliberately not a bare place id.
|
||||
*
|
||||
* Keyed by which picture it is, not by where it sat in the strip. Position was
|
||||
* the obvious choice and it was wrong: the ladder below returns different
|
||||
* numbers of candidates depending on which providers answered, so a slot that
|
||||
* held a Wikidata photo one minute held a category photo the next — and the
|
||||
* cache hands back whatever bytes it stored for that slot while the response
|
||||
* carries the new candidate's author. Under CC BY that is not a stale
|
||||
* thumbnail, it is the wrong person credited for someone's work.
|
||||
*
|
||||
* Hashed because the identity can be a Google resource path with slashes, and
|
||||
* this ends up in a URL.
|
||||
*/
|
||||
export function candidateKey(placeId: string, identity: string): string {
|
||||
const digest = createHash('sha1').update(identity).digest('hex').slice(0, 12);
|
||||
return `${placeId}~p${digest}`;
|
||||
}
|
||||
|
||||
/** A ranked Commons pick, tagged with the rung of the ladder it came from. */
|
||||
type CommonsPick = CommonsCandidate & { rung: 'wikidata' | 'wikipedia' | 'category' | 'nearby' };
|
||||
|
||||
interface PendingCandidate {
|
||||
/** Stable identity of the underlying file, for the cache key. */
|
||||
identity: string;
|
||||
candidate: PlacePhotoCandidate;
|
||||
fetchBytes: () => Promise<Buffer | null>;
|
||||
}
|
||||
|
||||
/** Appends candidates to the pool, remembering which rung produced them. */
|
||||
function push(pool: CommonsPick[], candidates: CommonsCandidate[], rung: CommonsPick['rung']): void {
|
||||
for (const candidate of candidates) pool.push({ ...candidate, rung });
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -207,53 +237,95 @@ export class PlaceEnrichmentService {
|
||||
const apiKey = this.maps.getMapsKey(userId);
|
||||
const wantsGoogle = !!apiKey && !this.maps.photosDisabled() && isGooglePlaceId(placeId);
|
||||
|
||||
// Ask both providers at once — one listing call each, and they know nothing
|
||||
// about one another.
|
||||
// Accuracy, best first. Wikidata's P18 is one picture somebody chose to
|
||||
// represent this exact object. A Commons category is pictures of it. The
|
||||
// coordinate search is only pictures NEAR it, which around a city centre
|
||||
// means statues and passers-by — it is the last resort, not the plan.
|
||||
const category = typeof details?.wikimedia_commons === 'string' ? details.wikimedia_commons : null;
|
||||
const wikidata = typeof details?.wikidata === 'string' ? details.wikidata : null;
|
||||
const tag = (key: string): string | null => (typeof details?.[key] === 'string' ? (details[key] as string) : null);
|
||||
const wikidata = tag('wikidata');
|
||||
const wikipedia = tag('wikipedia');
|
||||
|
||||
const [googleRefs, wikidataImage, commons] = await Promise.all([
|
||||
wantsGoogle ? this.maps.fetchGooglePhotoRefs(placeId, apiKey!, GOOGLE_CAP) : Promise.resolve([]),
|
||||
wikidata ? this.maps.fetchWikidataImage(wikidata) : Promise.resolve(null),
|
||||
category
|
||||
? this.maps
|
||||
.fetchCommonsCategoryCandidates(category, COMMONS_CAP)
|
||||
.then((hits) => (hits.length ? hits : this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP)))
|
||||
: this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP),
|
||||
]);
|
||||
// Google's listing is one call for the whole strip and knows nothing about
|
||||
// the free sources, so it runs alongside them rather than in the ladder.
|
||||
const googlePending = wantsGoogle
|
||||
? this.maps.fetchGooglePhotoRefs(placeId, apiKey!, GOOGLE_CAP)
|
||||
: Promise.resolve([] as { name: string; attribution: string | null }[]);
|
||||
|
||||
// The Wikidata picture leads, and is dropped from the rest of the list when
|
||||
// a category happens to contain it too.
|
||||
const commonsList = wikidataImage
|
||||
? [wikidataImage, ...commons.filter((c) => c.photoUrl !== wikidataImage.photoUrl)].slice(0, COMMONS_CAP)
|
||||
: commons;
|
||||
// The free ladder, in order of how much anyone vouched that the picture
|
||||
// shows THIS place:
|
||||
// Wikidata — a person attached this file to this exact object.
|
||||
// Wiki lead — the article about it opens with this picture.
|
||||
// Category — the set of pictures of it.
|
||||
// Nearby — anything photographed within 300m.
|
||||
// Only the last one has no claim on the subject at all, which is how an
|
||||
// airport ended up represented by aerial survey tiles of its runway and a
|
||||
// gate by the underground station beneath it. It is the fallback, and it
|
||||
// only runs when the ladder above came up short.
|
||||
const commonsPool: CommonsPick[] = [];
|
||||
let categoryName = tag('wikimedia_commons');
|
||||
|
||||
// Google first — its pictures are of the place itself, while Commons
|
||||
// geosearch finds whatever happens to have been photographed nearby.
|
||||
const pending: PlacePhotoCandidate[] = [
|
||||
if (wikidata) {
|
||||
const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP);
|
||||
push(commonsPool, fromWikidata.candidates, 'wikidata');
|
||||
categoryName ??= fromWikidata.commonsCategory;
|
||||
}
|
||||
|
||||
if (commonsPool.length < COMMONS_CAP && wikipedia) {
|
||||
const leadName = await this.maps.fetchWikiLeadImageName(wikipedia);
|
||||
if (leadName) {
|
||||
const byName = await this.maps.fetchCommonsFilesByName([leadName]);
|
||||
push(commonsPool, [...byName.values()], 'wikipedia');
|
||||
}
|
||||
}
|
||||
|
||||
if (commonsPool.length < COMMONS_CAP && categoryName) {
|
||||
push(commonsPool, await this.maps.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category');
|
||||
}
|
||||
|
||||
// Two is the bar: one curated picture plus the nearby noise reads worse
|
||||
// than one curated picture on its own.
|
||||
const curated = commonsPool.length;
|
||||
// Started here, decided below. Waiting for Google's listing first would put
|
||||
// two round trips end to end, and that is precisely the sequencing that
|
||||
// used to push this endpoint past the client's timeout. Geosearch is free
|
||||
// and unmetered, so a speculative call that gets discarded costs nothing
|
||||
// anyone pays for.
|
||||
const nearbyPending =
|
||||
curated < 2 ? this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP) : Promise.resolve([]);
|
||||
|
||||
const googleRefs = await googlePending;
|
||||
if (curated < 2 && googleRefs.length < GOOGLE_CAP) {
|
||||
push(commonsPool, await nearbyPending, 'nearby');
|
||||
}
|
||||
|
||||
// One ranking pass over everything: the same file reaches us from several
|
||||
// rungs and only the page id catches that.
|
||||
const ranked = rankCommonsCandidates(commonsPool, COMMONS_CAP, { perAuthor: curated > 0 ? 2 : 1 });
|
||||
// Google leads the strip — its pictures are of the business itself.
|
||||
const pending: PendingCandidate[] = [
|
||||
...googleRefs.map((ref) => ({
|
||||
key: '',
|
||||
url: '',
|
||||
attribution: ref.attribution,
|
||||
// Google grants no reusable licence for these; the author line it hands
|
||||
// back is all we may show, so the rest stays honestly empty.
|
||||
license: null,
|
||||
licenseUrl: null,
|
||||
sourceUrl: null,
|
||||
source: 'google' as const,
|
||||
identity: `google:${ref.name}`,
|
||||
candidate: {
|
||||
key: '',
|
||||
url: '',
|
||||
attribution: ref.attribution,
|
||||
// Google grants no reusable licence for these; the author line it
|
||||
// hands back is all we may show, so the rest stays honestly empty.
|
||||
license: null,
|
||||
licenseUrl: null,
|
||||
sourceUrl: null,
|
||||
source: 'google' as const,
|
||||
},
|
||||
fetchBytes: () => this.maps.fetchGooglePhotoBytes(ref.name, apiKey!),
|
||||
})),
|
||||
...commonsList.map((candidate) => ({
|
||||
key: '',
|
||||
url: '',
|
||||
attribution: candidate.attribution,
|
||||
license: candidate.license,
|
||||
licenseUrl: candidate.licenseUrl,
|
||||
sourceUrl: candidate.sourceUrl,
|
||||
source: 'wikimedia' as const,
|
||||
...ranked.map((pick) => ({
|
||||
identity: `commons:${pick.pageId ?? pick.photoUrl}`,
|
||||
candidate: {
|
||||
key: '',
|
||||
url: '',
|
||||
attribution: pick.attribution,
|
||||
license: pick.license,
|
||||
licenseUrl: pick.licenseUrl,
|
||||
sourceUrl: pick.sourceUrl,
|
||||
source: (pick.rung === 'wikipedia' ? 'wikipedia' : 'wikimedia') as 'wikipedia' | 'wikimedia',
|
||||
},
|
||||
fetchBytes: () => this.fetchRemoteBytes(pick.photoUrl),
|
||||
})),
|
||||
];
|
||||
|
||||
@@ -262,16 +334,9 @@ export class PlaceEnrichmentService {
|
||||
// which pushed the whole request past the client's request timeout. The
|
||||
// shared slot limiter still caps how many run at once.
|
||||
const results = await Promise.all(
|
||||
pending.map(async (candidate, index) => {
|
||||
const key = candidateKey(placeId, index);
|
||||
const fetchBytes =
|
||||
candidate.source === 'google'
|
||||
? () => this.maps.fetchGooglePhotoBytes(googleRefs[index].name, apiKey!)
|
||||
: () => this.fetchRemoteBytes(commonsList[index - googleRefs.length].photoUrl);
|
||||
pending.map(async ({ identity, candidate, fetchBytes }) => {
|
||||
const key = candidateKey(placeId, identity);
|
||||
const url = await this.storeCandidate(key, creditLine(candidate.attribution, candidate.license), fetchBytes);
|
||||
// The index is fixed before any download, so a picture that fails to
|
||||
// load leaves a gap rather than renumbering the ones after it — the key
|
||||
// has to keep pointing at the same file across requests.
|
||||
return url ? { ...candidate, key, url } : null;
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -30,6 +30,7 @@ const { db } = vi.hoisted(() => {
|
||||
vi.mock('../../src/db/database', () => ({ db, closeDb: () => {}, reinitialize: () => {} }));
|
||||
|
||||
import { PlaceEnrichmentModule } from '../../src/nest/place-enrichment/place-enrichment.module';
|
||||
import { candidateKey } from '../../src/nest/place-enrichment/place-enrichment.service';
|
||||
import { MapsService } from '../../src/nest/maps/maps.service';
|
||||
import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service';
|
||||
import { DatabaseModule } from '../../src/nest/database/database.module';
|
||||
@@ -123,6 +124,11 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
license: 'CC BY-SA 4.0',
|
||||
licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/',
|
||||
sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg',
|
||||
pageId: 4711,
|
||||
title: 'File:X.jpg',
|
||||
width: 1600,
|
||||
height: 1200,
|
||||
descriptors: null,
|
||||
},
|
||||
]);
|
||||
// The bytes download goes through the SSRF-guarded fetch — stub the network.
|
||||
@@ -133,7 +139,8 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.photos).toHaveLength(1);
|
||||
expect(res.body.photos[0]).toMatchObject({
|
||||
key: 'way:12345~p0',
|
||||
// Keyed by the file's Commons page id, not by its slot in the strip.
|
||||
key: candidateKey('way:12345', 'commons:4711'),
|
||||
attribution: 'Alice',
|
||||
license: 'CC BY-SA 4.0',
|
||||
source: 'wikimedia',
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* MAPS-147..156 — the curated picture sources behind the place detail column.
|
||||
*
|
||||
* These are the rungs above "photographed within 300m": the Wikidata item, the
|
||||
* article's lead image, and the Commons batch that turns a file name into
|
||||
* something creditable. Payload shapes are taken from live responses for Q1097
|
||||
* (Berlin Hauptbahnhof), Q27706 (Hamburg Airport) and Q82425 (Brandenburg
|
||||
* Gate) — the three places whose pictures were wrong.
|
||||
*
|
||||
* fetch is stubbed; the DB is mocked the same way maps.service.test.ts does it.
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('../../../src/db/database', () => ({
|
||||
db: { prepare: () => ({ get: () => undefined, run: () => undefined, all: () => [] }) },
|
||||
}));
|
||||
|
||||
vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) }));
|
||||
|
||||
vi.mock('../../../src/utils/ssrfGuard', () => ({
|
||||
safeFetchFollow: vi.fn(),
|
||||
checkSsrf: vi.fn(async () => ({ allowed: true })),
|
||||
SsrfBlockedError: class extends Error {},
|
||||
}));
|
||||
|
||||
import { db } from '../../../src/db/database';
|
||||
import { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
import { MapsService } from '../../../src/nest/maps/maps.service';
|
||||
|
||||
const svcOf = () => new MapsService(new DatabaseService(db as never), {} as never);
|
||||
|
||||
const filePage = (over: Record<string, unknown> = {}) => ({
|
||||
pageid: 4711,
|
||||
title: 'File:Outside.jpg',
|
||||
imageinfo: [
|
||||
{
|
||||
url: 'https://commons.org/full.jpg',
|
||||
thumburl: 'https://commons.org/thumb.jpg',
|
||||
mime: 'image/jpeg',
|
||||
width: 1600,
|
||||
height: 1200,
|
||||
descriptionurl: 'https://commons.wikimedia.org/wiki/File:X.jpg',
|
||||
extmetadata: { Artist: { value: 'Alice' }, LicenseShortName: { value: 'CC BY-SA 4.0' } },
|
||||
},
|
||||
],
|
||||
...over,
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('fetchWikidataCandidates', () => {
|
||||
it('MAPS-147: refuses anything that is not a Q-id without calling out', async () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svcOf().fetchWikidataCandidates('not-a-qid')).toEqual({ candidates: [], commonsCategory: null });
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('MAPS-148: puts the preferred P18 statement ahead of a normal one', async () => {
|
||||
// The API returns statements in edit order, not rank order, so reading the
|
||||
// first one is a coin toss between the picture an editor chose to represent
|
||||
// the place and whatever happened to be added first.
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
entities: {
|
||||
Q82425: {
|
||||
claims: {
|
||||
P18: [
|
||||
{ mainsnak: { datavalue: { value: 'Berlin-Brandenburg Gate overview.jpg' } }, rank: 'normal' },
|
||||
{ mainsnak: { datavalue: { value: 'Brandenburger Tor morgens.jpg' } }, rank: 'preferred' },
|
||||
],
|
||||
P373: [{ mainsnak: { datavalue: { value: 'Brandenburg Gate' } } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svcOf().fetchWikidataCandidates('Q82425');
|
||||
|
||||
const batchUrl = decodeURIComponent(String(fetchMock.mock.calls[1][0]));
|
||||
expect(batchUrl.indexOf('Brandenburger Tor morgens')).toBeLessThan(batchUrl.indexOf('overview'));
|
||||
// P373 comes back too, so a place with no wikimedia_commons tag still finds
|
||||
// its category — Berlin Hauptbahnhof is exactly that case.
|
||||
expect(out.commonsCategory).toBe('Brandenburg Gate');
|
||||
});
|
||||
|
||||
it('MAPS-149: ignores a deprecated statement', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
entities: { Q1: { claims: { P18: [{ mainsnak: { datavalue: { value: 'Wrong.jpg' } }, rank: 'deprecated' }] } } },
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect((await svcOf().fetchWikidataCandidates('Q1')).candidates).toEqual([]);
|
||||
// Nothing worth asking Commons about, so Commons is not asked.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('MAPS-150: gathers the other view properties, not just P18', async () => {
|
||||
// A station is not one view. P5775 (interior), P3451 (night) and P8592
|
||||
// (aerial) are what make the strip genuinely different pictures rather than
|
||||
// four takes on the same facade.
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
entities: {
|
||||
Q1097: {
|
||||
claims: {
|
||||
P18: [{ mainsnak: { datavalue: { value: 'Outside.jpg' } } }],
|
||||
P5775: [{ mainsnak: { datavalue: { value: 'Inside.jpg' } } }],
|
||||
P3451: [{ mainsnak: { datavalue: { value: 'AtNight.jpg' } } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svcOf().fetchWikidataCandidates('Q1097');
|
||||
|
||||
const batchUrl = decodeURIComponent(String(fetchMock.mock.calls[1][0]));
|
||||
expect(batchUrl).toContain('File:Outside.jpg|File:Inside.jpg|File:AtNight.jpg');
|
||||
});
|
||||
|
||||
it('MAPS-151: hands the files back in the order the claims implied', async () => {
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
entities: {
|
||||
Q1: {
|
||||
claims: {
|
||||
P18: [{ mainsnak: { datavalue: { value: 'First.jpg' } } }],
|
||||
P5775: [{ mainsnak: { datavalue: { value: 'Second.jpg' } } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
// Deliberately the other way round: a batch response does not preserve
|
||||
// the order the titles went in.
|
||||
json: async () => ({
|
||||
query: {
|
||||
pages: {
|
||||
'2': filePage({ pageid: 2, title: 'File:Second.jpg' }),
|
||||
'1': filePage({ pageid: 1, title: 'File:First.jpg' }),
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svcOf().fetchWikidataCandidates('Q1');
|
||||
expect(out.candidates.map((c) => c.title)).toEqual(['File:First.jpg', 'File:Second.jpg']);
|
||||
});
|
||||
|
||||
it('MAPS-152: survives a bad response and a throw', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect(await svcOf().fetchWikidataCandidates('Q1')).toEqual({ candidates: [], commonsCategory: null });
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svcOf().fetchWikidataCandidates('Q1')).toEqual({ candidates: [], commonsCategory: null });
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchCommonsFilesByName', () => {
|
||||
it('MAPS-153: follows a rename back to the name the caller asked for', async () => {
|
||||
// Wikidata claims and article lead images routinely name files that have
|
||||
// since been renamed. Without redirects the API answers with a `missing`
|
||||
// page and the picture disappears without a word.
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
query: {
|
||||
normalized: [{ from: 'File:old_name.jpg', to: 'File:Old name.jpg' }],
|
||||
redirects: [{ from: 'File:Old name.jpg', to: 'File:New name.jpg' }],
|
||||
pages: { '9': filePage({ pageid: 9, title: 'File:New name.jpg' }) },
|
||||
},
|
||||
}),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svcOf().fetchCommonsFilesByName(['old_name.jpg']);
|
||||
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('redirects=1');
|
||||
expect(out.get('old name.jpg')?.pageId).toBe(9);
|
||||
// And under its current name, for callers that already resolved it.
|
||||
expect(out.get('new name.jpg')?.pageId).toBe(9);
|
||||
});
|
||||
|
||||
it('MAPS-154: asks nothing for an empty list', async () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect((await svcOf().fetchCommonsFilesByName([])).size).toBe(0);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('MAPS-154b: returns an empty map on a bad response or a throw', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect((await svcOf().fetchCommonsFilesByName(['X.jpg'])).size).toBe(0);
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect((await svcOf().fetchCommonsFilesByName(['X.jpg'])).size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchWikiLeadImageName', () => {
|
||||
it('MAPS-155: asks Wikivoyage before Wikipedia and returns only the file name', async () => {
|
||||
// Only the name. The thumbnail URL the API offers alongside carries no
|
||||
// author, and a picture that cannot be credited cannot be shown.
|
||||
const fetchMock = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: true, json: async () => ({ query: { pages: { '1': {} } } }) })
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
json: async () => ({ query: { pages: { '2': { pageimage: 'Hamburg airport terminals.jpg' } } } }),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const name = await svcOf().fetchWikiLeadImageName('de:Flughafen Hamburg');
|
||||
|
||||
expect(name).toBe('Hamburg airport terminals.jpg');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('de.wikivoyage.org');
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('de.wikipedia.org');
|
||||
});
|
||||
|
||||
it('MAPS-156: yields null without a tag and survives a throw', async () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
expect(await svcOf().fetchWikiLeadImageName(null)).toBeNull();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svcOf().fetchWikiLeadImageName('de:X')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,8 @@ import {
|
||||
resolveOverpassTimeoutMs,
|
||||
stripWikiMarkup,
|
||||
parseWikipediaTag,
|
||||
rankCommonsCandidates,
|
||||
type RankableCommonsCandidate,
|
||||
} from '../../../src/nest/maps/maps.helpers';
|
||||
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
|
||||
@@ -104,7 +106,7 @@ const photoCacheStub = {
|
||||
|
||||
import { db } from '../../../src/db/database';
|
||||
import { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
import { MapsService, withPhotoFetchSlot } from '../../../src/nest/maps/maps.service';
|
||||
import { MapsService, withPhotoFetchSlot, readWikiIdentity } from '../../../src/nest/maps/maps.service';
|
||||
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
|
||||
|
||||
// The service under test, constructed over the mocked db stub — DatabaseService
|
||||
@@ -2343,13 +2345,16 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
);
|
||||
const out = await svc.fetchCommonsCandidates(48.8, 2.3, 5);
|
||||
expect(out).toHaveLength(2);
|
||||
expect(out[0]).toEqual({
|
||||
expect(out[0]).toMatchObject({
|
||||
photoUrl: 'https://commons.org/thumb.jpg',
|
||||
attribution: 'Alice',
|
||||
license: 'CC BY-SA 4.0',
|
||||
licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/',
|
||||
sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg',
|
||||
});
|
||||
// The page id travels with the candidate: it is what identifies the file
|
||||
// across the four routes that can reach it, and what keys its cached bytes.
|
||||
expect(out[0].pageId).toBe(1);
|
||||
});
|
||||
|
||||
it('MAPS-118: falls back to UsageTerms when there is no short licence name', async () => {
|
||||
@@ -2397,18 +2402,22 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
expect(out[0].photoUrl).toBe('https://commons.org/original.jpg');
|
||||
});
|
||||
|
||||
it('MAPS-120: honours the limit and clamps it into the range Commons accepts', async () => {
|
||||
it('MAPS-120: asks for more than the strip needs and clamps to what Commons accepts', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ query: { pages: { '1': page(), '2': page(), '3': page() } } }),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(2);
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=2');
|
||||
// Overfetch on purpose: the first hits around anything worth visiting are
|
||||
// survey tiles and the building next door, and the ranker can only reject
|
||||
// from a pool. Geosearch bills the same for one result as for twenty, so
|
||||
// the whole pool comes back and gets cut after ranking, not before.
|
||||
expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3);
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=8');
|
||||
|
||||
await svc.fetchCommonsCandidates(48.8, 2.3, 0);
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=1');
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=8');
|
||||
|
||||
await svc.fetchCommonsCandidates(48.8, 2.3, 99);
|
||||
expect(String(fetchMock.mock.calls[2][0])).toContain('ggslimit=20');
|
||||
@@ -2534,15 +2543,40 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => {
|
||||
const out = await svc.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/t.jpg', attribution: 'Alice', license: 'CC BY 4.0' });
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig');
|
||||
// Ranked search, not the category listing: `categorymembers` orders by sort
|
||||
// key, i.e. alphabetically by file name, and that put an .ogg pronunciation
|
||||
// and six near-identical press shots at the head of the Brandenburg Gate
|
||||
// category. The search index at least ranks by how well a file matches.
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('gsrsearch=incategory%3A%22Museum+Ludwig%22');
|
||||
});
|
||||
|
||||
it('MAPS-125d: adds the Category prefix when the tag omits it', async () => {
|
||||
it('MAPS-125d: takes the bare category name whether or not the tag is prefixed', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svc.fetchCommonsCategoryCandidates('Museum Ludwig');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22');
|
||||
// Empty search falls through to the category listing as a second chance.
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig');
|
||||
});
|
||||
|
||||
it('MAPS-125f: refuses a tag that names a file rather than a category', async () => {
|
||||
// Mappers do put `File:…` in wikimedia_commons. Prefixing it blindly asked
|
||||
// Commons for `Category:File:X.jpg`, which matches nothing and fell through
|
||||
// to the coordinate search without a word.
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svc.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('MAPS-125g: strips a localised category prefix', async () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svc.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22');
|
||||
});
|
||||
|
||||
it('MAPS-125e: yields nothing on an error response or a throw', async () => {
|
||||
@@ -2706,3 +2740,139 @@ describe('withPhotoFetchSlot', () => {
|
||||
await expect(withPhotoFetchSlot(async () => 'free')).resolves.toBe('free');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* MAPS-135..146 — the Commons ranker and the identity tags.
|
||||
*
|
||||
* Every fixture here is a real payload recorded while working out why the
|
||||
* picker showed runways and train front-ends for Hamburg Airport, Berlin
|
||||
* Hauptbahnhof and the Brandenburg Gate.
|
||||
*/
|
||||
describe('rankCommonsCandidates', () => {
|
||||
const pic = (over: Partial<RankableCommonsCandidate> & { pageId: number }): RankableCommonsCandidate => ({
|
||||
title: `File:pic ${over.pageId}.jpg`,
|
||||
attribution: 'Someone',
|
||||
width: 1600,
|
||||
height: 1200,
|
||||
descriptors: null,
|
||||
photoUrl: `https://commons/thumb/${over.pageId}.jpg`,
|
||||
...over,
|
||||
});
|
||||
|
||||
it('MAPS-135: drops a file that reached us twice under different thumbnail urls', () => {
|
||||
const first = pic({ pageId: 47529341, photoUrl: 'https://commons/x.jpg?utm_source=commons.wikimedia.org' });
|
||||
const again = { ...first, photoUrl: 'https://commons/x.jpg?utm_source=de.wikipedia.org' };
|
||||
expect(rankCommonsCandidates([first, again], 5)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('MAPS-136: rejects the aerial survey tiles that fill an airport geosearch', () => {
|
||||
const tiles = [2013, 2015, 2016, 2019].map((year, i) =>
|
||||
pic({
|
||||
pageId: 100 + i,
|
||||
title: `File:Dop20rgb 32565 5943 ${year}.jpg`,
|
||||
attribution: 'Landesbetrieb Geoinformation Hamburg',
|
||||
width: 5000,
|
||||
height: 5000,
|
||||
descriptors: `Orthophoto Sommerbefliegung ${year}`,
|
||||
}),
|
||||
);
|
||||
expect(rankCommonsCandidates(tiles, 5)).toEqual([]);
|
||||
});
|
||||
|
||||
it('MAPS-137: rejects noise maps and terminal layouts from a category', () => {
|
||||
const docs = [
|
||||
pic({ pageId: 200, title: 'File:Lärmkarte Flughafen Hamburg.png' }),
|
||||
pic({ pageId: 201, title: 'File:EDDH HAM Layout.png' }),
|
||||
pic({ pageId: 202, title: 'File:Grundriss Terminal 1.jpg' }),
|
||||
];
|
||||
expect(rankCommonsCandidates(docs, 5)).toEqual([]);
|
||||
});
|
||||
|
||||
it('MAPS-138: keeps one frame of a camera burst, not four', () => {
|
||||
// The four shots that filled the Brandenburg Gate picker were one person
|
||||
// walking through a station concourse.
|
||||
const burst = [807, 808, 809, 810].map((n) =>
|
||||
pic({ pageId: 300 + n, title: `File:Hauptbahnhof Berlin interior 0${n}.jpg`, attribution: 'Dosseman' }),
|
||||
);
|
||||
const kept = rankCommonsCandidates(burst, 5);
|
||||
expect(kept).toHaveLength(1);
|
||||
expect(kept[0].title).toBe('File:Hauptbahnhof Berlin interior 0807.jpg');
|
||||
});
|
||||
|
||||
it('MAPS-139: collapses a press set numbered with trailing counters', () => {
|
||||
const set = [16, 17, 18, 19, 20, 21].map((n) =>
|
||||
pic({ pageId: 400 + n, title: `File:Alexander Schallenberg in Berlin on 7 June 2024 - ${n}.jpg` }),
|
||||
);
|
||||
expect(rankCommonsCandidates(set, 5)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('MAPS-140: caps how much of the strip one author may supply', () => {
|
||||
// Unrelated titles, same photographer — the series rule cannot see these,
|
||||
// only the author cap can. On the coordinate rung nothing vouches that the
|
||||
// pictures are even of the right subject, so variety is the only defence
|
||||
// and one contributor gets one slot; a curated rung gets two.
|
||||
const same = ['Rear entrance', 'Platform at dusk', 'Ticket hall'].map((subject, i) =>
|
||||
pic({ pageId: 500 + i, title: `File:${subject}.jpg`, attribution: 'Bahnthaler' }),
|
||||
);
|
||||
expect(rankCommonsCandidates(same, 5, { perAuthor: 1 })).toHaveLength(1);
|
||||
expect(rankCommonsCandidates(same, 5, { perAuthor: 2 })).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('MAPS-140b: the series rule catches a numbered pair before the author cap does', () => {
|
||||
// "Berlin-Hamburg-Express 1/2.JPG", both by Bahnthaler — two files, one
|
||||
// subject. This is the pair that filled the Berlin Hauptbahnhof picker.
|
||||
const pair = [1, 2].map((n) =>
|
||||
pic({ pageId: 510 + n, title: `File:Berlin-Hamburg-Express ${n}.JPG`, attribution: 'Bahnthaler' }),
|
||||
);
|
||||
expect(rankCommonsCandidates(pair, 5, { perAuthor: 2 })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('MAPS-141: rejects panorama strips that would crop to nothing in a square tile', () => {
|
||||
expect(rankCommonsCandidates([pic({ pageId: 600, width: 9000, height: 1200 })], 5)).toEqual([]);
|
||||
});
|
||||
|
||||
it('MAPS-142: keeps a large square photo that is not a survey tile', () => {
|
||||
const square = pic({ pageId: 601, width: 1200, height: 1200 });
|
||||
expect(rankCommonsCandidates([square], 5)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('MAPS-143: falls back to the url when a candidate carries no page id', () => {
|
||||
const noId = { ...pic({ pageId: 0 }), pageId: null };
|
||||
expect(rankCommonsCandidates([noId, { ...noId }], 5)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('MAPS-144: stops at the limit', () => {
|
||||
const many = Array.from({ length: 9 }, (_, i) =>
|
||||
pic({ pageId: 700 + i, title: `File:Distinct subject ${String.fromCharCode(97 + i)}.jpg`, attribution: `Author ${i}` }),
|
||||
);
|
||||
expect(rankCommonsCandidates(many, 3)).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readWikiIdentity', () => {
|
||||
it('MAPS-145: reads the three identity tags and nothing else', () => {
|
||||
expect(
|
||||
readWikiIdentity({
|
||||
wikidata: 'Q82425',
|
||||
wikipedia: 'de:Brandenburger Tor',
|
||||
wikimedia_commons: 'Category:Brandenburg Gate',
|
||||
cuisine: 'italian',
|
||||
}),
|
||||
).toEqual({
|
||||
wikidata: 'Q82425',
|
||||
wikipedia: 'de:Brandenburger Tor',
|
||||
wikimedia_commons: 'Category:Brandenburg Gate',
|
||||
});
|
||||
});
|
||||
|
||||
it('MAPS-146: ignores brand tags, which describe the chain and not the branch', () => {
|
||||
// Real payload for "L'Osteria Rostock": following brand:wikidata would
|
||||
// describe (and illustrate) L'Osteria the company.
|
||||
const identity = readWikiIdentity({
|
||||
'brand:wikidata': 'Q17323478',
|
||||
'brand:wikipedia': 'de:L’Osteria',
|
||||
cuisine: 'pizza',
|
||||
});
|
||||
expect(identity).toEqual({ wikidata: null, wikipedia: null, wikimedia_commons: null });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -36,7 +36,7 @@ vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KE
|
||||
|
||||
import { db } from '../../../src/db/database';
|
||||
import { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
import { PlaceEnrichmentService, creditLine, collectFacts } from '../../../src/nest/place-enrichment/place-enrichment.service';
|
||||
import { candidateKey, PlaceEnrichmentService, creditLine, collectFacts } from '../../../src/nest/place-enrichment/place-enrichment.service';
|
||||
import type { MapsService } from '../../../src/nest/maps/maps.service';
|
||||
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
|
||||
|
||||
@@ -55,6 +55,9 @@ function mapsStub(over: Partial<Record<keyof MapsService, unknown>> = {}) {
|
||||
fetchEditorialSummary: vi.fn(async () => null as string | null),
|
||||
fetchWikiExtract: vi.fn(async () => null as { text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null),
|
||||
fetchCommonsCategoryCandidates: vi.fn(async () => [] as any[]),
|
||||
fetchWikidataCandidates: vi.fn(async () => ({ candidates: [] as any[], commonsCategory: null as string | null })),
|
||||
fetchCommonsFilesByName: vi.fn(async () => new Map<string, any>()),
|
||||
fetchWikiLeadImageName: vi.fn(async () => null as string | null),
|
||||
details: vi.fn(async () => ({ place: null })),
|
||||
...over,
|
||||
} as unknown as MapsService;
|
||||
@@ -76,16 +79,31 @@ function make(maps: MapsService, cache: PlacePhotoCacheService) {
|
||||
return new PlaceEnrichmentService(new DatabaseService(db as never), maps, cache);
|
||||
}
|
||||
|
||||
const commonsCandidate = (over: Record<string, unknown> = {}) => ({
|
||||
photoUrl: 'https://commons.org/thumb.jpg',
|
||||
attribution: 'Alice',
|
||||
license: 'CC BY-SA 4.0',
|
||||
licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/',
|
||||
sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg',
|
||||
...over,
|
||||
});
|
||||
let candidateSeq = 0;
|
||||
const commonsCandidate = (over: Record<string, unknown> = {}) => {
|
||||
// Distinct by default: the ranker deduplicates by page id and collapses
|
||||
// same-stem titles, so fixtures that share either would silently collapse
|
||||
// into one and make a test about ordering pass for the wrong reason.
|
||||
const n = ++candidateSeq;
|
||||
return {
|
||||
photoUrl: 'https://commons.org/thumb.jpg',
|
||||
attribution: 'Alice',
|
||||
license: 'CC BY-SA 4.0',
|
||||
licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/',
|
||||
sourceUrl: 'https://commons.wikimedia.org/wiki/File:X.jpg',
|
||||
pageId: 1000 + n,
|
||||
title: `File:Subject ${String.fromCharCode(96 + n)}.jpg`,
|
||||
width: 1600,
|
||||
height: 1200,
|
||||
descriptors: null,
|
||||
...over,
|
||||
};
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
// Per test, so a fixture's page id is a property of the case rather than of
|
||||
// where the case sits in the file.
|
||||
candidateSeq = 0;
|
||||
mockDbGet.mockReset();
|
||||
mockDbGet.mockReturnValue(undefined);
|
||||
mockDbRun.mockReset();
|
||||
@@ -129,10 +147,12 @@ describe('collectPhotos', () => {
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
// Keyed by which file it is, not by where it landed in the strip.
|
||||
const key = candidateKey('ChIJmuseum', 'commons:1001');
|
||||
expect(out.photos).toEqual([
|
||||
{
|
||||
key: 'ChIJmuseum~p0',
|
||||
url: '/api/maps/place-photo/ChIJmuseum~p0/bytes',
|
||||
key,
|
||||
url: `/api/maps/place-photo/${encodeURIComponent(key)}/bytes`,
|
||||
attribution: 'Alice',
|
||||
license: 'CC BY-SA 4.0',
|
||||
licenseUrl: 'https://creativecommons.org/licenses/by-sa/4.0/',
|
||||
@@ -166,14 +186,14 @@ describe('collectPhotos', () => {
|
||||
|
||||
expect(out.photos.map((p) => p.source)).toEqual(['google', 'wikimedia']);
|
||||
expect(out.photos[0]).toMatchObject({
|
||||
key: 'ChIJmuseum~p0',
|
||||
key: candidateKey('ChIJmuseum', 'google:places/p/photos/a'),
|
||||
attribution: 'Bob',
|
||||
// Google grants no reusable licence — the fields stay empty rather than invented.
|
||||
license: null,
|
||||
licenseUrl: null,
|
||||
sourceUrl: null,
|
||||
});
|
||||
expect(out.photos[1].key).toBe('ChIJmuseum~p1');
|
||||
expect(out.photos[1].key).toBe(candidateKey('ChIJmuseum', 'commons:1001'));
|
||||
});
|
||||
|
||||
it('ENRICH-006: respects the existing photos kill switch for the Google half', async () => {
|
||||
@@ -212,7 +232,10 @@ describe('collectPhotos', () => {
|
||||
|
||||
it('ENRICH-009: skips a candidate whose bytes will not download', async () => {
|
||||
const maps = mapsStub({
|
||||
fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ photoUrl: 'https://commons.org/gone.jpg' }), commonsCandidate()]),
|
||||
fetchCommonsCandidates: vi.fn(async () => [
|
||||
commonsCandidate({ photoUrl: 'https://commons.org/gone.jpg', attribution: 'Alice' }),
|
||||
commonsCandidate({ attribution: 'Bob' }),
|
||||
]),
|
||||
});
|
||||
mockSafeFetchFollow
|
||||
.mockResolvedValueOnce({ ok: false, arrayBuffer: async () => new ArrayBuffer(0) })
|
||||
@@ -220,11 +243,11 @@ describe('collectPhotos', () => {
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
// The survivor keeps index 1 — keys number the provider's list, so a picture
|
||||
// that fails leaves a gap instead of renumbering the ones behind it. Keys
|
||||
// have to point at the same file on the next request.
|
||||
// The survivor keeps its own key. That used to depend on the failed
|
||||
// picture leaving a numbered gap behind it; now the key is derived from
|
||||
// which file it is, so nothing in front of it can shift it.
|
||||
expect(out.photos).toHaveLength(1);
|
||||
expect(out.photos[0].key).toBe('ChIJmuseum~p1');
|
||||
expect(out.photos[0].key).toBe(candidateKey('ChIJmuseum', 'commons:1002'));
|
||||
});
|
||||
|
||||
it('ENRICH-010: treats an empty body and a throwing download as a miss', async () => {
|
||||
@@ -277,7 +300,10 @@ describe('collectPhotos', () => {
|
||||
const out = await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
expect(out.photos).toHaveLength(1);
|
||||
expect(out.photos[0]).toMatchObject({ key: 'ChIJmuseum~p1', attribution: 'Cara' });
|
||||
expect(out.photos[0]).toMatchObject({
|
||||
key: candidateKey('ChIJmuseum', 'google:places/p/photos/ok'),
|
||||
attribution: 'Cara',
|
||||
});
|
||||
});
|
||||
|
||||
it('ENRICH-012c: downloads the pictures concurrently, not one after another', async () => {
|
||||
@@ -286,7 +312,9 @@ describe('collectPhotos', () => {
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const maps = mapsStub({
|
||||
fetchCommonsCandidates: vi.fn(async () => [commonsCandidate(), commonsCandidate(), commonsCandidate()]),
|
||||
fetchCommonsCandidates: vi.fn(async () =>
|
||||
['Alice', 'Bob', 'Cara'].map((attribution) => commonsCandidate({ attribution })),
|
||||
),
|
||||
});
|
||||
mockSafeFetchFollow.mockImplementation(async () => {
|
||||
active++;
|
||||
@@ -329,7 +357,7 @@ describe('collectPhotos', () => {
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, { lat: 50.9, lng: 6.96, name: 'Bench' });
|
||||
|
||||
expect(out.photos[0].key).toBe('coords:50.9:6.96~p0');
|
||||
expect(out.photos[0].key).toBe(candidateKey('coords:50.9:6.96', 'commons:1001'));
|
||||
expect(maps.fetchGooglePhotoRefs).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -473,7 +501,7 @@ describe('collectDescription', () => {
|
||||
describe('result cache', () => {
|
||||
// v is the cache format version; an entry written before a release that
|
||||
// changes the shape or the sources is discarded rather than served.
|
||||
const cachedRow = (photos: unknown[], fetchedAt = Date.now(), v: number | undefined = 2) => ({
|
||||
const cachedRow = (photos: unknown[], fetchedAt = Date.now(), v: number | undefined = 3) => ({
|
||||
payload_json: JSON.stringify({ photos, description: null, facts: [], v }),
|
||||
fetched_at: fetchedAt,
|
||||
});
|
||||
@@ -610,7 +638,11 @@ describe('creditLine', () => {
|
||||
|
||||
await make(maps, cache).enrich(1, REQ);
|
||||
|
||||
expect(cache.put).toHaveBeenCalledWith('ChIJmuseum~p0', expect.any(Buffer), 'Alice · CC BY-SA 4.0');
|
||||
expect(cache.put).toHaveBeenCalledWith(
|
||||
candidateKey('ChIJmuseum', 'commons:1001'),
|
||||
expect.any(Buffer),
|
||||
'Alice · CC BY-SA 4.0',
|
||||
);
|
||||
});
|
||||
|
||||
it('ENRICH-029: reads a stored credit back by cache key', () => {
|
||||
@@ -702,14 +734,17 @@ describe('picture source selection', () => {
|
||||
// category of a place returns the place.
|
||||
const maps = mapsStub({
|
||||
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikimedia_commons: 'Category:Museum Ludwig' } })),
|
||||
fetchCommonsCategoryCandidates: vi.fn(async () => [commonsCandidate()]),
|
||||
fetchCommonsCategoryCandidates: vi.fn(async () =>
|
||||
['Alice', 'Bob'].map((attribution) => commonsCandidate({ attribution })),
|
||||
),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
expect(maps.fetchCommonsCategoryCandidates).toHaveBeenCalledWith('Category:Museum Ludwig', 5);
|
||||
// Two pictures of the place is a strip; the coordinate search is not asked.
|
||||
expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled();
|
||||
expect(out.photos).toHaveLength(1);
|
||||
expect(out.photos).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('ENRICH-056: falls back to the coordinate search when the category is empty', async () => {
|
||||
@@ -740,3 +775,175 @@ describe('picture source selection', () => {
|
||||
expect(out.description).toMatchObject({ source: 'wikivoyage', license: 'CC BY-SA 4.0' });
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* ENRICH-060..069 — the source ladder.
|
||||
*
|
||||
* The picker used to show aerial survey tiles for Hamburg Airport and the
|
||||
* underground station for the Brandenburg Gate, because the only rung that ever
|
||||
* ran was "photographed within 300m". These cases pin the order it climbs and
|
||||
* when it is allowed to fall back that far.
|
||||
*/
|
||||
describe('picture source ladder', () => {
|
||||
const withTags = (tags: Record<string, unknown>) =>
|
||||
vi.fn(async () => ({ place: { source: 'openstreetmap', ...tags } }));
|
||||
|
||||
it('ENRICH-060: asks Wikidata first when the place carries a wikidata tag', async () => {
|
||||
const maps = mapsStub({
|
||||
details: withTags({ wikidata: 'Q1097' }),
|
||||
fetchWikidataCandidates: vi.fn(async () => ({
|
||||
candidates: ['Alice', 'Bob', 'Cara', 'Dana', 'Erin'].map((attribution) => commonsCandidate({ attribution })),
|
||||
commonsCategory: 'Berlin Hauptbahnhof',
|
||||
})),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
expect(maps.fetchWikidataCandidates).toHaveBeenCalledWith('Q1097', 5);
|
||||
// A full strip of pictures somebody attached to this exact object — no rung
|
||||
// below it is asked, least of all the coordinate search.
|
||||
expect(maps.fetchCommonsCategoryCandidates).not.toHaveBeenCalled();
|
||||
expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled();
|
||||
expect(out.photos).toHaveLength(5);
|
||||
});
|
||||
|
||||
it('ENRICH-061: takes the Commons category Wikidata names when the tag omits it', async () => {
|
||||
// Berlin Hauptbahnhof carries wikidata but no wikimedia_commons in OSM; the
|
||||
// item's P373 knows the category anyway.
|
||||
const maps = mapsStub({
|
||||
details: withTags({ wikidata: 'Q1097' }),
|
||||
fetchWikidataCandidates: vi.fn(async () => ({
|
||||
candidates: [commonsCandidate()],
|
||||
commonsCategory: 'Berlin Hauptbahnhof',
|
||||
})),
|
||||
fetchCommonsCategoryCandidates: vi.fn(async () => [commonsCandidate({ attribution: 'Bob' })]),
|
||||
});
|
||||
|
||||
await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
expect(maps.fetchCommonsCategoryCandidates).toHaveBeenCalledWith('Berlin Hauptbahnhof', 5);
|
||||
});
|
||||
|
||||
it('ENRICH-062: reaches for the article lead image when there is no wikidata tag', async () => {
|
||||
const maps = mapsStub({
|
||||
details: withTags({ wikipedia: 'de:Flughafen Hamburg' }),
|
||||
fetchWikiLeadImageName: vi.fn(async () => 'Hamburg airport terminals.jpg'),
|
||||
fetchCommonsFilesByName: vi.fn(async () =>
|
||||
new Map([['hamburg airport terminals.jpg', commonsCandidate({ attribution: 'Alice' })]]),
|
||||
),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
expect(maps.fetchWikiLeadImageName).toHaveBeenCalledWith('de:Flughafen Hamburg');
|
||||
expect(maps.fetchCommonsFilesByName).toHaveBeenCalledWith(['Hamburg airport terminals.jpg']);
|
||||
// Reported as a Wikipedia picture, because that is where it was chosen.
|
||||
expect(out.photos[0].source).toBe('wikipedia');
|
||||
});
|
||||
|
||||
it('ENRICH-063: only falls back to what was photographed nearby when the rest came up thin', async () => {
|
||||
const maps = mapsStub({
|
||||
details: withTags({ wikidata: 'Q1097' }),
|
||||
fetchWikidataCandidates: vi.fn(async () => ({ candidates: [commonsCandidate()], commonsCategory: null })),
|
||||
fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ attribution: 'Bob' })]),
|
||||
});
|
||||
|
||||
await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
// One curated picture is not a strip, so the coordinate search tops it up.
|
||||
expect(maps.fetchCommonsCandidates).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ENRICH-064: does not run the coordinate search behind a full Google strip', async () => {
|
||||
const maps = mapsStub({
|
||||
getMapsKey: vi.fn(() => 'key'),
|
||||
fetchGooglePhotoRefs: vi.fn(async () => [
|
||||
{ name: 'places/p/photos/a', attribution: 'Bob' },
|
||||
{ name: 'places/p/photos/b', attribution: 'Cara' },
|
||||
{ name: 'places/p/photos/c', attribution: 'Dana' },
|
||||
]),
|
||||
fetchGooglePhotoBytes: vi.fn(async () => Buffer.from([1, 2, 3])),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
expect(out.photos).toHaveLength(3);
|
||||
expect(out.photos.every((p) => p.source === 'google')).toBe(true);
|
||||
});
|
||||
|
||||
it('ENRICH-065: keeps the coordinate search off the critical path of Google\'s listing', async () => {
|
||||
// Waiting for the listing before deciding put two round trips end to end,
|
||||
// which is what pushed this endpoint past the client timeout once before.
|
||||
const order: string[] = [];
|
||||
const maps = mapsStub({
|
||||
getMapsKey: vi.fn(() => 'key'),
|
||||
fetchGooglePhotoRefs: vi.fn(async () => {
|
||||
order.push('google:start');
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
order.push('google:end');
|
||||
return [];
|
||||
}),
|
||||
fetchCommonsCandidates: vi.fn(async () => {
|
||||
order.push('commons:start');
|
||||
return [];
|
||||
}),
|
||||
});
|
||||
|
||||
await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
expect(order.indexOf('commons:start')).toBeLessThan(order.indexOf('google:end'));
|
||||
});
|
||||
|
||||
it('ENRICH-066: drops a picture that two rungs both produced', async () => {
|
||||
const shared = commonsCandidate({ pageId: 4242, attribution: 'Alice' });
|
||||
const maps = mapsStub({
|
||||
details: withTags({ wikidata: 'Q1097', wikimedia_commons: 'Category:X' }),
|
||||
fetchWikidataCandidates: vi.fn(async () => ({ candidates: [shared], commonsCategory: null })),
|
||||
fetchCommonsCategoryCandidates: vi.fn(async () => [{ ...shared, photoUrl: 'https://commons.org/other-url.jpg' }]),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, OSM_REQ);
|
||||
|
||||
// Same page id, different thumbnail url — the url alone would let it through.
|
||||
expect(out.photos).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('ENRICH-067: throws away the survey tiles a geosearch around an airport returns', async () => {
|
||||
const maps = mapsStub({
|
||||
fetchCommonsCandidates: vi.fn(async () =>
|
||||
[2013, 2015, 2016, 2019].map((year) =>
|
||||
commonsCandidate({
|
||||
title: `File:Dop20rgb 32565 5943 ${year}.jpg`,
|
||||
attribution: 'Landesbetrieb Geoinformation Hamburg',
|
||||
width: 5000,
|
||||
height: 5000,
|
||||
descriptors: 'Orthophoto Sommerbefliegung',
|
||||
}),
|
||||
),
|
||||
),
|
||||
});
|
||||
|
||||
const out = await make(maps, cacheStub()).enrich(1, REQ);
|
||||
|
||||
expect(out.photos).toEqual([]);
|
||||
});
|
||||
|
||||
it('ENRICH-068: keys a picture by which file it is, not by where it sat', async () => {
|
||||
// The strip length changes with which providers answered. Keyed by position,
|
||||
// a slot that held a Wikidata photo one minute held a category photo the
|
||||
// next while the cache kept serving the first one's bytes — under CC BY
|
||||
// that credits the wrong person, not just the wrong thumbnail.
|
||||
const one = mapsStub({ fetchCommonsCandidates: vi.fn(async () => [commonsCandidate({ pageId: 777 })]) });
|
||||
const first = (await make(one, cacheStub()).enrich(1, REQ)).photos[0].key;
|
||||
|
||||
const two = mapsStub({
|
||||
fetchCommonsCandidates: vi.fn(async () => [
|
||||
commonsCandidate({ pageId: 111, attribution: 'Zoe' }),
|
||||
commonsCandidate({ pageId: 777 }),
|
||||
]),
|
||||
});
|
||||
const shifted = (await make(two, cacheStub()).enrich(1, REQ)).photos.find((p) => p.attribution === 'Alice');
|
||||
|
||||
expect(shifted?.key).toBe(first);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user