mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-19 05:26:47 +00:00
refactor(maps): wikimedia lookups and brand logos move into their own client
This commit is contained in:
Maurice
@@ -102,6 +102,7 @@ export const PUBLIC_FILES = [
|
||||
// asks Google and Wikimedia directly instead of through a MapsService facade.
|
||||
'maps/providers/google-places.provider.ts',
|
||||
'maps/providers/osm.client.ts',
|
||||
'maps/providers/wikimedia.client.ts',
|
||||
];
|
||||
|
||||
function walk(dir, files = []) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { MapsService } from './maps.service';
|
||||
import { MapsMcp } from './maps.mcp';
|
||||
import { GooglePlacesClient } from './providers/google-places.provider';
|
||||
import { OsmClient } from './providers/osm.client';
|
||||
import { WikimediaClient } from './providers/wikimedia.client';
|
||||
import { PlacePhotosModule } from '../place-photos/place-photos.module';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { GoogleQuotaModule } from '../google-quota/google-quota.module';
|
||||
@@ -30,7 +31,7 @@ import { Places } from '../../db/entities/Places.entity';
|
||||
@Module({
|
||||
imports: [PlacePhotosModule, StorageModule, GoogleQuotaModule, MikroOrmModule.forFeature([AppSettings, Users, PlaceDetailsCache, Places])],
|
||||
controllers: [MapsController],
|
||||
providers: [MapsService, MapsMcp, GooglePlacesClient, OsmClient],
|
||||
exports: [MapsService, GooglePlacesClient, OsmClient],
|
||||
providers: [MapsService, MapsMcp, GooglePlacesClient, OsmClient, WikimediaClient],
|
||||
exports: [MapsService, GooglePlacesClient, OsmClient, WikimediaClient],
|
||||
})
|
||||
export class MapsModule {}
|
||||
|
||||
@@ -8,10 +8,9 @@ import type {
|
||||
MapsReverseResult,
|
||||
MapsResolveUrlResult,
|
||||
} from '@trek/shared';
|
||||
import { Jimp } from 'jimp';
|
||||
import { readEnv } from '../../app-config';
|
||||
import { safeFetchFollow, SsrfBlockedError } from '../../utils/ssrfGuard';
|
||||
import { discardBody, exceedsDeclaredLength, readCapped, readCappedText } from '../../utils/cappedFetch';
|
||||
import { discardBody, exceedsDeclaredLength, readCappedText } from '../../utils/cappedFetch';
|
||||
import { resolveApiKey, type ApiKeySource } from '../settings/instance-api-keys';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { AppSettings } from '../../db/entities/AppSettings.entity';
|
||||
@@ -25,6 +24,8 @@ import type { PlacesRepository } from '../../db/repositories/Places.repository';
|
||||
import { isPlacesProviderChoice, type PlacesProviderChoice } from './providers/places-provider';
|
||||
import { GooglePlacesClient } from './providers/google-places.provider';
|
||||
import { OsmClient, POI_RESULT_CAP, type PoiSearchResult } from './providers/osm.client';
|
||||
import { WikimediaClient, type BrandLogo } from './providers/wikimedia.client';
|
||||
import { withPhotoFetchSlot } from './photo-fetch-slot';
|
||||
import {
|
||||
AMAP_SHORT_HOSTS,
|
||||
AmapPlacesProvider,
|
||||
@@ -57,9 +58,6 @@ import {
|
||||
googleFtidFromMapsUrl,
|
||||
OSM_PLACE_ID,
|
||||
parsePoiCategories,
|
||||
stripWikiMarkup,
|
||||
parseWikipediaTag,
|
||||
toWikiLang,
|
||||
mergeSearchResults,
|
||||
} from './maps.helpers';
|
||||
import {
|
||||
@@ -73,153 +71,9 @@ import {
|
||||
|
||||
export { readBrandIdentity, readWikiIdentity, type WikiIdentity } from './providers/wiki-identity';
|
||||
|
||||
interface WikidataSnak {
|
||||
mainsnak?: { datavalue?: { value?: string } };
|
||||
rank?: 'preferred' | 'normal' | 'deprecated';
|
||||
}
|
||||
type WikidataClaims = Record<string, WikidataSnak[] | undefined>;
|
||||
export { withPhotoFetchSlot } from './photo-fetch-slot';
|
||||
export type { BrandLogo, CommonsCandidate } from './providers/wikimedia.client';
|
||||
|
||||
/**
|
||||
* Wikidata properties that name a picture of a place, in the order we want them.
|
||||
*
|
||||
* P18 is the representative image. The rest exist because a station or a
|
||||
* monument is not one view: asking for the interior, the night shot, the
|
||||
* panorama and the aerial gives a picker four genuinely different pictures
|
||||
* instead of four frames of the same façade.
|
||||
*/
|
||||
const WIKIDATA_IMAGE_PROPERTIES = [
|
||||
'P18', // image
|
||||
'P5775', // interior view
|
||||
'P3451', // night view
|
||||
'P8592', // aerial view
|
||||
'P4291', // panoramic view
|
||||
'P5252', // winter view
|
||||
'P948', // Wikivoyage banner
|
||||
] as const;
|
||||
|
||||
/** First non-empty string value of a claim list. */
|
||||
function claimValue(snaks: WikidataSnak[] | undefined): string | null {
|
||||
for (const snak of snaks ?? []) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value === 'string' && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* File names from an item's picture properties, best first.
|
||||
*
|
||||
* Within P18 the statement rank decides: an item with several images marks one
|
||||
* `preferred`, and that is the one an editor considers representative. The API
|
||||
* returns statements in edit order, not rank order, so taking `[0]` picks
|
||||
* whichever was added first — for the Brandenburg Gate that is a coin toss
|
||||
* between the morning shot and a wide overview.
|
||||
*/
|
||||
function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] {
|
||||
const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const;
|
||||
const names: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const property of WIKIDATA_IMAGE_PROPERTIES) {
|
||||
const snaks = [...(claims[property] ?? [])]
|
||||
.filter((snak) => snak.rank !== 'deprecated')
|
||||
.sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1));
|
||||
for (const snak of snaks) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value !== 'string' || !value.trim()) continue;
|
||||
const key = normalizeFileTitle(value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
names.push(value.trim());
|
||||
if (names.length >= limit) return names;
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */
|
||||
function normalizeFileTitle(title: string): string {
|
||||
return title.replace(/^File:/i, '').replaceAll('_', ' ').trim().toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* A bare Commons category name out of whatever an OSM `wikimedia_commons` tag
|
||||
* holds.
|
||||
*
|
||||
* The tag is free text and mappers put three different things in it: a bare
|
||||
* name, a prefixed `Category:…`, or — against the wiki's own advice — a single
|
||||
* `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`,
|
||||
* which matches nothing and fell through to the coordinate search without a
|
||||
* word. Localised prefixes (`Kategorie:`) appear too.
|
||||
*/
|
||||
function normalizeCategoryName(raw: string): string | null {
|
||||
const value = raw.trim();
|
||||
if (!value) return null;
|
||||
// A file name is not a category, and there is nothing sensible to derive.
|
||||
if (/^(file|datei|image|bild)\s*:/i.test(value)) return null;
|
||||
return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null;
|
||||
}
|
||||
|
||||
interface WikiCommonsPage {
|
||||
pageid?: number;
|
||||
title?: string;
|
||||
imageinfo?: {
|
||||
url?: string;
|
||||
thumburl?: string;
|
||||
mime?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
/** The file description page — where the full licence terms live. */
|
||||
descriptionurl?: string;
|
||||
extmetadata?: {
|
||||
Artist?: { value?: string };
|
||||
LicenseShortName?: { value?: string };
|
||||
LicenseUrl?: { value?: string };
|
||||
UsageTerms?: { value?: string };
|
||||
/** Used to spot survey imagery and diagrams, which are not pictures of a place. */
|
||||
Categories?: { value?: string };
|
||||
ObjectName?: { value?: string };
|
||||
ImageDescription?: { value?: string };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
/**
|
||||
* One Commons image with everything needed to credit it. Commons is mostly
|
||||
* CC BY / CC BY-SA, so a candidate that cannot be attributed is not usable in a
|
||||
* picker — the fields are nullable because Commons metadata is user-maintained
|
||||
* and genuinely incomplete on some files, not because they are optional to show.
|
||||
*/
|
||||
export interface CommonsCandidate {
|
||||
photoUrl: string;
|
||||
attribution: string | null;
|
||||
license: string | null;
|
||||
licenseUrl: string | null;
|
||||
sourceUrl: string | null;
|
||||
/**
|
||||
* Commons page id — the only stable identity a file has across the four ways
|
||||
* we reach it. The thumbnail URL is not: the same file comes back from
|
||||
* commons.wikimedia.org and from a language Wikipedia with different query
|
||||
* strings, so deduplicating on the URL silently lets the same picture through
|
||||
* twice. It is also what keys the cached bytes, so it must survive.
|
||||
*/
|
||||
pageId: number | null;
|
||||
/** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */
|
||||
title: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
/** Free text used to reject survey imagery, floor plans and logos. */
|
||||
descriptors: string | null;
|
||||
}
|
||||
|
||||
// ── Concurrency limiter for outbound photo fetches ───────────────────────────
|
||||
// Caps simultaneous Wikimedia/Google photo requests so a bulk import of hundreds
|
||||
// of places cannot monopolise the event loop or trigger external API rate limits.
|
||||
// Module-scoped ON PURPOSE (permissions-cache precedent): the bridge instance and
|
||||
// the DI singleton must share one limiter, one POI cache and one call counter.
|
||||
// Wikimedia is normally well under a second, but a cold TLS handshake from a
|
||||
// fresh container has been seen at eight. Enrichment answers a live dialog, so
|
||||
// a slow provider is dropped rather than waited out.
|
||||
// A Google Maps place page is a few hundred KB; the coordinates sit in the
|
||||
// embedded map data near the top, so two megabytes is plenty and keeps an
|
||||
// unbounded body out of memory.
|
||||
@@ -239,144 +93,6 @@ export function isGoogleMapsHost(hostname: string): boolean {
|
||||
|| /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname);
|
||||
}
|
||||
|
||||
const WIKI_TIMEOUT_MS = 6000;
|
||||
|
||||
// ── Brand logos ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// A road trip corridor is mostly chains — Shell, Aral, JET — and the brand is the
|
||||
// fastest thing to recognise on a map. OSM carries `brand:wikidata` on most of them,
|
||||
// Wikidata carries the logo (P154), and Commons serves it.
|
||||
//
|
||||
// The bytes are proxied rather than linked so the browser never talks to Wikimedia:
|
||||
// one self-hosted instance asking for a handful of logos is a very different egress
|
||||
// profile from every visitor's browser announcing which petrol stations they are
|
||||
// looking at. The cache is in memory on purpose — a GET must not write to the DB, and
|
||||
// there are only so many fuel brands.
|
||||
const BRAND_LOGO_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const BRAND_LOGO_CACHE_MAX = 300;
|
||||
/** Well past any logo; a Commons original can be a multi-megabyte SVG or print-res PNG. */
|
||||
const BRAND_LOGO_MAX_BYTES = 512 * 1024;
|
||||
/** A place photo is a full-size image rather than a 128px mark, so its own ceiling. */
|
||||
const WIKIMEDIA_PHOTO_MAX_BYTES = 8 * 1024 * 1024;
|
||||
const BRAND_LOGO_WIDTH = 128;
|
||||
/** The square the logo is centred in, and the breathing room around it. */
|
||||
const BRAND_LOGO_CANVAS = 96;
|
||||
const BRAND_LOGO_PADDING = 8;
|
||||
const WIKIDATA_ID_RE = /^Q[1-9][0-9]{0,11}$/;
|
||||
|
||||
export interface BrandLogo {
|
||||
bytes: Buffer;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts a logo on a background it can actually be seen against.
|
||||
*
|
||||
* Half the fuel brands ship a white wordmark with a transparent background —
|
||||
* TotalEnergies, Esso and JET among them — and on the white pill the marker used to
|
||||
* draw they were invisible. Which background is right is a property of the image, not
|
||||
* of the brand, so it is measured: the mean brightness of the pixels that are actually
|
||||
* opaque decides between a white and a near-black backdrop.
|
||||
*
|
||||
* Returns the flattened PNG, or null when the bytes cannot be read at all — the pin
|
||||
* then keeps its category icon, which is a fine outcome.
|
||||
*/
|
||||
async function flattenBrandLogo(bytes: Buffer): Promise<BrandLogo | null> {
|
||||
try {
|
||||
const image = await Jimp.read(bytes);
|
||||
const width = image.bitmap.width;
|
||||
const height = image.bitmap.height;
|
||||
if (!width || !height) return null;
|
||||
|
||||
let sum = 0;
|
||||
let opaque = 0;
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const idx = (y * width + x) * 4;
|
||||
const alpha = image.bitmap.data[idx + 3];
|
||||
// Half-transparent pixels are anti-aliasing, not the mark itself.
|
||||
if (alpha < 128) continue;
|
||||
const r = image.bitmap.data[idx];
|
||||
const g = image.bitmap.data[idx + 1];
|
||||
const b = image.bitmap.data[idx + 2];
|
||||
sum += 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
opaque++;
|
||||
}
|
||||
}
|
||||
// A logo with nothing opaque in it is not a logo.
|
||||
if (opaque === 0) return null;
|
||||
|
||||
// Square, with the logo scaled to fit inside it. Most brand marks are wide wordmarks
|
||||
// — TotalEnergies is three times as wide as it is tall — and a round pin showing the
|
||||
// middle of one is unreadable. Fitting it into a square means the pin can crop to a
|
||||
// circle without ever cutting the mark itself.
|
||||
const scale = Math.min(
|
||||
(BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / width,
|
||||
(BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / height,
|
||||
);
|
||||
// Never upscale: a 16-pixel favicon blown up to 96 looks worse than a small one.
|
||||
if (scale < 1) image.scale(scale);
|
||||
|
||||
const light = sum / opaque > 150;
|
||||
const canvas = new Jimp({
|
||||
width: BRAND_LOGO_CANVAS,
|
||||
height: BRAND_LOGO_CANVAS,
|
||||
color: light ? 0x111827ff : 0xffffffff,
|
||||
});
|
||||
canvas.composite(
|
||||
image,
|
||||
Math.round((BRAND_LOGO_CANVAS - image.bitmap.width) / 2),
|
||||
Math.round((BRAND_LOGO_CANVAS - image.bitmap.height) / 2),
|
||||
);
|
||||
const out = await canvas.getBuffer('image/png');
|
||||
return { bytes: Buffer.from(out), contentType: 'image/png' };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Tighter than the wiki calls, because this one sits at the FRONT of a chain:
|
||||
// identity, then sitelinks, then the extract. Nominatim answers a bounded
|
||||
// search in 0.2-0.7s in practice, so anything past a couple of seconds is a bad
|
||||
// day at the provider rather than a slow answer worth waiting for.
|
||||
const IDENTITY_TIMEOUT_MS = 2500;
|
||||
|
||||
const MAX_CONCURRENT_PHOTO_FETCHES = 5;
|
||||
let photoFetchActive = 0;
|
||||
const photoFetchQueue: Array<() => void> = [];
|
||||
|
||||
function acquirePhotoFetchSlot(): Promise<void> {
|
||||
if (photoFetchActive < MAX_CONCURRENT_PHOTO_FETCHES) {
|
||||
photoFetchActive++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => photoFetchQueue.push(resolve));
|
||||
}
|
||||
|
||||
function releasePhotoFetchSlot(): void {
|
||||
const next = photoFetchQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
photoFetchActive--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs an outbound photo fetch under the shared slot limit. Exported so the
|
||||
* enrichment module queues behind the same five slots — a picker that grabs
|
||||
* three images per selected place would otherwise sail past a cap the rest of
|
||||
* the app respects.
|
||||
*/
|
||||
export async function withPhotoFetchSlot<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await acquirePhotoFetchSlot();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
releasePhotoFetchSlot();
|
||||
}
|
||||
}
|
||||
|
||||
// Places near a point (#976): cached longer than the POI boxes, because the
|
||||
// likely caller is an import asking the same photo location again, and Google
|
||||
// bills every one of those.
|
||||
@@ -452,11 +168,9 @@ export class MapsService {
|
||||
private readonly googleQuota: GoogleQuotaService,
|
||||
private readonly googlePlaces: GooglePlacesClient,
|
||||
private readonly osm: OsmClient,
|
||||
private readonly wiki: WikimediaClient,
|
||||
) {}
|
||||
|
||||
/** Brand id → logo bytes, or null for "asked, has none". Insertion-ordered, so the
|
||||
* oldest entry is the one evicted when it fills up. */
|
||||
private readonly brandLogoCache = new Map<string, { at: number; logo: BrandLogo | null }>();
|
||||
|
||||
/** Amap autocomplete tips for the details fallback. Here rather than on the provider:
|
||||
* a provider is built per request, and a pick is two requests. */
|
||||
@@ -562,74 +276,9 @@ export class MapsService {
|
||||
return this.resolveGoogleMapsUrl(url) as Promise<MapsResolveUrlResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* The logo of a brand, by its Wikidata id, as bytes.
|
||||
*
|
||||
* Two hops: Wikidata says which Commons file is the logo (property P154), Commons
|
||||
* serves a thumbnail of it. Both answers are cached, including "this brand has no
|
||||
* logo" — otherwise every map pan would ask Wikidata about the same supermarket
|
||||
* chain again. Returns null whenever anything is missing or unreadable; a marker
|
||||
* without a logo falls back to its category icon, which is a fine outcome.
|
||||
*/
|
||||
async brandLogo(wikidataId: string): Promise<BrandLogo | null> {
|
||||
if (!WIKIDATA_ID_RE.test(wikidataId)) return null;
|
||||
|
||||
const cached = this.brandLogoCache.get(wikidataId);
|
||||
if (cached && Date.now() - cached.at < BRAND_LOGO_CACHE_TTL_MS) return cached.logo;
|
||||
|
||||
const remember = (logo: BrandLogo | null): BrandLogo | null => {
|
||||
if (this.brandLogoCache.size >= BRAND_LOGO_CACHE_MAX) {
|
||||
const oldest = this.brandLogoCache.keys().next().value;
|
||||
if (oldest !== undefined) this.brandLogoCache.delete(oldest);
|
||||
}
|
||||
this.brandLogoCache.set(wikidataId, { at: Date.now(), logo });
|
||||
return logo;
|
||||
};
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
action: 'wbgetclaims',
|
||||
entity: wikidataId,
|
||||
property: 'P154',
|
||||
format: 'json',
|
||||
});
|
||||
const claimRes = await fetch(`https://www.wikidata.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!claimRes.ok) return remember(null);
|
||||
const claims = (await claimRes.json()) as {
|
||||
claims?: { P154?: { mainsnak?: { datavalue?: { value?: unknown } } }[] };
|
||||
};
|
||||
const file = claims.claims?.P154?.[0]?.mainsnak?.datavalue?.value;
|
||||
if (typeof file !== 'string' || !file.trim()) return remember(null);
|
||||
|
||||
// Special:FilePath renders a thumbnail at the width asked for and redirects to
|
||||
// the CDN, so each hop is re-checked by the guard rather than trusted.
|
||||
const url = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(file)}?width=${BRAND_LOGO_WIDTH}`;
|
||||
// The same six seconds the Wikidata hop above allows. Without a deadline
|
||||
// this waited on undici's five-minute default, holding a request context
|
||||
// and a socket per stalled logo while the pin sat on its fallback icon.
|
||||
const imgRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true });
|
||||
if (!imgRes.ok) return remember(null);
|
||||
|
||||
if (exceedsDeclaredLength(imgRes, BRAND_LOGO_MAX_BYTES)) {
|
||||
discardBody(imgRes);
|
||||
return remember(null);
|
||||
}
|
||||
// Streamed rather than buffered whole: a chunked answer declares no length,
|
||||
// so the post-check only ever ran after the bytes were already in memory.
|
||||
const { bytes, truncated } = await readCapped(imgRes, BRAND_LOGO_MAX_BYTES);
|
||||
if (truncated || bytes.byteLength === 0) return remember(null);
|
||||
|
||||
const contentType = imgRes.headers.get('content-type') ?? '';
|
||||
if (!contentType.startsWith('image/')) return remember(null);
|
||||
|
||||
return remember(await flattenBrandLogo(bytes));
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return remember(null);
|
||||
return remember(null);
|
||||
}
|
||||
/** The logo of a brand, by its Wikidata id, as bytes (null: the pin keeps its category icon). */
|
||||
brandLogo(wikidataId: string): Promise<BrandLogo | null> {
|
||||
return this.wiki.brandLogo(wikidataId);
|
||||
}
|
||||
|
||||
// POI search by category within a viewport. Never calls Google.
|
||||
@@ -929,484 +578,6 @@ export class MapsService {
|
||||
return hit?.lat != null && hit?.lng != null ? { lat: hit.lat, lng: hit.lng } : null;
|
||||
}
|
||||
|
||||
// ── Wikimedia Commons photo lookup ─────────────────────────────────────────
|
||||
|
||||
async fetchWikimediaPhoto(
|
||||
lat: number,
|
||||
lng: number,
|
||||
name?: string,
|
||||
): Promise<{ photoUrl: string; attribution: string | null } | null> {
|
||||
// Strategy 1: Search Wikipedia for the place name -> get the article image
|
||||
if (name) {
|
||||
try {
|
||||
const searchParams = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: name,
|
||||
prop: 'pageimages',
|
||||
piprop: 'thumbnail',
|
||||
pithumbsize: '400',
|
||||
pilimit: '1',
|
||||
redirects: '1',
|
||||
});
|
||||
const res = await fetch(`https://en.wikipedia.org/w/api.php?${searchParams}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
// This runs inside one of the few shared photo-fetch slots; a stalled
|
||||
// answer would hold it for everyone.
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, { thumbnail?: { source?: string } }> } };
|
||||
const pages = data.query?.pages;
|
||||
if (pages) {
|
||||
for (const page of Object.values(pages)) {
|
||||
if (page.thumbnail?.source) {
|
||||
return { photoUrl: page.thumbnail.source, attribution: 'Wikipedia' };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* fall through to geosearch */
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: Wikimedia Commons geosearch by coordinates
|
||||
const candidates = await this.fetchCommonsCandidates(lat, lng, 5);
|
||||
const first = candidates[0];
|
||||
return first ? { photoUrl: first.photoUrl, attribution: first.attribution } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commons images near a coordinate, licence metadata included.
|
||||
*
|
||||
* geosearch already returns up to `limit` files in a single request, so asking
|
||||
* for a whole strip costs the same as asking for one picture. Callers that only
|
||||
* want a single image (fetchWikimediaPhoto, and through it getPlacePhoto) take
|
||||
* the first entry and get exactly the file they got before this existed.
|
||||
*/
|
||||
/**
|
||||
* Anything photographed near a coordinate. The bottom rung of the picture
|
||||
* ladder, and the only one with no claim on the subject at all.
|
||||
*
|
||||
* 60 metres rather than the 300 it used to be. Three hundred metres in a city
|
||||
* centre is a whole block: the town hall, the church and the underground
|
||||
* entrance are all inside it, and every one of them outranks the doner shop
|
||||
* we were actually asked about. Sixty is roughly "the same building and its
|
||||
* neighbours", which is the widest a picture can be taken and still plausibly
|
||||
* show the place. It does not fix the real problem, which is that nobody
|
||||
* photographed the shop, but it stops the wrong answer from being confident.
|
||||
*/
|
||||
async fetchCommonsCandidates(lat: number, lng: number, limit = 5): Promise<CommonsCandidate[]> {
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'geosearch',
|
||||
ggsprimary: 'all',
|
||||
ggsnamespace: '6',
|
||||
ggsradius: '60',
|
||||
ggscoord: `${lat}|${lng}`,
|
||||
// Deliberately more than the caller asked for. Around anything worth
|
||||
// visiting the first few hits are survey tiles, passers-by and the
|
||||
// building next door; the ranker needs a pool to reject from, and
|
||||
// geosearch charges the same for one result as for twenty.
|
||||
ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))),
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
// A hanging provider must not hold the whole enrichment request open;
|
||||
// no pictures is a fine answer, a request that never returns is not.
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return [];
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
// Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets
|
||||
// what it always got; the enrichment column ranks before it cuts.
|
||||
return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit')));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */
|
||||
private toCommonsCandidates(
|
||||
pages: Record<string, WikiCommonsPage> | undefined,
|
||||
limit: number,
|
||||
): CommonsCandidate[] {
|
||||
if (!pages) return [];
|
||||
const out: CommonsCandidate[] = [];
|
||||
// entries(), not values(): the map key is the page id, and for the queries
|
||||
// that reach a file by title it is the only place the id appears.
|
||||
for (const [key, page] of Object.entries(pages)) {
|
||||
const info = page.imageinfo?.[0];
|
||||
// Only use actual photos (JPEG/PNG), skip SVGs and PDFs
|
||||
const mime = info?.mime || '';
|
||||
if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue;
|
||||
const meta = info.extmetadata;
|
||||
const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null);
|
||||
out.push({
|
||||
// iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it —
|
||||
// info.url is the full-resolution original (multi-megapixel camera exports).
|
||||
photoUrl: info.thumburl ?? info.url,
|
||||
attribution: stripWikiMarkup(meta?.Artist?.value),
|
||||
license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value),
|
||||
licenseUrl: meta?.LicenseUrl?.value?.trim() || null,
|
||||
sourceUrl: info.descriptionurl || null,
|
||||
pageId: pageId && pageId > 0 ? pageId : null,
|
||||
title: page.title ?? null,
|
||||
width: info.width ?? null,
|
||||
height: info.height ?? null,
|
||||
descriptors: [
|
||||
stripWikiMarkup(meta?.ObjectName?.value),
|
||||
stripWikiMarkup(meta?.ImageDescription?.value),
|
||||
stripWikiMarkup(meta?.Categories?.value),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' | ') || null,
|
||||
});
|
||||
if (out.length >= limit) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lead paragraph of a wiki article, from Wikivoyage first and Wikipedia after.
|
||||
*
|
||||
* Wikivoyage is the travel sibling: same MediaWiki API, same CC BY-SA, but it
|
||||
* describes a place for someone about to go there, where Wikipedia opens with
|
||||
* area in square kilometres and pronunciation. Both are resolved from the OSM
|
||||
* `wikipedia` tag — guessing the article from the place name lands on the
|
||||
* wrong one for every ambiguous name, so no tag means no description rather
|
||||
* than a confident description of somewhere else.
|
||||
*/
|
||||
async fetchWikiExtract(
|
||||
wikipediaTag: string | null | undefined,
|
||||
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
|
||||
const parsed = parseWikipediaTag(wikipediaTag);
|
||||
if (!parsed) return null;
|
||||
|
||||
// Two sentences, not three: this sits next to a form, and a fourth line of
|
||||
// prose pushes the pictures out of view.
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: parsed.title,
|
||||
prop: 'extracts',
|
||||
exintro: '1',
|
||||
explaintext: '1',
|
||||
exsentences: '2',
|
||||
redirects: '1',
|
||||
});
|
||||
|
||||
for (const host of ['wikivoyage', 'wikipedia'] as const) {
|
||||
const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title);
|
||||
if (hit) return hit;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The lead paragraph of one named article on one named wiki.
|
||||
*
|
||||
* Split out from `fetchWikiExtract` because the article is not always found
|
||||
* through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag
|
||||
* at all (Berlin Hauptbahnhof is exactly that), and then the title comes from
|
||||
* the item's sitelinks instead.
|
||||
*/
|
||||
async fetchWikiExtractFor(
|
||||
host: 'wikivoyage' | 'wikipedia',
|
||||
lang: string,
|
||||
title: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
|
||||
if (!lang || !title) return null;
|
||||
// Two sentences, not three: this sits next to a form, and a fourth line of
|
||||
// prose pushes the pictures out of view.
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: title,
|
||||
prop: 'extracts',
|
||||
exintro: '1',
|
||||
explaintext: '1',
|
||||
exsentences: '2',
|
||||
redirects: '1',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { title?: string; extract?: string }> };
|
||||
};
|
||||
for (const page of Object.values(data.query?.pages ?? {})) {
|
||||
const text = page.extract?.trim();
|
||||
// A missing article comes back as a page with no extract, not a 404.
|
||||
if (!text) continue;
|
||||
const resolved = page.title ?? title;
|
||||
return {
|
||||
text,
|
||||
sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`,
|
||||
source: host,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Which articles a Wikidata item is linked to, for the wikis we care about.
|
||||
*
|
||||
* The way to an article when a place has a Wikidata id but no `wikipedia`
|
||||
* tag — which is most of them, because mappers add one or the other. One
|
||||
* request, a few hundred bytes.
|
||||
*/
|
||||
async fetchWikidataSitelinks(
|
||||
wikidataId: string,
|
||||
sites: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<Record<string, string>> {
|
||||
const qid = wikidataId.trim();
|
||||
if (!/^Q\d+$/.test(qid) || sites.length === 0) return {};
|
||||
const params = new URLSearchParams({
|
||||
action: 'wbgetentities',
|
||||
props: 'sitelinks',
|
||||
ids: qid,
|
||||
sitefilter: sites.join('|'),
|
||||
format: 'json',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return {};
|
||||
const data = (await res.json()) as {
|
||||
entities?: Record<string, { sitelinks?: Record<string, { title?: string }> }>;
|
||||
};
|
||||
const out: Record<string, string> = {};
|
||||
for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) {
|
||||
if (link?.title) out[site] = link.title;
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The pictures Wikidata records for a place, best first.
|
||||
*
|
||||
* By far the most accurate source there is: a person chose each of these to
|
||||
* represent this exact object, where a coordinate search only knows what was
|
||||
* photographed nearby. Wikidata also keeps them apart by what they show, so
|
||||
* asking for more than P18 buys genuine variety rather than another frame of
|
||||
* the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a
|
||||
* night shot, a panorama and a winter view, all curated.
|
||||
*
|
||||
* Two calls total whatever the item holds: one for the claims, one batch for
|
||||
* the file metadata.
|
||||
*/
|
||||
async fetchWikidataCandidates(
|
||||
wikidataId: string,
|
||||
limit = 5,
|
||||
): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> {
|
||||
const empty = { candidates: [], commonsCategory: null };
|
||||
const qid = wikidataId.trim();
|
||||
if (!/^Q\d+$/.test(qid)) return empty;
|
||||
try {
|
||||
const res = await fetch(
|
||||
`https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`,
|
||||
{ headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) },
|
||||
);
|
||||
if (!res.ok) return empty;
|
||||
const data = (await res.json()) as { entities?: Record<string, { claims?: WikidataClaims }> };
|
||||
const claims = data.entities?.[qid]?.claims;
|
||||
if (!claims) return empty;
|
||||
|
||||
const fileNames = wikidataImageClaims(claims, limit);
|
||||
const commonsCategory = claimValue(claims.P373) ?? null;
|
||||
if (fileNames.length === 0) return { candidates: [], commonsCategory };
|
||||
|
||||
const byTitle = await this.fetchCommonsFilesByName(fileNames);
|
||||
// Back into the order Wikidata implied, which the batch response loses.
|
||||
const candidates = fileNames.map((name) => byTitle.get(normalizeFileTitle(name))).filter((c): c is CommonsCandidate => !!c);
|
||||
return { candidates, commonsCategory };
|
||||
} catch {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for a list of Commons files, in one request.
|
||||
*
|
||||
* `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia
|
||||
* lead image often names a file that has since been renamed, and without it
|
||||
* the API answers with a `missing` page and the picture disappears silently.
|
||||
* Keyed by normalised title so callers can restore their own ordering.
|
||||
*/
|
||||
async fetchCommonsFilesByName(fileNames: string[]): Promise<Map<string, CommonsCandidate>> {
|
||||
const out = new Map<string, CommonsCandidate>();
|
||||
const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`));
|
||||
if (titles.length === 0) return out;
|
||||
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: titles.join('|'),
|
||||
redirects: '1',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return out;
|
||||
const data = (await res.json()) as {
|
||||
query?: {
|
||||
pages?: Record<string, WikiCommonsPage>;
|
||||
normalized?: { from: string; to: string }[];
|
||||
redirects?: { from: string; to: string }[];
|
||||
};
|
||||
};
|
||||
// The API renames titles twice on the way in (normalisation, then
|
||||
// redirects), so walk the chain back to what the caller asked for.
|
||||
const aliases = new Map<string, string>();
|
||||
for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) {
|
||||
aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from));
|
||||
}
|
||||
const resolveOriginal = (title: string): string => {
|
||||
let key = normalizeFileTitle(title);
|
||||
for (let hop = 0; hop < 4; hop++) {
|
||||
const previous = aliases.get(key);
|
||||
if (!previous || previous === key) break;
|
||||
key = previous;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) {
|
||||
if (!candidate.title) continue;
|
||||
out.set(resolveOriginal(candidate.title), candidate);
|
||||
// Also reachable under its own name, for callers that already resolved.
|
||||
out.set(normalizeFileTitle(candidate.title), candidate);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The lead image a wiki article picked for a place.
|
||||
*
|
||||
* Only the file NAME is taken from here; the bytes and the licence come from
|
||||
* the same Commons batch as everything else. The thumbnail URL the API offers
|
||||
* alongside it carries no attribution, and a picture we cannot credit is a
|
||||
* picture we cannot show.
|
||||
*/
|
||||
async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise<string | null> {
|
||||
const parsed = parseWikipediaTag(wikipediaTag);
|
||||
if (!parsed) return null;
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: parsed.title,
|
||||
prop: 'pageimages',
|
||||
piprop: 'name',
|
||||
redirects: '1',
|
||||
});
|
||||
for (const host of ['wikivoyage', 'wikipedia'] as const) {
|
||||
try {
|
||||
const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { pageimage?: string }> };
|
||||
};
|
||||
for (const page of Object.values(data.query?.pages ?? {})) {
|
||||
if (page.pageimage) return page.pageimage;
|
||||
}
|
||||
} catch {
|
||||
/* try the next wiki */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commons images from a category, which is the set of pictures OF a place.
|
||||
*
|
||||
* Preferred over the coordinate search wherever a place carries a
|
||||
* `wikimedia_commons` tag: geosearch around a city centre returns statues and
|
||||
* passers-by, while the category of a restaurant returns the restaurant.
|
||||
*/
|
||||
async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise<CommonsCandidate[]> {
|
||||
const name = normalizeCategoryName(category);
|
||||
if (!name) return [];
|
||||
// Overfetch: the ranker throws away survey imagery, diagrams and repeats,
|
||||
// and it can only do that from a pool bigger than the strip.
|
||||
const poolSize = String(Math.max(1, Math.min(limit * 3, 20)));
|
||||
|
||||
// `generator=search` first. `categorymembers` orders by sort key, i.e.
|
||||
// alphabetically by file name, which is not a quality signal in any
|
||||
// direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation,
|
||||
// a marathon photo and six near-identical press shots, and
|
||||
// "Category:Hamburg Airport" with a noise map and a terminal layout. The
|
||||
// search index at least ranks by how well a file matches its category.
|
||||
const search = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'search',
|
||||
gsrsearch: `incategory:"${name}" filetype:bitmap`,
|
||||
gsrnamespace: '6',
|
||||
gsrlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
const members = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'categorymembers',
|
||||
gcmtitle: `Category:${name}`,
|
||||
gcmtype: 'file',
|
||||
gcmlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
|
||||
for (const params of [search, members]) {
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize));
|
||||
if (hits.length) return hits;
|
||||
} catch {
|
||||
/* fall through to the second strategy */
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
// ── Search places (Google or Nominatim fallback) ───────────────────────────
|
||||
|
||||
async searchPlaces(
|
||||
@@ -1971,8 +1142,7 @@ export class MapsService {
|
||||
let providerFailed = false;
|
||||
|
||||
const fetchPromise = (async (): Promise<{ attribution: string | null } | null> => {
|
||||
await acquirePhotoFetchSlot();
|
||||
try {
|
||||
return withPhotoFetchSlot(async () => {
|
||||
const apiKey = await this.getMapsKey(userId);
|
||||
|
||||
// Coordinate-based Wikipedia/Wikimedia lookup. Used for coordinate-only
|
||||
@@ -1980,30 +1150,11 @@ export class MapsService {
|
||||
// so a place added via search still gets a marker image when Google returns
|
||||
// nothing. Returns null (without marking an error) so the caller decides.
|
||||
const fetchWikimediaFallback = async (): Promise<{ attribution: string | null } | null> => {
|
||||
if (Number.isNaN(lat) || Number.isNaN(lng)) return null;
|
||||
const outcome = await this.wiki.downloadPhoto(lat, lng, name);
|
||||
if (outcome.kind === 'failed') providerFailed = true;
|
||||
if (outcome.kind !== 'photo') return null;
|
||||
try {
|
||||
const wiki = await this.fetchWikimediaPhoto(lat, lng, name);
|
||||
if (!wiki) return null;
|
||||
// Follow redirects manually so each hop (the image URL can 3xx to a CDN
|
||||
// host) is re-validated against the SSRF guard, not just the first URL.
|
||||
const imgRes = await safeFetchFollow(
|
||||
wiki.photoUrl, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true },
|
||||
);
|
||||
if (!imgRes.ok) {
|
||||
providerFailed = true;
|
||||
return null;
|
||||
}
|
||||
if (exceedsDeclaredLength(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES)) {
|
||||
discardBody(imgRes);
|
||||
providerFailed = true;
|
||||
return null;
|
||||
}
|
||||
const { bytes, truncated } = await readCapped(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES);
|
||||
if (truncated || bytes.byteLength === 0) {
|
||||
providerFailed = true;
|
||||
return null;
|
||||
}
|
||||
const cached = await this.photoCache.put(placeId, bytes, wiki.attribution);
|
||||
const cached = await this.photoCache.put(placeId, outcome.bytes, outcome.attribution);
|
||||
return { attribution: cached.attribution };
|
||||
} catch {
|
||||
providerFailed = true;
|
||||
@@ -2046,9 +1197,7 @@ export class MapsService {
|
||||
|
||||
await this.photoCache.markError(placeId, providerFailed ? 'provider-error' : 'no-photo');
|
||||
return null;
|
||||
} finally {
|
||||
releasePhotoFetchSlot();
|
||||
}
|
||||
});
|
||||
})();
|
||||
|
||||
this.photoCache.setInFlight(placeId, fetchPromise);
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// ── Concurrency limiter for outbound photo fetches ───────────────────────────
|
||||
// Caps simultaneous Wikimedia/Google photo requests so a bulk import of hundreds
|
||||
// of places cannot monopolise the event loop or trigger external API rate limits.
|
||||
// Module-scoped ON PURPOSE (permissions-cache precedent): every consumer — the
|
||||
// marker photo in MapsService and the enrichment column — must share one limiter.
|
||||
|
||||
const MAX_CONCURRENT_PHOTO_FETCHES = 5;
|
||||
let photoFetchActive = 0;
|
||||
const photoFetchQueue: Array<() => void> = [];
|
||||
|
||||
function acquirePhotoFetchSlot(): Promise<void> {
|
||||
if (photoFetchActive < MAX_CONCURRENT_PHOTO_FETCHES) {
|
||||
photoFetchActive++;
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => photoFetchQueue.push(resolve));
|
||||
}
|
||||
|
||||
function releasePhotoFetchSlot(): void {
|
||||
const next = photoFetchQueue.shift();
|
||||
if (next) {
|
||||
next();
|
||||
} else {
|
||||
photoFetchActive--;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs an outbound photo fetch under the shared slot limit. Exported so the
|
||||
* enrichment module queues behind the same five slots — a picker that grabs
|
||||
* three images per selected place would otherwise sail past a cap the rest of
|
||||
* the app respects.
|
||||
*/
|
||||
export async function withPhotoFetchSlot<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await acquirePhotoFetchSlot();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
releasePhotoFetchSlot();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,830 @@
|
||||
/**
|
||||
* Wikipedia, Wikivoyage, Wikidata and Wikimedia Commons: descriptions,
|
||||
* curated pictures and brand logos for a place.
|
||||
*
|
||||
* Everything here is free and keyless, and every call carries TREK's user
|
||||
* agent and a deadline, because the enrichment column and the marker photo
|
||||
* answer a live dialog: a slow wiki is dropped rather than waited out. Commons
|
||||
* images are mostly CC BY / CC BY-SA, so a picture is only ever handed back
|
||||
* with the metadata needed to credit it.
|
||||
*
|
||||
* MapsService (the marker photo, the brand logo route) and the enrichment
|
||||
* column both ask through this client. It never imports MapsService:
|
||||
* providers sit below the orchestrator (lint:boundaries holds that).
|
||||
*/
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Jimp } from 'jimp';
|
||||
import { safeFetchFollow } from '../../../utils/ssrfGuard';
|
||||
import { discardBody, exceedsDeclaredLength, readCapped } from '../../../utils/cappedFetch';
|
||||
import { UA, stripWikiMarkup, parseWikipediaTag } from '../maps.helpers';
|
||||
|
||||
interface WikidataSnak {
|
||||
mainsnak?: { datavalue?: { value?: string } };
|
||||
rank?: 'preferred' | 'normal' | 'deprecated';
|
||||
}
|
||||
type WikidataClaims = Record<string, WikidataSnak[] | undefined>;
|
||||
|
||||
/**
|
||||
* Wikidata properties that name a picture of a place, in the order we want them.
|
||||
*
|
||||
* P18 is the representative image. The rest exist because a station or a
|
||||
* monument is not one view: asking for the interior, the night shot, the
|
||||
* panorama and the aerial gives a picker four genuinely different pictures
|
||||
* instead of four frames of the same façade.
|
||||
*/
|
||||
const WIKIDATA_IMAGE_PROPERTIES = [
|
||||
'P18', // image
|
||||
'P5775', // interior view
|
||||
'P3451', // night view
|
||||
'P8592', // aerial view
|
||||
'P4291', // panoramic view
|
||||
'P5252', // winter view
|
||||
'P948', // Wikivoyage banner
|
||||
] as const;
|
||||
|
||||
/** First non-empty string value of a claim list. */
|
||||
function claimValue(snaks: WikidataSnak[] | undefined): string | null {
|
||||
for (const snak of snaks ?? []) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value === 'string' && value.trim()) return value.trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* File names from an item's picture properties, best first.
|
||||
*
|
||||
* Within P18 the statement rank decides: an item with several images marks one
|
||||
* `preferred`, and that is the one an editor considers representative. The API
|
||||
* returns statements in edit order, not rank order, so taking `[0]` picks
|
||||
* whichever was added first — for the Brandenburg Gate that is a coin toss
|
||||
* between the morning shot and a wide overview.
|
||||
*/
|
||||
function wikidataImageClaims(claims: WikidataClaims, limit: number): string[] {
|
||||
const rankOrder = { preferred: 0, normal: 1, deprecated: 2 } as const;
|
||||
const names: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const property of WIKIDATA_IMAGE_PROPERTIES) {
|
||||
const snaks = [...(claims[property] ?? [])]
|
||||
.filter((snak) => snak.rank !== 'deprecated')
|
||||
.sort((a, b) => (rankOrder[a.rank ?? 'normal'] ?? 1) - (rankOrder[b.rank ?? 'normal'] ?? 1));
|
||||
for (const snak of snaks) {
|
||||
const value = snak.mainsnak?.datavalue?.value;
|
||||
if (typeof value !== 'string' || !value.trim()) continue;
|
||||
const key = normalizeFileTitle(value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
names.push(value.trim());
|
||||
if (names.length >= limit) return names;
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
/** `File:` prefix off, underscores and case normalised — Commons treats these as one title. */
|
||||
function normalizeFileTitle(title: string): string {
|
||||
return title.replace(/^File:/i, '').replaceAll('_', ' ').trim().toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* A bare Commons category name out of whatever an OSM `wikimedia_commons` tag
|
||||
* holds.
|
||||
*
|
||||
* The tag is free text and mappers put three different things in it: a bare
|
||||
* name, a prefixed `Category:…`, or — against the wiki's own advice — a single
|
||||
* `File:…`. Prefixing blindly turned the last one into `Category:File:X.jpg`,
|
||||
* which matches nothing and fell through to the coordinate search without a
|
||||
* word. Localised prefixes (`Kategorie:`) appear too.
|
||||
*/
|
||||
function normalizeCategoryName(raw: string): string | null {
|
||||
const value = raw.trim();
|
||||
if (!value) return null;
|
||||
// A file name is not a category, and there is nothing sensible to derive.
|
||||
if (/^(file|datei|image|bild)\s*:/i.test(value)) return null;
|
||||
return value.replace(/^(category|kategorie|categorie|categoría|categoria)\s*:/i, '').trim() || null;
|
||||
}
|
||||
|
||||
interface WikiCommonsPage {
|
||||
pageid?: number;
|
||||
title?: string;
|
||||
imageinfo?: {
|
||||
url?: string;
|
||||
thumburl?: string;
|
||||
mime?: string;
|
||||
width?: number;
|
||||
height?: number;
|
||||
/** The file description page — where the full licence terms live. */
|
||||
descriptionurl?: string;
|
||||
extmetadata?: {
|
||||
Artist?: { value?: string };
|
||||
LicenseShortName?: { value?: string };
|
||||
LicenseUrl?: { value?: string };
|
||||
UsageTerms?: { value?: string };
|
||||
/** Used to spot survey imagery and diagrams, which are not pictures of a place. */
|
||||
Categories?: { value?: string };
|
||||
ObjectName?: { value?: string };
|
||||
ImageDescription?: { value?: string };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
/**
|
||||
* One Commons image with everything needed to credit it. Commons is mostly
|
||||
* CC BY / CC BY-SA, so a candidate that cannot be attributed is not usable in a
|
||||
* picker — the fields are nullable because Commons metadata is user-maintained
|
||||
* and genuinely incomplete on some files, not because they are optional to show.
|
||||
*/
|
||||
export interface CommonsCandidate {
|
||||
photoUrl: string;
|
||||
attribution: string | null;
|
||||
license: string | null;
|
||||
licenseUrl: string | null;
|
||||
sourceUrl: string | null;
|
||||
/**
|
||||
* Commons page id — the only stable identity a file has across the four ways
|
||||
* we reach it. The thumbnail URL is not: the same file comes back from
|
||||
* commons.wikimedia.org and from a language Wikipedia with different query
|
||||
* strings, so deduplicating on the URL silently lets the same picture through
|
||||
* twice. It is also what keys the cached bytes, so it must survive.
|
||||
*/
|
||||
pageId: number | null;
|
||||
/** File page title, e.g. `File:Brandenburger Tor morgens.jpg`. */
|
||||
title: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
/** Free text used to reject survey imagery, floor plans and logos. */
|
||||
descriptors: string | null;
|
||||
}
|
||||
|
||||
/** Wikimedia is normally well under a second, but a cold TLS handshake from a
|
||||
* fresh container has been seen at eight. Enrichment answers a live dialog, so
|
||||
* a slow provider is dropped rather than waited out. */
|
||||
const WIKI_TIMEOUT_MS = 6000;
|
||||
|
||||
/**
|
||||
* The sitelinks call sits at the front of the description chain (identity,
|
||||
* then sitelinks, then the extract), so it gets the same tight deadline the
|
||||
* OSM identity lookup in front of it has.
|
||||
*/
|
||||
const SITELINKS_TIMEOUT_MS = 2500;
|
||||
|
||||
// ── Brand logos ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// A road trip corridor is mostly chains — Shell, Aral, JET — and the brand is the
|
||||
// fastest thing to recognise on a map. OSM carries `brand:wikidata` on most of them,
|
||||
// Wikidata carries the logo (P154), and Commons serves it.
|
||||
//
|
||||
// The bytes are proxied rather than linked so the browser never talks to Wikimedia:
|
||||
// one self-hosted instance asking for a handful of logos is a very different egress
|
||||
// profile from every visitor's browser announcing which petrol stations they are
|
||||
// looking at. The cache is in memory on purpose — a GET must not write to the DB, and
|
||||
// there are only so many fuel brands.
|
||||
const BRAND_LOGO_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
const BRAND_LOGO_CACHE_MAX = 300;
|
||||
/** Well past any logo; a Commons original can be a multi-megabyte SVG or print-res PNG. */
|
||||
const BRAND_LOGO_MAX_BYTES = 512 * 1024;
|
||||
/** A place photo is a full-size image rather than a 128px mark, so its own ceiling. */
|
||||
const WIKIMEDIA_PHOTO_MAX_BYTES = 8 * 1024 * 1024;
|
||||
const BRAND_LOGO_WIDTH = 128;
|
||||
/** The square the logo is centred in, and the breathing room around it. */
|
||||
const BRAND_LOGO_CANVAS = 96;
|
||||
const BRAND_LOGO_PADDING = 8;
|
||||
const WIKIDATA_ID_RE = /^Q[1-9][0-9]{0,11}$/;
|
||||
|
||||
export interface BrandLogo {
|
||||
bytes: Buffer;
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
/** What a marker-photo download from Wikimedia came back with. */
|
||||
export type WikimediaPhotoOutcome =
|
||||
| { kind: 'photo'; bytes: Buffer; attribution: string | null }
|
||||
/** Nothing photographed here (or no coordinates to look with): a plain miss. */
|
||||
| { kind: 'none' }
|
||||
/** Wikimedia refused, timed out or sent something unusable: remembered only briefly. */
|
||||
| { kind: 'failed' };
|
||||
|
||||
/**
|
||||
* Puts a logo on a background it can actually be seen against.
|
||||
*
|
||||
* Half the fuel brands ship a white wordmark with a transparent background —
|
||||
* TotalEnergies, Esso and JET among them — and on the white pill the marker used to
|
||||
* draw they were invisible. Which background is right is a property of the image, not
|
||||
* of the brand, so it is measured: the mean brightness of the pixels that are actually
|
||||
* opaque decides between a white and a near-black backdrop.
|
||||
*
|
||||
* Returns the flattened PNG, or null when the bytes cannot be read at all — the pin
|
||||
* then keeps its category icon, which is a fine outcome.
|
||||
*/
|
||||
async function flattenBrandLogo(bytes: Buffer): Promise<BrandLogo | null> {
|
||||
try {
|
||||
const image = await Jimp.read(bytes);
|
||||
const width = image.bitmap.width;
|
||||
const height = image.bitmap.height;
|
||||
if (!width || !height) return null;
|
||||
|
||||
let sum = 0;
|
||||
let opaque = 0;
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const idx = (y * width + x) * 4;
|
||||
const alpha = image.bitmap.data[idx + 3];
|
||||
// Half-transparent pixels are anti-aliasing, not the mark itself.
|
||||
if (alpha < 128) continue;
|
||||
const r = image.bitmap.data[idx];
|
||||
const g = image.bitmap.data[idx + 1];
|
||||
const b = image.bitmap.data[idx + 2];
|
||||
sum += 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
opaque++;
|
||||
}
|
||||
}
|
||||
// A logo with nothing opaque in it is not a logo.
|
||||
if (opaque === 0) return null;
|
||||
|
||||
// Square, with the logo scaled to fit inside it. Most brand marks are wide wordmarks
|
||||
// — TotalEnergies is three times as wide as it is tall — and a round pin showing the
|
||||
// middle of one is unreadable. Fitting it into a square means the pin can crop to a
|
||||
// circle without ever cutting the mark itself.
|
||||
const scale = Math.min(
|
||||
(BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / width,
|
||||
(BRAND_LOGO_CANVAS - 2 * BRAND_LOGO_PADDING) / height,
|
||||
);
|
||||
// Never upscale: a 16-pixel favicon blown up to 96 looks worse than a small one.
|
||||
if (scale < 1) image.scale(scale);
|
||||
|
||||
const light = sum / opaque > 150;
|
||||
const canvas = new Jimp({
|
||||
width: BRAND_LOGO_CANVAS,
|
||||
height: BRAND_LOGO_CANVAS,
|
||||
color: light ? 0x111827ff : 0xffffffff,
|
||||
});
|
||||
canvas.composite(
|
||||
image,
|
||||
Math.round((BRAND_LOGO_CANVAS - image.bitmap.width) / 2),
|
||||
Math.round((BRAND_LOGO_CANVAS - image.bitmap.height) / 2),
|
||||
);
|
||||
const out = await canvas.getBuffer('image/png');
|
||||
return { bytes: Buffer.from(out), contentType: 'image/png' };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class WikimediaClient {
|
||||
/** Brand id → logo bytes, or null for "asked, has none". Insertion-ordered, so the
|
||||
* oldest entry is the one evicted when it fills up. */
|
||||
private readonly brandLogoCache = new Map<string, { at: number; logo: BrandLogo | null }>();
|
||||
|
||||
/**
|
||||
* The logo of a brand, by its Wikidata id, as bytes.
|
||||
*
|
||||
* Two hops: Wikidata says which Commons file is the logo (property P154), Commons
|
||||
* serves a thumbnail of it. Both answers are cached, including "this brand has no
|
||||
* logo" — otherwise every map pan would ask Wikidata about the same supermarket
|
||||
* chain again. Returns null whenever anything is missing or unreadable; a marker
|
||||
* without a logo falls back to its category icon, which is a fine outcome.
|
||||
*/
|
||||
async brandLogo(wikidataId: string): Promise<BrandLogo | null> {
|
||||
if (!WIKIDATA_ID_RE.test(wikidataId)) return null;
|
||||
|
||||
const cached = this.brandLogoCache.get(wikidataId);
|
||||
if (cached && Date.now() - cached.at < BRAND_LOGO_CACHE_TTL_MS) return cached.logo;
|
||||
|
||||
const remember = (logo: BrandLogo | null): BrandLogo | null => {
|
||||
if (this.brandLogoCache.size >= BRAND_LOGO_CACHE_MAX) {
|
||||
const oldest = this.brandLogoCache.keys().next().value;
|
||||
if (oldest !== undefined) this.brandLogoCache.delete(oldest);
|
||||
}
|
||||
this.brandLogoCache.set(wikidataId, { at: Date.now(), logo });
|
||||
return logo;
|
||||
};
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
action: 'wbgetclaims',
|
||||
entity: wikidataId,
|
||||
property: 'P154',
|
||||
format: 'json',
|
||||
});
|
||||
const claimRes = await fetch(`https://www.wikidata.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!claimRes.ok) return remember(null);
|
||||
const claims = (await claimRes.json()) as {
|
||||
claims?: { P154?: { mainsnak?: { datavalue?: { value?: unknown } } }[] };
|
||||
};
|
||||
const file = claims.claims?.P154?.[0]?.mainsnak?.datavalue?.value;
|
||||
if (typeof file !== 'string' || !file.trim()) return remember(null);
|
||||
|
||||
// Special:FilePath renders a thumbnail at the width asked for and redirects to
|
||||
// the CDN, so each hop is re-checked by the guard rather than trusted.
|
||||
const url = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(file)}?width=${BRAND_LOGO_WIDTH}`;
|
||||
// The same six seconds the Wikidata hop above allows. Without a deadline
|
||||
// this waited on undici's five-minute default, holding a request context
|
||||
// and a socket per stalled logo while the pin sat on its fallback icon.
|
||||
const imgRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true });
|
||||
if (!imgRes.ok) return remember(null);
|
||||
|
||||
if (exceedsDeclaredLength(imgRes, BRAND_LOGO_MAX_BYTES)) {
|
||||
discardBody(imgRes);
|
||||
return remember(null);
|
||||
}
|
||||
// Streamed rather than buffered whole: a chunked answer declares no length,
|
||||
// so the post-check only ever ran after the bytes were already in memory.
|
||||
const { bytes, truncated } = await readCapped(imgRes, BRAND_LOGO_MAX_BYTES);
|
||||
if (truncated || bytes.byteLength === 0) return remember(null);
|
||||
|
||||
const contentType = imgRes.headers.get('content-type') ?? '';
|
||||
if (!contentType.startsWith('image/')) return remember(null);
|
||||
|
||||
return remember(await flattenBrandLogo(bytes));
|
||||
} catch {
|
||||
// An SSRF refusal on a redirect hop and a network failure end the same
|
||||
// way: no logo, remembered, and the pin keeps its category icon.
|
||||
return remember(null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The marker photo for a place without a Google one, as bytes and a credit.
|
||||
*
|
||||
* Used for coordinate-only (right-click) places and as the fallback when a
|
||||
* Google place yields no photo. The image URL can 3xx to a CDN host, so the
|
||||
* download follows redirects through the SSRF guard, every hop re-validated.
|
||||
*/
|
||||
async downloadPhoto(lat: number, lng: number, name?: string): Promise<WikimediaPhotoOutcome> {
|
||||
if (Number.isNaN(lat) || Number.isNaN(lng)) return { kind: 'none' };
|
||||
try {
|
||||
const wiki = await this.fetchWikimediaPhoto(lat, lng, name);
|
||||
if (!wiki) return { kind: 'none' };
|
||||
const imgRes = await safeFetchFollow(wiki.photoUrl, { signal: AbortSignal.timeout(WIKI_TIMEOUT_MS) }, { bypassInternalIpAllowed: true });
|
||||
if (!imgRes.ok) return { kind: 'failed' };
|
||||
if (exceedsDeclaredLength(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES)) {
|
||||
discardBody(imgRes);
|
||||
return { kind: 'failed' };
|
||||
}
|
||||
const { bytes, truncated } = await readCapped(imgRes, WIKIMEDIA_PHOTO_MAX_BYTES);
|
||||
if (truncated || bytes.byteLength === 0) return { kind: 'failed' };
|
||||
return { kind: 'photo', bytes, attribution: wiki.attribution };
|
||||
} catch {
|
||||
return { kind: 'failed' };
|
||||
}
|
||||
}
|
||||
|
||||
// ── Wikimedia Commons photo lookup ─────────────────────────────────────────
|
||||
|
||||
async fetchWikimediaPhoto(lat: number, lng: number, name?: string): Promise<{ photoUrl: string; attribution: string | null } | null> {
|
||||
// Strategy 1: Search Wikipedia for the place name -> get the article image
|
||||
if (name) {
|
||||
try {
|
||||
const searchParams = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: name,
|
||||
prop: 'pageimages',
|
||||
piprop: 'thumbnail',
|
||||
pithumbsize: '400',
|
||||
pilimit: '1',
|
||||
redirects: '1',
|
||||
});
|
||||
const res = await fetch(`https://en.wikipedia.org/w/api.php?${searchParams}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
// This runs inside one of the few shared photo-fetch slots; a stalled
|
||||
// answer would hold it for everyone.
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, { thumbnail?: { source?: string } }> } };
|
||||
const pages = data.query?.pages;
|
||||
if (pages) {
|
||||
for (const page of Object.values(pages)) {
|
||||
if (page.thumbnail?.source) {
|
||||
return { photoUrl: page.thumbnail.source, attribution: 'Wikipedia' };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* fall through to geosearch */
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: Wikimedia Commons geosearch by coordinates
|
||||
const candidates = await this.fetchCommonsCandidates(lat, lng, 5);
|
||||
const first = candidates[0];
|
||||
return first ? { photoUrl: first.photoUrl, attribution: first.attribution } : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Anything photographed near a coordinate, licence metadata included. The
|
||||
* bottom rung of the picture ladder, and the only one with no claim on the
|
||||
* subject at all.
|
||||
*
|
||||
* geosearch already returns up to `limit` files in a single request, so asking
|
||||
* for a whole strip costs the same as asking for one picture. Callers that only
|
||||
* want a single image (fetchWikimediaPhoto, and through it the marker photo)
|
||||
* take the first entry.
|
||||
*
|
||||
* 60 metres rather than the 300 it used to be. Three hundred metres in a city
|
||||
* centre is a whole block: the town hall, the church and the underground
|
||||
* entrance are all inside it, and every one of them outranks the doner shop
|
||||
* we were actually asked about. Sixty is roughly "the same building and its
|
||||
* neighbours", which is the widest a picture can be taken and still plausibly
|
||||
* show the place. It does not fix the real problem, which is that nobody
|
||||
* photographed the shop, but it stops the wrong answer from being confident.
|
||||
*/
|
||||
async fetchCommonsCandidates(lat: number, lng: number, limit = 5): Promise<CommonsCandidate[]> {
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'geosearch',
|
||||
ggsprimary: 'all',
|
||||
ggsnamespace: '6',
|
||||
ggsradius: '60',
|
||||
ggscoord: `${lat}|${lng}`,
|
||||
// Deliberately more than the caller asked for. Around anything worth
|
||||
// visiting the first few hits are survey tiles, passers-by and the
|
||||
// building next door; the ranker needs a pool to reject from, and
|
||||
// geosearch charges the same for one result as for twenty.
|
||||
ggslimit: String(Math.max(1, Math.min(Math.max(limit * 4, 8), 20))),
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
// A hanging provider must not hold the whole enrichment request open;
|
||||
// no pictures is a fine answer, a request that never returns is not.
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return [];
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
// Hand back the whole pool. fetchWikimediaPhoto still takes [0] and gets
|
||||
// what it always got; the enrichment column ranks before it cuts.
|
||||
return this.toCommonsCandidates(data.query?.pages, Number(params.get('ggslimit')));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Shared shaping for every Commons query (coordinate, category, Wikidata, batch). */
|
||||
private toCommonsCandidates(pages: Record<string, WikiCommonsPage> | undefined, limit: number): CommonsCandidate[] {
|
||||
if (!pages) return [];
|
||||
const out: CommonsCandidate[] = [];
|
||||
// entries(), not values(): the map key is the page id, and for the queries
|
||||
// that reach a file by title it is the only place the id appears.
|
||||
for (const [key, page] of Object.entries(pages)) {
|
||||
const info = page.imageinfo?.[0];
|
||||
// Only use actual photos (JPEG/PNG), skip SVGs and PDFs
|
||||
const mime = info?.mime || '';
|
||||
if (!info?.url || !(mime.startsWith('image/jpeg') || mime.startsWith('image/png'))) continue;
|
||||
const meta = info.extmetadata;
|
||||
const pageId = page.pageid ?? (Number.isInteger(Number(key)) ? Number(key) : null);
|
||||
out.push({
|
||||
// iiurlwidth=400 makes Commons also return a scaled thumburl. Prefer it —
|
||||
// info.url is the full-resolution original (multi-megapixel camera exports).
|
||||
photoUrl: info.thumburl ?? info.url,
|
||||
attribution: stripWikiMarkup(meta?.Artist?.value),
|
||||
license: stripWikiMarkup(meta?.LicenseShortName?.value) ?? stripWikiMarkup(meta?.UsageTerms?.value),
|
||||
licenseUrl: meta?.LicenseUrl?.value?.trim() || null,
|
||||
sourceUrl: info.descriptionurl || null,
|
||||
pageId: pageId && pageId > 0 ? pageId : null,
|
||||
title: page.title ?? null,
|
||||
width: info.width ?? null,
|
||||
height: info.height ?? null,
|
||||
descriptors:
|
||||
[
|
||||
stripWikiMarkup(meta?.ObjectName?.value),
|
||||
stripWikiMarkup(meta?.ImageDescription?.value),
|
||||
stripWikiMarkup(meta?.Categories?.value),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' | ') || null,
|
||||
});
|
||||
if (out.length >= limit) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lead paragraph of a wiki article, from Wikivoyage first and Wikipedia after.
|
||||
*
|
||||
* Wikivoyage is the travel sibling: same MediaWiki API, same CC BY-SA, but it
|
||||
* describes a place for someone about to go there, where Wikipedia opens with
|
||||
* area in square kilometres and pronunciation. Both are resolved from the OSM
|
||||
* `wikipedia` tag — guessing the article from the place name lands on the
|
||||
* wrong one for every ambiguous name, so no tag means no description rather
|
||||
* than a confident description of somewhere else.
|
||||
*/
|
||||
async fetchWikiExtract(
|
||||
wikipediaTag: string | null | undefined,
|
||||
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
|
||||
const parsed = parseWikipediaTag(wikipediaTag);
|
||||
if (!parsed) return null;
|
||||
for (const host of ['wikivoyage', 'wikipedia'] as const) {
|
||||
const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title);
|
||||
if (hit) return hit;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The lead paragraph of one named article on one named wiki.
|
||||
*
|
||||
* Split out from `fetchWikiExtract` because the article is not always found
|
||||
* through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag
|
||||
* at all (Berlin Hauptbahnhof is exactly that), and then the title comes from
|
||||
* the item's sitelinks instead.
|
||||
*/
|
||||
async fetchWikiExtractFor(
|
||||
host: 'wikivoyage' | 'wikipedia',
|
||||
lang: string,
|
||||
title: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
|
||||
if (!lang || !title) return null;
|
||||
// Two sentences, not three: this sits next to a form, and a fourth line of
|
||||
// prose pushes the pictures out of view.
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: title,
|
||||
prop: 'extracts',
|
||||
exintro: '1',
|
||||
explaintext: '1',
|
||||
exsentences: '2',
|
||||
redirects: '1',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { title?: string; extract?: string }> };
|
||||
};
|
||||
for (const page of Object.values(data.query?.pages ?? {})) {
|
||||
const text = page.extract?.trim();
|
||||
// A missing article comes back as a page with no extract, not a 404.
|
||||
if (!text) continue;
|
||||
const resolved = page.title ?? title;
|
||||
return {
|
||||
text,
|
||||
sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`,
|
||||
source: host,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Which articles a Wikidata item is linked to, for the wikis we care about.
|
||||
*
|
||||
* The way to an article when a place has a Wikidata id but no `wikipedia`
|
||||
* tag — which is most of them, because mappers add one or the other. One
|
||||
* request, a few hundred bytes.
|
||||
*/
|
||||
async fetchWikidataSitelinks(wikidataId: string, sites: string[], signal?: AbortSignal): Promise<Record<string, string>> {
|
||||
const qid = wikidataId.trim();
|
||||
if (!/^Q\d+$/.test(qid) || sites.length === 0) return {};
|
||||
const params = new URLSearchParams({
|
||||
action: 'wbgetentities',
|
||||
props: 'sitelinks',
|
||||
ids: qid,
|
||||
sitefilter: sites.join('|'),
|
||||
format: 'json',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: signal ?? AbortSignal.timeout(SITELINKS_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return {};
|
||||
const data = (await res.json()) as {
|
||||
entities?: Record<string, { sitelinks?: Record<string, { title?: string }> }>;
|
||||
};
|
||||
const out: Record<string, string> = {};
|
||||
for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) {
|
||||
if (link?.title) out[site] = link.title;
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The pictures Wikidata records for a place, best first.
|
||||
*
|
||||
* By far the most accurate source there is: a person chose each of these to
|
||||
* represent this exact object, where a coordinate search only knows what was
|
||||
* photographed nearby. Wikidata also keeps them apart by what they show, so
|
||||
* asking for more than P18 buys genuine variety rather than another frame of
|
||||
* the same burst — Berlin Hauptbahnhof has an exterior, two interiors, a
|
||||
* night shot, a panorama and a winter view, all curated.
|
||||
*
|
||||
* Two calls total whatever the item holds: one for the claims, one batch for
|
||||
* the file metadata.
|
||||
*/
|
||||
async fetchWikidataCandidates(
|
||||
wikidataId: string,
|
||||
limit = 5,
|
||||
): Promise<{ candidates: CommonsCandidate[]; commonsCategory: string | null }> {
|
||||
const empty = { candidates: [], commonsCategory: null };
|
||||
const qid = wikidataId.trim();
|
||||
if (!/^Q\d+$/.test(qid)) return empty;
|
||||
try {
|
||||
const res = await fetch(`https://www.wikidata.org/w/api.php?action=wbgetentities&props=claims&ids=${qid}&format=json`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return empty;
|
||||
const data = (await res.json()) as { entities?: Record<string, { claims?: WikidataClaims }> };
|
||||
const claims = data.entities?.[qid]?.claims;
|
||||
if (!claims) return empty;
|
||||
|
||||
const fileNames = wikidataImageClaims(claims, limit);
|
||||
const commonsCategory = claimValue(claims.P373) ?? null;
|
||||
if (fileNames.length === 0) return { candidates: [], commonsCategory };
|
||||
|
||||
const byTitle = await this.fetchCommonsFilesByName(fileNames);
|
||||
// Back into the order Wikidata implied, which the batch response loses.
|
||||
const candidates = fileNames
|
||||
.map((name) => byTitle.get(normalizeFileTitle(name)))
|
||||
.filter((c): c is CommonsCandidate => !!c);
|
||||
return { candidates, commonsCategory };
|
||||
} catch {
|
||||
return empty;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for a list of Commons files, in one request.
|
||||
*
|
||||
* `redirects=1` matters more than it looks: a Wikidata claim or a Wikipedia
|
||||
* lead image often names a file that has since been renamed, and without it
|
||||
* the API answers with a `missing` page and the picture disappears silently.
|
||||
* Keyed by normalised title so callers can restore their own ordering.
|
||||
*/
|
||||
async fetchCommonsFilesByName(fileNames: string[]): Promise<Map<string, CommonsCandidate>> {
|
||||
const out = new Map<string, CommonsCandidate>();
|
||||
const titles = fileNames.map((name) => (/^File:/i.test(name) ? name : `File:${name}`));
|
||||
if (titles.length === 0) return out;
|
||||
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: titles.join('|'),
|
||||
redirects: '1',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) return out;
|
||||
const data = (await res.json()) as {
|
||||
query?: {
|
||||
pages?: Record<string, WikiCommonsPage>;
|
||||
normalized?: { from: string; to: string }[];
|
||||
redirects?: { from: string; to: string }[];
|
||||
};
|
||||
};
|
||||
// The API renames titles twice on the way in (normalisation, then
|
||||
// redirects), so walk the chain back to what the caller asked for.
|
||||
const aliases = new Map<string, string>();
|
||||
for (const hop of [...(data.query?.normalized ?? []), ...(data.query?.redirects ?? [])]) {
|
||||
aliases.set(normalizeFileTitle(hop.to), normalizeFileTitle(hop.from));
|
||||
}
|
||||
const resolveOriginal = (title: string): string => {
|
||||
let key = normalizeFileTitle(title);
|
||||
for (let hop = 0; hop < 4; hop++) {
|
||||
const previous = aliases.get(key);
|
||||
if (!previous || previous === key) break;
|
||||
key = previous;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
for (const candidate of this.toCommonsCandidates(data.query?.pages, titles.length)) {
|
||||
if (!candidate.title) continue;
|
||||
out.set(resolveOriginal(candidate.title), candidate);
|
||||
// Also reachable under its own name, for callers that already resolved.
|
||||
out.set(normalizeFileTitle(candidate.title), candidate);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The lead image a wiki article picked for a place.
|
||||
*
|
||||
* Only the file NAME is taken from here; the bytes and the licence come from
|
||||
* the same Commons batch as everything else. The thumbnail URL the API offers
|
||||
* alongside it carries no attribution, and a picture we cannot credit is a
|
||||
* picture we cannot show.
|
||||
*/
|
||||
async fetchWikiLeadImageName(wikipediaTag: string | null | undefined): Promise<string | null> {
|
||||
const parsed = parseWikipediaTag(wikipediaTag);
|
||||
if (!parsed) return null;
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: parsed.title,
|
||||
prop: 'pageimages',
|
||||
piprop: 'name',
|
||||
redirects: '1',
|
||||
});
|
||||
for (const host of ['wikivoyage', 'wikipedia'] as const) {
|
||||
try {
|
||||
const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as {
|
||||
query?: { pages?: Record<string, { pageimage?: string }> };
|
||||
};
|
||||
for (const page of Object.values(data.query?.pages ?? {})) {
|
||||
if (page.pageimage) return page.pageimage;
|
||||
}
|
||||
} catch {
|
||||
/* try the next wiki */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commons images from a category, which is the set of pictures OF a place.
|
||||
*
|
||||
* Preferred over the coordinate search wherever a place carries a
|
||||
* `wikimedia_commons` tag: geosearch around a city centre returns statues and
|
||||
* passers-by, while the category of a restaurant returns the restaurant.
|
||||
*/
|
||||
async fetchCommonsCategoryCandidates(category: string, limit = 5): Promise<CommonsCandidate[]> {
|
||||
const name = normalizeCategoryName(category);
|
||||
if (!name) return [];
|
||||
// Overfetch: the ranker throws away survey imagery, diagrams and repeats,
|
||||
// and it can only do that from a pool bigger than the strip.
|
||||
const poolSize = String(Math.max(1, Math.min(limit * 3, 20)));
|
||||
|
||||
// `generator=search` first. `categorymembers` orders by sort key, i.e.
|
||||
// alphabetically by file name, which is not a quality signal in any
|
||||
// direction: "Category:Brandenburg Gate" opens with an .ogg pronunciation,
|
||||
// a marathon photo and six near-identical press shots, and
|
||||
// "Category:Hamburg Airport" with a noise map and a terminal layout. The
|
||||
// search index at least ranks by how well a file matches its category.
|
||||
const search = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'search',
|
||||
gsrsearch: `incategory:"${name}" filetype:bitmap`,
|
||||
gsrnamespace: '6',
|
||||
gsrlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
const members = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'categorymembers',
|
||||
gcmtitle: `Category:${name}`,
|
||||
gcmtype: 'file',
|
||||
gcmlimit: poolSize,
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|extmetadata|mime|size',
|
||||
iiurlwidth: '400',
|
||||
});
|
||||
|
||||
for (const params of [search, members]) {
|
||||
try {
|
||||
const res = await fetch(`https://commons.wikimedia.org/w/api.php?${params}`, {
|
||||
headers: { 'User-Agent': UA },
|
||||
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
|
||||
});
|
||||
if (!res.ok) continue;
|
||||
const data = (await res.json()) as { query?: { pages?: Record<string, WikiCommonsPage> } };
|
||||
const hits = this.toCommonsCandidates(data.query?.pages, Number(poolSize));
|
||||
if (hits.length) return hits;
|
||||
} catch {
|
||||
/* fall through to the second strategy */
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import { buildOsmDetails, isGooglePlaceId, parseWikipediaTag, rankCommonsCandida
|
||||
import { trekPlacesById } from '../maps/trek-places.client';
|
||||
import { GooglePlacesClient } from '../maps/providers/google-places.provider';
|
||||
import { OsmClient } from '../maps/providers/osm.client';
|
||||
import { WikimediaClient } from '../maps/providers/wikimedia.client';
|
||||
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
|
||||
import { readAppSetting } from '../common/app-settings.registry';
|
||||
|
||||
@@ -327,6 +328,7 @@ export class PlaceEnrichmentService {
|
||||
private readonly photoCache: PlacePhotoCacheService,
|
||||
private readonly googlePlaces: GooglePlacesClient,
|
||||
private readonly osm: OsmClient,
|
||||
private readonly wiki: WikimediaClient,
|
||||
) {}
|
||||
|
||||
/**
|
||||
@@ -490,21 +492,21 @@ export class PlaceEnrichmentService {
|
||||
let categoryName = identity.wikimedia_commons;
|
||||
|
||||
if (wikidata) {
|
||||
const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP);
|
||||
const fromWikidata = await this.wiki.fetchWikidataCandidates(wikidata, COMMONS_CAP);
|
||||
push(commonsPool, fromWikidata.candidates, 'wikidata');
|
||||
categoryName ??= fromWikidata.commonsCategory;
|
||||
}
|
||||
|
||||
if (commonsPool.length < COMMONS_CAP && wikipedia) {
|
||||
const leadName = await this.maps.fetchWikiLeadImageName(wikipedia);
|
||||
const leadName = await this.wiki.fetchWikiLeadImageName(wikipedia);
|
||||
if (leadName) {
|
||||
const byName = await this.maps.fetchCommonsFilesByName([leadName]);
|
||||
const byName = await this.wiki.fetchCommonsFilesByName([leadName]);
|
||||
push(commonsPool, [...byName.values()], 'wikipedia');
|
||||
}
|
||||
}
|
||||
|
||||
if (commonsPool.length < COMMONS_CAP && categoryName) {
|
||||
push(commonsPool, await this.maps.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category');
|
||||
push(commonsPool, await this.wiki.fetchCommonsCategoryCandidates(categoryName, COMMONS_CAP), 'category');
|
||||
}
|
||||
|
||||
// Two is the bar: one curated picture plus the nearby noise reads worse
|
||||
@@ -521,7 +523,7 @@ export class PlaceEnrichmentService {
|
||||
const skipNearby = nearbyWouldMislead(details) || nearbyWouldMislead(identity.osmTags);
|
||||
const nearbyPending =
|
||||
curated < 2 && !skipNearby
|
||||
? this.maps.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP)
|
||||
? this.wiki.fetchCommonsCandidates(req.lat, req.lng, COMMONS_CAP)
|
||||
: Promise.resolve([]);
|
||||
|
||||
const googleRefs = await googlePending;
|
||||
@@ -787,21 +789,21 @@ export class PlaceEnrichmentService {
|
||||
{ site: 'enwikivoyage', host: 'wikivoyage', lang: 'en' },
|
||||
{ site: 'enwiki', host: 'wikipedia', lang: 'en' },
|
||||
];
|
||||
const sitelinks = await this.maps.fetchWikidataSitelinks(
|
||||
const sitelinks = await this.wiki.fetchWikidataSitelinks(
|
||||
identity.wikidata,
|
||||
wanted.map((w) => w.site),
|
||||
);
|
||||
for (const { site, host, lang: hostLang } of wanted) {
|
||||
const title = sitelinks[site];
|
||||
if (!title) continue;
|
||||
const hit = await this.maps.fetchWikiExtractFor(host, hostLang, title);
|
||||
const hit = await this.wiki.fetchWikiExtractFor(host, hostLang, title);
|
||||
if (hit) return hit;
|
||||
}
|
||||
}
|
||||
|
||||
// No Wikidata id, or its sitelinks led nowhere: fall back to the tag, which
|
||||
// names an article directly.
|
||||
return identity.wikipedia ? this.maps.fetchWikiExtract(identity.wikipedia) : null;
|
||||
return identity.wikipedia ? this.wiki.fetchWikiExtract(identity.wikipedia) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -33,6 +33,7 @@ import { PlaceEnrichmentModule } from '../../src/nest/place-enrichment/place-enr
|
||||
import { candidateKey } from '../../src/nest/place-enrichment/place-enrichment.service';
|
||||
import { MapsService } from '../../src/nest/maps/maps.service';
|
||||
import { OsmClient } from '../../src/nest/maps/providers/osm.client';
|
||||
import { WikimediaClient } from '../../src/nest/maps/providers/wikimedia.client';
|
||||
import { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service';
|
||||
import { RateLimitService } from '../../src/nest/common/rate-limit.service';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
@@ -51,6 +52,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
let app: Awaited<ReturnType<typeof build>>;
|
||||
let maps: MapsService;
|
||||
let osm: OsmClient;
|
||||
let wiki: WikimediaClient;
|
||||
let orm: TestOrm;
|
||||
|
||||
async function build() {
|
||||
@@ -73,6 +75,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
|
||||
maps = app.get(MapsService);
|
||||
osm = app.get(OsmClient);
|
||||
wiki = app.get(WikimediaClient);
|
||||
const photoCache = app.get(PlacePhotoCacheService);
|
||||
vi.spyOn(photoCache, 'get').mockReturnValue(null);
|
||||
vi.spyOn(photoCache, 'put').mockImplementation(async (key: string, _b: Buffer, attribution: string | null) => ({
|
||||
@@ -93,19 +96,19 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
await app.get(RateLimitService).reset('place_enrichment');
|
||||
// spyOn hands back the same spy on a second call, so the call counts carry
|
||||
// over between tests unless they are cleared explicitly.
|
||||
vi.spyOn(maps, 'fetchCommonsCandidates').mockClear().mockResolvedValue([]);
|
||||
vi.spyOn(maps, 'fetchWikiExtract').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(maps, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]);
|
||||
vi.spyOn(wiki, 'fetchCommonsCandidates').mockClear().mockResolvedValue([]);
|
||||
vi.spyOn(wiki, 'fetchWikiExtract').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(wiki, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]);
|
||||
vi.spyOn(maps, 'details').mockClear().mockResolvedValue({ place: null });
|
||||
// Every provider the service can reach has to be stubbed here, not just the
|
||||
// ones a given case cares about: anything left open goes out over the real
|
||||
// network from CI, which is both slow and rude to the provider.
|
||||
vi.spyOn(osm, 'resolveOsmIdentity').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(maps, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({});
|
||||
vi.spyOn(maps, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(maps, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null });
|
||||
vi.spyOn(maps, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(maps, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map());
|
||||
vi.spyOn(wiki, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({});
|
||||
vi.spyOn(wiki, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(wiki, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null });
|
||||
vi.spyOn(wiki, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null);
|
||||
vi.spyOn(wiki, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map());
|
||||
});
|
||||
|
||||
it('401 without a session cookie', async () => {
|
||||
@@ -137,7 +140,7 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
});
|
||||
|
||||
it('200 with a Commons candidate carrying its licence', async () => {
|
||||
vi.spyOn(maps, 'fetchCommonsCandidates').mockResolvedValue([
|
||||
vi.spyOn(wiki, 'fetchCommonsCandidates').mockResolvedValue([
|
||||
{
|
||||
photoUrl: 'https://commons.org/thumb.jpg',
|
||||
attribution: 'Alice',
|
||||
@@ -175,13 +178,13 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ photos: [], description: null, facts: [], disabled: true });
|
||||
expect(maps.fetchCommonsCandidates).not.toHaveBeenCalled();
|
||||
expect(wiki.fetchCommonsCandidates).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('stays enabled on an instance that has never seen the setting', async () => {
|
||||
const res = await request(server).post('/api/maps/enrichment').set('Cookie', sessionCookie(1)).send(BODY);
|
||||
expect(res.body.disabled).toBeUndefined();
|
||||
expect(maps.fetchCommonsCandidates).toHaveBeenCalled();
|
||||
expect(wiki.fetchCommonsCandidates).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('serves the second request for the same place from the row cache', async () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { MapsService } from '../../src/nest/maps/maps.service';
|
||||
import { GooglePlacesClient } from '../../src/nest/maps/providers/google-places.provider';
|
||||
import { OsmClient } from '../../src/nest/maps/providers/osm.client';
|
||||
import { WikimediaClient } from '../../src/nest/maps/providers/wikimedia.client';
|
||||
import type { PlacePhotoCacheService } from '../../src/nest/place-photos/place-photo-cache.service';
|
||||
import type { AppSettingsRepository } from '../../src/db/repositories/AppSettings.repository';
|
||||
import type { UsersRepository } from '../../src/db/repositories/Users.repository';
|
||||
@@ -14,6 +15,7 @@ export interface MapsParts {
|
||||
svc: MapsService;
|
||||
google: GooglePlacesClient;
|
||||
osm: OsmClient;
|
||||
wiki: WikimediaClient;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -32,8 +34,9 @@ export function buildMapsParts(
|
||||
): MapsParts {
|
||||
const google = new GooglePlacesClient(googleQuota);
|
||||
const osm = new OsmClient();
|
||||
const svc = new MapsService(photoCache, appSettings, users, placeDetailsCache, places, googleQuota, google, osm);
|
||||
return { svc, google, osm };
|
||||
const wiki = new WikimediaClient();
|
||||
const svc = new MapsService(photoCache, appSettings, users, placeDetailsCache, places, googleQuota, google, osm, wiki);
|
||||
return { svc, google, osm, wiki };
|
||||
}
|
||||
|
||||
export function buildMapsService(
|
||||
|
||||
@@ -23,10 +23,9 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({
|
||||
SsrfBlockedError: class extends Error {},
|
||||
}));
|
||||
|
||||
import { buildMapsService } from '../../helpers/maps-service';
|
||||
import { noGoogleQuota } from '../../helpers/google-quota';
|
||||
import { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client';
|
||||
|
||||
const svcOf = () => buildMapsService({} as never, {} as never, {} as never, {} as never, {} as never, noGoogleQuota);
|
||||
const svcOf = () => new WikimediaClient();
|
||||
|
||||
const filePage = (over: Record<string, unknown> = {}) => ({
|
||||
pageid: 4711,
|
||||
|
||||
@@ -20,11 +20,10 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({
|
||||
}));
|
||||
|
||||
import { OsmClient } from '../../../src/nest/maps/providers/osm.client';
|
||||
import { buildMapsService } from '../../helpers/maps-service';
|
||||
import { noGoogleQuota } from '../../helpers/google-quota';
|
||||
import { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client';
|
||||
import { toWikiLang, haversineMetres, namesOverlap } from '../../../src/nest/maps/maps.helpers';
|
||||
|
||||
const svcOf = () => buildMapsService({} as never, {} as never, {} as never, {} as never, {} as never, noGoogleQuota);
|
||||
const wikiOf = () => new WikimediaClient();
|
||||
const osmOf = () => new OsmClient();
|
||||
|
||||
// The Brandenburg Gate and the underground station named after it, 250m apart.
|
||||
@@ -201,7 +200,7 @@ describe('fetchWikidataSitelinks', () => {
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svcOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']);
|
||||
const out = await wikiOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']);
|
||||
|
||||
expect(decodeURIComponent(String(fetchMock.mock.calls[0][0]))).toContain('sitefilter=dewikivoyage|dewiki|enwiki');
|
||||
expect(out).toEqual({ dewiki: 'Berlin Hauptbahnhof' });
|
||||
@@ -211,16 +210,16 @@ describe('fetchWikidataSitelinks', () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svcOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({});
|
||||
expect(await svcOf().fetchWikidataSitelinks('Q1', [])).toEqual({});
|
||||
expect(await wikiOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({});
|
||||
expect(await wikiOf().fetchWikidataSitelinks('Q1', [])).toEqual({});
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('MAPS-173: yields an empty map on a bad response or a throw', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
|
||||
expect(await wikiOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
|
||||
expect(await wikiOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -232,7 +232,7 @@ const placesStub = {
|
||||
// through a repository stub that flows into the SAME mockDbGet/mockDbRun/
|
||||
// mockInstanceGet/mockProviderGet functions, so they keep firing exactly as
|
||||
// they did for the legacy module.
|
||||
const { svc, google, osm } = buildMapsParts(photoCacheStub, appSettingsStub, usersStub, placeDetailsCacheStub, placesStub, noGoogleQuota);
|
||||
const { svc, google, osm, wiki } = buildMapsParts(photoCacheStub, appSettingsStub, usersStub, placeDetailsCacheStub, placesStub, noGoogleQuota);
|
||||
|
||||
/**
|
||||
* Switch the TREK Places index off for one case.
|
||||
@@ -1094,7 +1094,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Eiffel Tower');
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Eiffel Tower');
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.photoUrl).toBe('https://example.com/thumb.jpg');
|
||||
expect(result!.attribution).toBe('Wikipedia');
|
||||
@@ -1117,7 +1117,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
};
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiResponse).mockResolvedValueOnce(commonsResponse));
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.photoUrl).toBe('https://commons.org/img.jpg');
|
||||
expect(result!.attribution).toBe('Alice');
|
||||
@@ -1145,7 +1145,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
};
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiResponse).mockResolvedValueOnce(commonsResponse));
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.photoUrl).toBe('https://commons.org/thumb-400.jpg');
|
||||
expect(result!.attribution).toBe('Alice');
|
||||
@@ -1159,7 +1159,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
json: async () => ({ query: { pages: {} } }),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
@@ -1170,7 +1170,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
json: async () => ({ query: { pages: {} } }),
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -1191,7 +1191,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
'fetch',
|
||||
vi.fn().mockRejectedValueOnce(new Error('Wikipedia network error')).mockResolvedValueOnce(commonsResponse),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.photoUrl).toBe('https://commons.org/fallback.jpg');
|
||||
// no Artist in extmetadata -> attribution null
|
||||
@@ -1219,7 +1219,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
};
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(wikiNotOk).mockResolvedValueOnce(commonsResponse));
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3, 'Some Place');
|
||||
expect(result).toBeDefined();
|
||||
// HTML tags stripped from attribution
|
||||
expect(result!.attribution).toBe('Bob');
|
||||
@@ -1227,7 +1227,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
|
||||
it('MAPS-037e: returns null when Commons geosearch returns not ok', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false }));
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
@@ -1239,13 +1239,13 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
json: async () => ({ query: {} }),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('MAPS-037g: returns null when Commons fetch throws', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('Commons network error')));
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
@@ -1265,7 +1265,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
@@ -1291,7 +1291,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result!.photoUrl).toBe('https://commons.org/photo.png');
|
||||
expect(result!.attribution).toBe('Carol');
|
||||
});
|
||||
@@ -1312,7 +1312,7 @@ describe('fetchWikimediaPhoto (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const result = await svc.fetchWikimediaPhoto(48.8, 2.3);
|
||||
const result = await wiki.fetchWikimediaPhoto(48.8, 2.3);
|
||||
expect(result!.attribution).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -2946,7 +2946,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
json: async () => ({ query: { pages: { '1': page(), '2': page({ thumburl: 'https://commons.org/t2.jpg' }) } } }),
|
||||
}),
|
||||
);
|
||||
const out = await svc.fetchCommonsCandidates(48.8, 2.3, 5);
|
||||
const out = await wiki.fetchCommonsCandidates(48.8, 2.3, 5);
|
||||
expect(out).toHaveLength(2);
|
||||
expect(out[0]).toMatchObject({
|
||||
photoUrl: 'https://commons.org/thumb.jpg',
|
||||
@@ -2976,7 +2976,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const out = await svc.fetchCommonsCandidates(48.8, 2.3);
|
||||
const out = await wiki.fetchCommonsCandidates(48.8, 2.3);
|
||||
expect(out[0].license).toBe('Public domain');
|
||||
expect(out[0].attribution).toBeNull();
|
||||
expect(out[0].licenseUrl).toBeNull();
|
||||
@@ -3000,7 +3000,7 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const out = await svc.fetchCommonsCandidates(48.8, 2.3);
|
||||
const out = await wiki.fetchCommonsCandidates(48.8, 2.3);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0].photoUrl).toBe('https://commons.org/original.jpg');
|
||||
});
|
||||
@@ -3016,25 +3016,25 @@ describe('fetchCommonsCandidates (fetch stubbed)', () => {
|
||||
// survey tiles and the building next door, and the ranker can only reject
|
||||
// from a pool. Geosearch bills the same for one result as for twenty, so
|
||||
// the whole pool comes back and gets cut after ranking, not before.
|
||||
expect(await svc.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3);
|
||||
expect(await wiki.fetchCommonsCandidates(48.8, 2.3, 2)).toHaveLength(3);
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('ggslimit=8');
|
||||
|
||||
await svc.fetchCommonsCandidates(48.8, 2.3, 0);
|
||||
await wiki.fetchCommonsCandidates(48.8, 2.3, 0);
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('ggslimit=8');
|
||||
|
||||
await svc.fetchCommonsCandidates(48.8, 2.3, 99);
|
||||
await wiki.fetchCommonsCandidates(48.8, 2.3, 99);
|
||||
expect(String(fetchMock.mock.calls[2][0])).toContain('ggslimit=20');
|
||||
});
|
||||
|
||||
it('MAPS-121: returns an empty list on a bad response, missing pages or a throw', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: {} }) }));
|
||||
expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svc.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCandidates(1, 2)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3049,7 +3049,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(page('Museum Ludwig', ' Ein Museum in Köln. '));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svc.fetchWikiExtract('de:Museum Ludwig');
|
||||
const out = await wiki.fetchWikiExtract('de:Museum Ludwig');
|
||||
expect(out).toEqual({
|
||||
text: 'Ein Museum in Köln.',
|
||||
sourceUrl: 'https://de.wikivoyage.org/wiki/Museum%20Ludwig',
|
||||
@@ -3067,7 +3067,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => {
|
||||
.mockResolvedValueOnce(page('Museum Ludwig', 'Das Museum Ludwig ist ein Museum.'));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svc.fetchWikiExtract('de:Museum Ludwig');
|
||||
const out = await wiki.fetchWikiExtract('de:Museum Ludwig');
|
||||
expect(out).toMatchObject({ source: 'wikipedia' });
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('https://de.wikipedia.org/w/api.php');
|
||||
});
|
||||
@@ -3076,36 +3076,36 @@ describe('fetchWikiExtract (fetch stubbed)', () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue(page('X', 'Kurz.'));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svc.fetchWikiExtract('de:X');
|
||||
await wiki.fetchWikiExtract('de:X');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('exsentences=2');
|
||||
});
|
||||
|
||||
it('MAPS-123: prefers the resolved title so a redirect links to where it landed', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(page('Eiffel Tower', 'A tower.')));
|
||||
const out = await svc.fetchWikiExtract('en:Eiffelturm');
|
||||
const out = await wiki.fetchWikiExtract('en:Eiffelturm');
|
||||
expect(out!.sourceUrl).toBe('https://en.wikivoyage.org/wiki/Eiffel%20Tower');
|
||||
});
|
||||
|
||||
it('MAPS-124: returns null without calling out when the tag has no language', async () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
expect(await svc.fetchWikiExtract('Museum Ludwig')).toBeNull();
|
||||
expect(await svc.fetchWikiExtract(null)).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract('Museum Ludwig')).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract(null)).toBeNull();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('MAPS-125: treats a miss on both wikis as no description', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(noArticle));
|
||||
expect(await svc.fetchWikiExtract('de:X')).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract('de:X')).toBeNull();
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({}) }));
|
||||
expect(await svc.fetchWikiExtract('de:X')).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract('de:X')).toBeNull();
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect(await svc.fetchWikiExtract('de:X')).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract('de:X')).toBeNull();
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svc.fetchWikiExtract('de:X')).toBeNull();
|
||||
expect(await wiki.fetchWikiExtract('de:X')).toBeNull();
|
||||
});
|
||||
|
||||
it('MAPS-125b: still tries Wikipedia after Wikivoyage threw', async () => {
|
||||
@@ -3115,7 +3115,7 @@ describe('fetchWikiExtract (fetch stubbed)', () => {
|
||||
.mockResolvedValueOnce(page('X', 'Ein Ort.'));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svc.fetchWikiExtract('de:X')).toMatchObject({ source: 'wikipedia' });
|
||||
expect(await wiki.fetchWikiExtract('de:X')).toMatchObject({ source: 'wikipedia' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3143,7 +3143,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => {
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const out = await svc.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3);
|
||||
const out = await wiki.fetchCommonsCategoryCandidates('Category:Museum Ludwig', 3);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0]).toMatchObject({ photoUrl: 'https://commons.org/t.jpg', attribution: 'Alice', license: 'CC BY 4.0' });
|
||||
// Ranked search, not the category listing: `categorymembers` orders by sort
|
||||
@@ -3157,7 +3157,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svc.fetchCommonsCategoryCandidates('Museum Ludwig');
|
||||
await wiki.fetchCommonsCategoryCandidates('Museum Ludwig');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22');
|
||||
// Empty search falls through to the category listing as a second chance.
|
||||
expect(String(fetchMock.mock.calls[1][0])).toContain('gcmtitle=Category%3AMuseum+Ludwig');
|
||||
@@ -3170,7 +3170,7 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => {
|
||||
const fetchMock = vi.fn();
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
expect(await svc.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCategoryCandidates('File:Museum Ludwig.jpg')).toEqual([]);
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -3178,16 +3178,16 @@ describe('fetchCommonsCategoryCandidates (fetch stubbed)', () => {
|
||||
const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ query: { pages: {} } }) });
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await svc.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig');
|
||||
await wiki.fetchCommonsCategoryCandidates('Kategorie:Museum Ludwig');
|
||||
expect(String(fetchMock.mock.calls[0][0])).toContain('incategory%3A%22Museum+Ludwig%22');
|
||||
});
|
||||
|
||||
it('MAPS-125e: yields nothing on an error response or a throw', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
|
||||
expect(await svc.fetchCommonsCategoryCandidates('Category:X')).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCategoryCandidates('Category:X')).toEqual([]);
|
||||
|
||||
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
|
||||
expect(await svc.fetchCommonsCategoryCandidates('Category:X')).toEqual([]);
|
||||
expect(await wiki.fetchCommonsCategoryCandidates('Category:X')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -34,10 +34,11 @@ import { readBrandIdentity } from '../../../src/nest/maps/maps.service';
|
||||
import type { MapsService } from '../../../src/nest/maps/maps.service';
|
||||
import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider';
|
||||
import type { OsmClient } from '../../../src/nest/maps/providers/osm.client';
|
||||
import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client';
|
||||
|
||||
/** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */
|
||||
type Seams<T> = { [K in keyof T]: T[K] };
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient>;
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient> & Seams<WikimediaClient>;
|
||||
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
|
||||
import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository';
|
||||
import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository';
|
||||
@@ -90,7 +91,7 @@ function cacheRepoStub(): PlaceDetailsCacheRepository {
|
||||
return { findEntry: mockFindEntry, upsertEntry: mockUpsertEntry } as unknown as PlaceDetailsCacheRepository;
|
||||
}
|
||||
|
||||
const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient);
|
||||
const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient);
|
||||
|
||||
/** A place whose own identity is empty but that belongs to a chain. */
|
||||
const branchOfAChain = (over: Partial<Record<keyof EnrichmentSeams, unknown>> = {}) =>
|
||||
|
||||
@@ -38,10 +38,11 @@ import { PlaceEnrichmentService } from '../../../src/nest/place-enrichment/place
|
||||
import type { MapsService } from '../../../src/nest/maps/maps.service';
|
||||
import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider';
|
||||
import type { OsmClient } from '../../../src/nest/maps/providers/osm.client';
|
||||
import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client';
|
||||
|
||||
/** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */
|
||||
type Seams<T> = { [K in keyof T]: T[K] };
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient>;
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient> & Seams<WikimediaClient>;
|
||||
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
|
||||
import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository';
|
||||
import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository';
|
||||
@@ -89,7 +90,7 @@ function cacheRepoStub(): PlaceDetailsCacheRepository {
|
||||
return { findEntry: mockFindEntry, upsertEntry: mockUpsertEntry } as unknown as PlaceDetailsCacheRepository;
|
||||
}
|
||||
|
||||
const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient);
|
||||
const make = (maps: EnrichmentSeams) => new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cacheStub(), maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient);
|
||||
|
||||
beforeEach(() => {
|
||||
mockGetValue.mockReset();
|
||||
|
||||
@@ -53,10 +53,11 @@ import {
|
||||
import type { MapsService } from '../../../src/nest/maps/maps.service';
|
||||
import type { GooglePlacesClient } from '../../../src/nest/maps/providers/google-places.provider';
|
||||
import type { OsmClient } from '../../../src/nest/maps/providers/osm.client';
|
||||
import type { WikimediaClient } from '../../../src/nest/maps/providers/wikimedia.client';
|
||||
|
||||
/** Every seam enrichment reaches: the maps orchestrator and the outbound clients it injects beside it. */
|
||||
type Seams<T> = { [K in keyof T]: T[K] };
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient>;
|
||||
type EnrichmentSeams = Seams<MapsService> & Seams<GooglePlacesClient> & Seams<OsmClient> & Seams<WikimediaClient>;
|
||||
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
|
||||
import type { PlaceDetailsCacheRepository } from '../../../src/db/repositories/PlaceDetailsCache.repository';
|
||||
import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository';
|
||||
@@ -109,7 +110,7 @@ function cacheStub(over: Record<string, unknown> = {}) {
|
||||
}
|
||||
|
||||
function make(maps: EnrichmentSeams, cache: PlacePhotoCacheService) {
|
||||
return new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cache, maps as unknown as GooglePlacesClient, maps as unknown as OsmClient);
|
||||
return new PlaceEnrichmentService(cacheRepoStub(), appSettingsStub(), maps as unknown as MapsService, cache, maps as unknown as GooglePlacesClient, maps as unknown as OsmClient, maps as unknown as WikimediaClient);
|
||||
}
|
||||
|
||||
let candidateSeq = 0;
|
||||
|
||||
Reference in New Issue
Block a user