Give a Google place its free half back

This commit is contained in:
Maurice
2026-08-10 09:53:08 +02:00
committed by Maurice
parent 8f9af76204
commit 0c8788ce44
7 changed files with 872 additions and 45 deletions
+67
View File
@@ -40,6 +40,73 @@ export function toApiLang(lang: string | undefined, fallback = 'en'): string {
return API_LANG_OVERRIDES[code] ?? code;
}
/**
* A wiki subdomain out of a TREK language code.
*
* Not the same normalisation as `toApiLang`, and the difference bites: TREK
* calls Brazilian Portuguese `br`, but `br.wikipedia.org` exists and is the
* BRETON Wikipedia. Passing the raw code through would not fail — it would
* quietly return Breton articles. `pt-br.wikipedia.org` does not resolve at
* all, so the region subtag has to go as well.
*/
const WIKI_LANG_OVERRIDES: Record<string, string> = {
br: 'pt',
gr: 'el',
};
export function toWikiLang(lang: string | undefined, fallback = 'en'): string {
const raw = (lang || '').trim();
if (!raw) return fallback;
const mapped = WIKI_LANG_OVERRIDES[raw] ?? raw;
const base = mapped.split('-')[0].toLowerCase();
return /^[a-z]{2,3}$/.test(base) ? base : fallback;
}
/** Metres between two coordinates. Spherical earth is plenty at this scale. */
export function haversineMetres(aLat: number, aLng: number, bLat: number, bLng: number): number {
const toRad = (deg: number) => (deg * Math.PI) / 180;
const dLat = toRad(bLat - aLat);
const dLng = toRad(bLng - aLng);
const h =
Math.sin(dLat / 2) ** 2 + Math.cos(toRad(aLat)) * Math.cos(toRad(bLat)) * Math.sin(dLng / 2) ** 2;
return 2 * 6371000 * Math.asin(Math.min(1, Math.sqrt(h)));
}
/**
* Whether two place names plausibly refer to the same thing.
*
* The gate that keeps a coordinate-based lookup from confidently describing the
* wrong building. "Hamburg Airport" and "Flughafen Hamburg" pass on "hamburg";
* "Hamburg Airport" and "Bahnhof Ohlsdorf" do not.
*
* One shared word is enough only when it carries some weight — four letters or
* more. Otherwise two have to match. A single short word is almost always an
* article or a generic noun, and TREK speaks 23 languages, so "Der Kiosk" and
* "Der Bahnhof" would sail through a plain word-overlap test while a stopword
* list for all of them is its own maintenance problem. Anything the pair rule
* lets through has already survived the distance check.
*/
export function namesOverlap(a: string, b: string): boolean {
const words = (value: string): Set<string> =>
new Set(
value
.toLowerCase()
.normalize('NFD')
.replace(/[̀-ͯ]/g, '')
.split(/[^a-z0-9]+/)
.filter((word) => word.length > 2),
);
const left = words(a);
if (left.size === 0) return false;
let shared = 0;
for (const word of words(b)) {
if (!left.has(word)) continue;
if (word.length >= 4) return true;
if (++shared >= 2) return true;
}
return false;
}
const GOOGLE_FTID_RE = /^0x[0-9a-f]+:0x[0-9a-f]+$/i;
// Extracts a Google Maps feature id (ftid, 0x..:0x..) from a URL's ?ftid= param.
+189 -25
View File
@@ -27,6 +27,9 @@ import {
resolveOverpassTimeoutMs,
stripWikiMarkup,
parseWikipediaTag,
toWikiLang,
haversineMetres,
namesOverlap,
type GoogleOpeningHours,
type OverpassPoi,
} from './maps.helpers';
@@ -275,6 +278,12 @@ interface GooglePlaceDetails extends GooglePlaceResult {
// a slow provider is dropped rather than waited out.
const WIKI_TIMEOUT_MS = 6000;
// Tighter than the wiki calls, because this one sits at the FRONT of a chain:
// identity, then sitelinks, then the extract. Nominatim answers a bounded
// search in 0.2-0.7s in practice, so anything past a couple of seconds is a bad
// day at the provider rather than a slow answer worth waiting for.
const IDENTITY_TIMEOUT_MS = 2500;
const MAX_CONCURRENT_PHOTO_FETCHES = 5;
let photoFetchActive = 0;
const photoFetchQueue: Array<() => void> = [];
@@ -552,6 +561,91 @@ export class MapsService {
});
}
/**
* Finds the OpenStreetMap record for a place we only know by name and
* coordinate, and hands back its tags.
*
* This is what gives a Google place a free identity. Google's payload has no
* `wikidata`, no `wikipedia` and no `wikimedia_commons` — it never had — so
* without this the entire free half of the enrichment column is unreachable
* for anyone who configured a Google key, which is the opposite of how this
* feature is meant to work. OSM knows these places perfectly well; nobody was
* asking it.
*
* Two gates keep it from describing the wrong building, because a confident
* description of somewhere else is worse than none:
* - the match has to be within `maxDistanceM` of where we are looking, and
* - it has to share a substantial word with the name we are looking for.
* Among what survives, Nominatim's own `importance` decides — that is what
* separates the Brandenburg Gate from the underground station named after it.
*/
async resolveOsmIdentity(
name: string,
lat: number,
lng: number,
opts: { lang?: string; maxDistanceM?: number; signal?: AbortSignal } = {},
): Promise<{ tags: Record<string, string>; osmUrl: string | null; matchedName: string } | null> {
const query = (name || '').trim();
if (!query || !Number.isFinite(lat) || !Number.isFinite(lng)) return null;
const maxDistanceM = opts.maxDistanceM ?? 2000;
// ~2km box around the point, so Nominatim ranks locally instead of handing
// back the most famous place on earth with this name.
const d = 0.02;
const params = new URLSearchParams({
q: query,
format: 'jsonv2',
extratags: '1',
limit: '5',
bounded: '1',
viewbox: `${lng - d},${lat - d},${lng + d},${lat + d}`,
'accept-language': toApiLang(opts.lang),
});
try {
const res = await fetch(`https://nominatim.openstreetmap.org/search?${params}`, {
headers: { 'User-Agent': UA },
signal: opts.signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS),
});
if (!res.ok) return null;
// Nominatim answers rate limiting in plain text, not JSON.
const data = (await res.json()) as (NominatimResult & { importance?: number })[];
if (!Array.isArray(data)) return null;
const best = data
.map((item) => ({
item,
lat: parseFloat(item.lat),
lng: parseFloat(item.lon),
}))
.filter(({ item, lat: hitLat, lng: hitLng }) => {
if (!Number.isFinite(hitLat) || !Number.isFinite(hitLng)) return false;
if (haversineMetres(lat, lng, hitLat, hitLng) > maxDistanceM) return false;
const label = item.name || item.display_name?.split(',')[0] || '';
return namesOverlap(query, label);
})
.sort((a, b) => {
const byImportance = (b.item.importance ?? 0) - (a.item.importance ?? 0);
if (byImportance !== 0) return byImportance;
return (
haversineMetres(lat, lng, a.lat, a.lng) - haversineMetres(lat, lng, b.lat, b.lng)
);
})[0];
if (!best) return null;
return {
tags: best.item.extratags ?? {},
osmUrl:
best.item.osm_type && best.item.osm_id
? `https://www.openstreetmap.org/${best.item.osm_type}/${best.item.osm_id}`
: null,
matchedName: best.item.name || best.item.display_name?.split(',')[0] || query,
};
} catch {
return null;
}
}
// ── Nominatim lookup (by OSM ID) ───────────────────────────────────────────
async lookupNominatim(
@@ -871,35 +965,105 @@ export class MapsService {
});
for (const host of ['wikivoyage', 'wikipedia'] as const) {
try {
const res = await fetch(`https://${parsed.lang}.${host}.org/w/api.php?${params}`, {
headers: { 'User-Agent': UA },
signal: AbortSignal.timeout(WIKI_TIMEOUT_MS),
});
if (!res.ok) continue;
const data = (await res.json()) as {
query?: { pages?: Record<string, { title?: string; extract?: string }> };
};
const pages = data.query?.pages;
if (!pages) continue;
for (const page of Object.values(pages)) {
const text = page.extract?.trim();
// A missing article comes back as a page with no extract, not a 404.
if (!text) continue;
const title = page.title ?? parsed.title;
return {
text,
sourceUrl: `https://${parsed.lang}.${host}.org/wiki/${encodeURIComponent(title)}`,
source: host,
};
}
} catch {
/* try the next wiki */
}
const hit = await this.fetchWikiExtractFor(host, parsed.lang, parsed.title);
if (hit) return hit;
}
return null;
}
/**
* The lead paragraph of one named article on one named wiki.
*
* Split out from `fetchWikiExtract` because the article is not always found
* through an OSM tag: a place can carry a Wikidata id and no `wikipedia` tag
* at all (Berlin Hauptbahnhof is exactly that), and then the title comes from
* the item's sitelinks instead.
*/
async fetchWikiExtractFor(
host: 'wikivoyage' | 'wikipedia',
lang: string,
title: string,
signal?: AbortSignal,
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
if (!lang || !title) return null;
// Two sentences, not three: this sits next to a form, and a fourth line of
// prose pushes the pictures out of view.
const params = new URLSearchParams({
action: 'query',
format: 'json',
titles: title,
prop: 'extracts',
exintro: '1',
explaintext: '1',
exsentences: '2',
redirects: '1',
});
try {
const res = await fetch(`https://${lang}.${host}.org/w/api.php?${params}`, {
headers: { 'User-Agent': UA },
signal: signal ?? AbortSignal.timeout(WIKI_TIMEOUT_MS),
});
if (!res.ok) return null;
const data = (await res.json()) as {
query?: { pages?: Record<string, { title?: string; extract?: string }> };
};
for (const page of Object.values(data.query?.pages ?? {})) {
const text = page.extract?.trim();
// A missing article comes back as a page with no extract, not a 404.
if (!text) continue;
const resolved = page.title ?? title;
return {
text,
sourceUrl: `https://${lang}.${host}.org/wiki/${encodeURIComponent(resolved)}`,
source: host,
};
}
return null;
} catch {
return null;
}
}
/**
* Which articles a Wikidata item is linked to, for the wikis we care about.
*
* The way to an article when a place has a Wikidata id but no `wikipedia`
* tag — which is most of them, because mappers add one or the other. One
* request, a few hundred bytes.
*/
async fetchWikidataSitelinks(
wikidataId: string,
sites: string[],
signal?: AbortSignal,
): Promise<Record<string, string>> {
const qid = wikidataId.trim();
if (!/^Q\d+$/.test(qid) || sites.length === 0) return {};
const params = new URLSearchParams({
action: 'wbgetentities',
props: 'sitelinks',
ids: qid,
sitefilter: sites.join('|'),
format: 'json',
});
try {
const res = await fetch(`https://www.wikidata.org/w/api.php?${params}`, {
headers: { 'User-Agent': UA },
signal: signal ?? AbortSignal.timeout(IDENTITY_TIMEOUT_MS),
});
if (!res.ok) return {};
const data = (await res.json()) as {
entities?: Record<string, { sitelinks?: Record<string, { title?: string }> }>;
};
const out: Record<string, string> = {};
for (const [site, link] of Object.entries(data.entities?.[qid]?.sitelinks ?? {})) {
if (link?.title) out[site] = link.title;
}
return out;
} catch {
return {};
}
}
/**
* The pictures Wikidata records for a place, best first.
*
@@ -9,8 +9,8 @@ import type {
} from '@trek/shared';
import { safeFetchFollow } from '../../utils/ssrfGuard';
import { DatabaseService } from '../database/database.service';
import { MapsService, withPhotoFetchSlot, type CommonsCandidate } from '../maps/maps.service';
import { isGooglePlaceId, rankCommonsCandidates } from '../maps/maps.helpers';
import { MapsService, readWikiIdentity, withPhotoFetchSlot, type CommonsCandidate, type WikiIdentity } from '../maps/maps.service';
import { isGooglePlaceId, parseWikipediaTag, rankCommonsCandidates, toWikiLang } from '../maps/maps.helpers';
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
/**
@@ -59,6 +59,15 @@ export function candidateKey(placeId: string, identity: string): string {
return `${placeId}~p${digest}`;
}
/**
* What the free sources need to know about a place, kept apart from whatever
* the maps provider returned. See `resolveIdentity` for why they do not merge.
*/
interface PlaceIdentity extends WikiIdentity {
/** Full OSM tag set, when the identity came from a Nominatim lookup. */
osmTags: Record<string, string> | null;
}
/** A ranked Commons pick, tagged with the rung of the ladder it came from. */
type CommonsPick = CommonsCandidate & { rung: 'wikidata' | 'wikipedia' | 'category' | 'nearby' };
@@ -204,10 +213,11 @@ export class PlaceEnrichmentService {
// comes along with the request. Refetching cost 12.8 seconds on a large
// OSM relation, and it ran in parallel with the client's own lookup.
const details = req.details ?? (await this.readDetails(userId, placeId, lang));
const identity = await this.resolveIdentity(req, details);
const [photos, description] = await Promise.all([
this.collectPhotos(userId, placeId, req, details),
this.collectDescription(userId, placeId, req, details),
this.collectPhotos(userId, placeId, req, identity),
this.collectDescription(userId, placeId, req, details, identity),
]);
const result: CachedEnrichment = { photos, description, facts: collectFacts(details) };
@@ -226,20 +236,53 @@ export class PlaceEnrichmentService {
return { credit: this.photoCache.get(key)?.attribution ?? null };
}
/**
* Which encyclopaedia entry, Wikidata item and Commons category describe this
* place.
*
* Kept beside the provider payload rather than merged into it, and that is
* not tidiness. `collectFacts` and the OSM branch of `collectDescription`
* both gate on `details.source`, so OSM tags folded into a Google payload
* would be carried around and never read — or, worse, would make a Google
* place start claiming to be an OpenStreetMap one.
*
* The lookup only runs when the payload brought nothing, which is every
* Google place: Google's response has no wiki tags and never had, so before
* this a configured API key silently switched the free half of this feature
* off.
*/
private async resolveIdentity(
req: MapsPlaceEnrichmentRequest,
details: Record<string, unknown> | null,
): Promise<PlaceIdentity> {
const fromPayload = (key: string): string | null =>
typeof details?.[key] === 'string' && (details[key] as string).trim() ? (details[key] as string).trim() : null;
const carried: PlaceIdentity = {
wikipedia: fromPayload('wikipedia'),
wikidata: fromPayload('wikidata'),
wikimedia_commons: fromPayload('wikimedia_commons'),
osmTags: null,
};
if (carried.wikipedia || carried.wikidata || carried.wikimedia_commons) return carried;
const resolved = await this.maps.resolveOsmIdentity(req.name, req.lat, req.lng, { lang: req.lang });
if (!resolved) return carried;
return { ...readWikiIdentity(resolved.tags), osmTags: resolved.tags };
}
// ── Photos ─────────────────────────────────────────────────────────────────
private async collectPhotos(
userId: number,
placeId: string,
req: MapsPlaceEnrichmentRequest,
details: Record<string, unknown> | null,
identity: PlaceIdentity,
): Promise<PlacePhotoCandidate[]> {
const apiKey = this.maps.getMapsKey(userId);
const wantsGoogle = !!apiKey && !this.maps.photosDisabled() && isGooglePlaceId(placeId);
const tag = (key: string): string | null => (typeof details?.[key] === 'string' ? (details[key] as string) : null);
const wikidata = tag('wikidata');
const wikipedia = tag('wikipedia');
const { wikidata, wikipedia } = identity;
// Google's listing is one call for the whole strip and knows nothing about
// the free sources, so it runs alongside them rather than in the ladder.
@@ -258,7 +301,7 @@ export class PlaceEnrichmentService {
// gate by the underground station beneath it. It is the fallback, and it
// only runs when the ladder above came up short.
const commonsPool: CommonsPick[] = [];
let categoryName = tag('wikimedia_commons');
let categoryName = identity.wikimedia_commons;
if (wikidata) {
const fromWikidata = await this.maps.fetchWikidataCandidates(wikidata, COMMONS_CAP);
@@ -392,6 +435,7 @@ export class PlaceEnrichmentService {
placeId: string,
req: MapsPlaceEnrichmentRequest,
details: Record<string, unknown> | null,
identity: PlaceIdentity,
): Promise<PlaceDescription | null> {
// OpenStreetMap first: it costs nothing, it is already fetched, and a
// description someone wrote into the map data beats a generated blurb.
@@ -405,6 +449,16 @@ export class PlaceEnrichmentService {
};
}
// Then the encyclopaedias. This used to sit BELOW Google, which read as a
// detail and was in fact the whole problem: an instance with a key asked
// Google, got nothing back for most places, and stopped — while a perfectly
// good Wikivoyage article sat one call away. Free sources are the normal
// case here and Google is the addition, so the order says so.
const extract = await this.fetchWikiDescription(identity, req.lang);
if (extract) {
return { text: extract.text, source: extract.source, sourceUrl: extract.sourceUrl, license: 'CC BY-SA 4.0' };
}
const apiKey = this.maps.getMapsKey(userId);
if (apiKey && !this.maps.detailsDisabled() && isGooglePlaceId(placeId)) {
const summary = await this.maps.fetchEditorialSummary(placeId, apiKey, req.lang);
@@ -418,16 +472,56 @@ export class PlaceEnrichmentService {
}
}
// Wikipedia last, and only when the place carries a wiki tag. Looking the
// article up by name lands on the wrong one for anything ambiguous, and a
// confident description of somewhere else is worse than none.
const wikipediaTag = typeof details?.wikipedia === 'string' ? details.wikipedia : null;
const extract = await this.maps.fetchWikiExtract(wikipediaTag);
if (extract) {
return { text: extract.text, source: extract.source, sourceUrl: extract.sourceUrl, license: 'CC BY-SA 4.0' };
return null;
}
/**
* The article about this place, in the reader's language where one exists.
*
* Two ways in, and a place usually has only one of them: mappers tag either
* `wikipedia` or `wikidata`, rarely both. The Wikidata route goes through
* sitelinks, which is also what makes the language choice honest — asking
* `<userlang>.wikipedia.org` for a title taken from the German tag returns
* nothing, so without sitelinks a German article is all a Korean reader could
* ever get.
*
* Wikivoyage before Wikipedia at every step: it describes a place for someone
* about to go there, where Wikipedia opens with area in square kilometres.
*/
private async fetchWikiDescription(
identity: PlaceIdentity,
lang: string | undefined,
): Promise<{ text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null> {
const userLang = toWikiLang(lang);
const tag = parseWikipediaTag(identity.wikipedia);
if (identity.wikidata) {
// The tag's own language is in the list because the place named that
// article specifically; it beats falling through to English.
const wanted: { site: string; host: 'wikivoyage' | 'wikipedia'; lang: string }[] = [
{ site: `${userLang}wikivoyage`, host: 'wikivoyage', lang: userLang },
{ site: `${userLang}wiki`, host: 'wikipedia', lang: userLang },
...(tag && tag.lang !== userLang
? [{ site: `${tag.lang}wiki`, host: 'wikipedia' as const, lang: tag.lang }]
: []),
{ site: 'enwikivoyage', host: 'wikivoyage', lang: 'en' },
{ site: 'enwiki', host: 'wikipedia', lang: 'en' },
];
const sitelinks = await this.maps.fetchWikidataSitelinks(
identity.wikidata,
wanted.map((w) => w.site),
);
for (const { site, host, lang: hostLang } of wanted) {
const title = sitelinks[site];
if (!title) continue;
const hit = await this.maps.fetchWikiExtractFor(host, hostLang, title);
if (hit) return hit;
}
}
return null;
// No Wikidata id, or its sitelinks led nowhere: fall back to the tag, which
// names an article directly.
return identity.wikipedia ? this.maps.fetchWikiExtract(identity.wikipedia) : null;
}
/**
@@ -86,6 +86,15 @@ describe('Place enrichment e2e (real auth guard + real validation pipe)', () =>
vi.spyOn(maps, 'fetchWikiExtract').mockClear().mockResolvedValue(null);
vi.spyOn(maps, 'fetchCommonsCategoryCandidates').mockClear().mockResolvedValue([]);
vi.spyOn(maps, 'details').mockClear().mockResolvedValue({ place: null });
// Every provider the service can reach has to be stubbed here, not just the
// ones a given case cares about: anything left open goes out over the real
// network from CI, which is both slow and rude to the provider.
vi.spyOn(maps, 'resolveOsmIdentity').mockClear().mockResolvedValue(null);
vi.spyOn(maps, 'fetchWikidataSitelinks').mockClear().mockResolvedValue({});
vi.spyOn(maps, 'fetchWikiExtractFor').mockClear().mockResolvedValue(null);
vi.spyOn(maps, 'fetchWikidataCandidates').mockClear().mockResolvedValue({ candidates: [], commonsCategory: null });
vi.spyOn(maps, 'fetchWikiLeadImageName').mockClear().mockResolvedValue(null);
vi.spyOn(maps, 'fetchCommonsFilesByName').mockClear().mockResolvedValue(new Map());
});
it('401 without a session cookie', async () => {
@@ -0,0 +1,213 @@
/**
* MAPS-157..170 — resolving a place's identity from a name and a coordinate.
*
* This is what gives a Google place its free half back: Google's payload has no
* wiki tags, OpenStreetMap does. The two gates below are the whole safety
* argument — a confident description of the wrong building is worse than none.
*/
import { describe, it, expect, vi, afterEach } from 'vitest';
vi.mock('../../../src/db/database', () => ({
db: { prepare: () => ({ get: () => undefined, run: () => undefined, all: () => [] }) },
}));
vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) }));
vi.mock('../../../src/utils/ssrfGuard', () => ({
safeFetchFollow: vi.fn(),
checkSsrf: vi.fn(async () => ({ allowed: true })),
SsrfBlockedError: class extends Error {},
}));
import { db } from '../../../src/db/database';
import { DatabaseService } from '../../../src/nest/database/database.service';
import { MapsService } from '../../../src/nest/maps/maps.service';
import { toWikiLang, haversineMetres, namesOverlap } from '../../../src/nest/maps/maps.helpers';
const svcOf = () => new MapsService(new DatabaseService(db as never), {} as never);
// The Brandenburg Gate and the underground station named after it, 250m apart.
const GATE = { lat: 52.5163, lng: 13.3777 };
const hit = (over: Record<string, unknown> = {}) => ({
osm_type: 'way',
osm_id: '518071791',
name: 'Brandenburger Tor',
display_name: 'Brandenburger Tor, Pariser Platz, Berlin',
lat: '52.5163',
lon: '13.3777',
importance: 0.72,
extratags: { wikidata: 'Q82425', wikimedia_commons: 'Category:Brandenburg Gate' },
...over,
});
afterEach(() => {
vi.unstubAllGlobals();
});
describe('toWikiLang', () => {
it('MAPS-157: maps TREK\'s "br" to Portuguese, not Breton', () => {
// br.wikipedia.org EXISTS and answers 200 — it is the Breton Wikipedia.
// Passing the raw code through fails silently, in Breton.
expect(toWikiLang('br')).toBe('pt');
});
it('MAPS-158: drops the region subtag, which no wiki subdomain has', () => {
// pt-br.wikipedia.org does not resolve at all.
expect(toWikiLang('pt-BR')).toBe('pt');
expect(toWikiLang('zh-TW')).toBe('zh');
expect(toWikiLang('en-US')).toBe('en');
});
it('MAPS-159: maps gr to el and falls back to English on nonsense', () => {
expect(toWikiLang('gr')).toBe('el');
expect(toWikiLang('')).toBe('en');
expect(toWikiLang(undefined)).toBe('en');
// Anything that is not shaped like a language code at all.
expect(toWikiLang('deutsch')).toBe('en');
expect(toWikiLang('12')).toBe('en');
});
});
describe('namesOverlap', () => {
it('MAPS-160: matches across languages on a shared substantial word', () => {
expect(namesOverlap('Hamburg Airport', 'Flughafen Hamburg')).toBe(true);
});
it('MAPS-161: rejects two different places in the same area', () => {
expect(namesOverlap('Hamburg Airport', 'Bahnhof Ohlsdorf')).toBe(false);
});
it('MAPS-162: ignores accents', () => {
expect(namesOverlap('Café Zürich', 'Cafe Zurich')).toBe(true);
});
it('MAPS-162b: will not match on a single short word', () => {
// An article, and TREK speaks 23 languages — a stopword list for all of
// them is its own problem, so weight decides instead.
expect(namesOverlap('Der Kiosk', 'Der Bahnhof')).toBe(false);
// Two short words together are evidence enough.
expect(namesOverlap('Bar zum Tor', 'Cafe zum Tor')).toBe(true);
// And one substantial word on its own is.
expect(namesOverlap('Brandenburger Tor', 'Brandenburger Tor (U-Bahn)')).toBe(true);
});
});
describe('haversineMetres', () => {
it('MAPS-163: measures the gate to its underground station at roughly 250m', () => {
const metres = haversineMetres(GATE.lat, GATE.lng, 52.5166047, 13.3809897);
expect(metres).toBeGreaterThan(150);
expect(metres).toBeLessThan(400);
});
it('MAPS-164: is zero for the same point', () => {
expect(haversineMetres(GATE.lat, GATE.lng, GATE.lat, GATE.lng)).toBeCloseTo(0, 5);
});
});
describe('resolveOsmIdentity', () => {
it('MAPS-165: bounds the search to the area around the point', async () => {
const fetchMock = vi.fn().mockResolvedValue({ ok: true, json: async () => [] });
vi.stubGlobal('fetch', fetchMock);
await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng, { lang: 'de' });
const url = decodeURIComponent(String(fetchMock.mock.calls[0][0]));
// Without bounded+viewbox, Nominatim answers with the most famous place on
// earth carrying this name rather than the one being looked at.
expect(url).toContain('bounded=1');
expect(url).toContain('viewbox=');
expect(url).toContain('extratags=1');
});
it('MAPS-166: hands back the tags of the best local match', async () => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [hit()] }));
const out = await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng);
expect(out?.tags.wikidata).toBe('Q82425');
expect(out?.osmUrl).toBe('https://www.openstreetmap.org/way/518071791');
});
it('MAPS-167: prefers the more important match over the merely closer one', async () => {
// The gate and the station share a name and sit 250m apart. Distance alone
// picks whichever the search coordinate happened to land on.
const station = hit({
osm_id: '3862767512',
osm_type: 'node',
lat: '52.5166047',
lon: '13.3809897',
importance: 0.41,
extratags: { wikidata: 'Q477185' },
});
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [station, hit()] }));
const out = await svcOf().resolveOsmIdentity('Brandenburger Tor', 52.5166, 13.3809);
expect(out?.tags.wikidata).toBe('Q82425');
});
it('MAPS-168: refuses a match too far from where we are looking', async () => {
const faraway = hit({ lat: '48.8584', lon: '2.2945' }); // Paris
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [faraway] }));
expect(await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng)).toBeNull();
});
it('MAPS-169: refuses a match that shares no word with the name', async () => {
// Right area, wrong building — this is the gate that stops the column
// describing the neighbour.
const neighbour = hit({ name: 'Hotel Adlon', display_name: 'Hotel Adlon, Berlin' });
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => [neighbour] }));
expect(await svcOf().resolveOsmIdentity('Brandenburger Tor', GATE.lat, GATE.lng)).toBeNull();
});
it('MAPS-170: survives an empty name, a bad response, a throw and a non-array body', async () => {
const fetchMock = vi.fn();
vi.stubGlobal('fetch', fetchMock);
expect(await svcOf().resolveOsmIdentity('', GATE.lat, GATE.lng)).toBeNull();
expect(fetchMock).not.toHaveBeenCalled();
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => [] }));
expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull();
// Nominatim answers rate limiting in plain text, not JSON.
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => 'Bandwidth limit exceeded' }));
expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull();
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
expect(await svcOf().resolveOsmIdentity('X Tor', GATE.lat, GATE.lng)).toBeNull();
});
});
describe('fetchWikidataSitelinks', () => {
it('MAPS-171: asks only for the sites it will use', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ entities: { Q1097: { sitelinks: { dewiki: { title: 'Berlin Hauptbahnhof' } } } } }),
});
vi.stubGlobal('fetch', fetchMock);
const out = await svcOf().fetchWikidataSitelinks('Q1097', ['dewikivoyage', 'dewiki', 'enwiki']);
expect(decodeURIComponent(String(fetchMock.mock.calls[0][0]))).toContain('sitefilter=dewikivoyage|dewiki|enwiki');
expect(out).toEqual({ dewiki: 'Berlin Hauptbahnhof' });
});
it('MAPS-172: refuses a non-Q id and an empty site list without calling out', async () => {
const fetchMock = vi.fn();
vi.stubGlobal('fetch', fetchMock);
expect(await svcOf().fetchWikidataSitelinks('nope', ['enwiki'])).toEqual({});
expect(await svcOf().fetchWikidataSitelinks('Q1', [])).toEqual({});
expect(fetchMock).not.toHaveBeenCalled();
});
it('MAPS-173: yields an empty map on a bad response or a throw', async () => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: false, json: async () => ({}) }));
expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('network')));
expect(await svcOf().fetchWikidataSitelinks('Q1', ['enwiki'])).toEqual({});
});
});
@@ -0,0 +1,270 @@
/**
* ENRICH-070..082 — where a description comes from.
*
* The cases the column got wrong on dev1: L'Osteria Rostock, Hamburg Airport
* and Berlin Brandenburg Airport all showed no description at all with a Google
* key configured, and Berlin Hauptbahnhof showed none without one. Two distinct
* causes, both pinned here — Google short-circuiting the free chain, and the
* chain itself only ever starting from an OSM `wikipedia` tag.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const { mockDbGet, mockDbRun } = vi.hoisted(() => ({
mockDbGet: vi.fn(() => undefined as any),
mockDbRun: vi.fn(),
}));
vi.mock('../../../src/db/database', () => ({
db: {
prepare: (sql: string) => ({
get: (...params: unknown[]) => mockDbGet(sql, ...params),
run: (...params: unknown[]) => mockDbRun(sql, ...params),
all: () => [],
}),
},
}));
vi.mock('../../../src/utils/ssrfGuard', () => ({
safeFetchFollow: vi.fn(async () => ({ ok: false, arrayBuffer: async () => new ArrayBuffer(0) })),
checkSsrf: vi.fn(async () => ({ allowed: true })),
SsrfBlockedError: class extends Error {},
}));
vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) }));
import { db } from '../../../src/db/database';
import { DatabaseService } from '../../../src/nest/database/database.service';
import { PlaceEnrichmentService } from '../../../src/nest/place-enrichment/place-enrichment.service';
import type { MapsService } from '../../../src/nest/maps/maps.service';
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
const GOOGLE_REQ = { lat: 53.6323, lng: 10.0067, name: 'Hamburg Airport', placeId: 'ChIJham', lang: 'de' };
const OSM_REQ = { lat: 52.525, lng: 13.3694, name: 'Berlin Hauptbahnhof', placeId: 'relation:3600565', lang: 'de' };
const EXTRACT = {
text: 'Der Berliner Hauptbahnhof ist der größte Kreuzungsbahnhof Europas.',
sourceUrl: 'https://de.wikipedia.org/wiki/Berlin_Hauptbahnhof',
source: 'wikipedia' as const,
};
function mapsStub(over: Partial<Record<keyof MapsService, unknown>> = {}) {
return {
getMapsKey: vi.fn(() => null as string | null),
photosDisabled: vi.fn(() => false),
detailsDisabled: vi.fn(() => false),
fetchGooglePhotoRefs: vi.fn(async () => []),
fetchGooglePhotoBytes: vi.fn(async () => null),
fetchCommonsCandidates: vi.fn(async () => []),
fetchCommonsCategoryCandidates: vi.fn(async () => []),
fetchWikidataCandidates: vi.fn(async () => ({ candidates: [], commonsCategory: null })),
fetchCommonsFilesByName: vi.fn(async () => new Map()),
fetchWikiLeadImageName: vi.fn(async () => null),
fetchEditorialSummary: vi.fn(async () => null as string | null),
fetchWikiExtract: vi.fn(async () => null as typeof EXTRACT | null),
fetchWikiExtractFor: vi.fn(async () => null as typeof EXTRACT | null),
fetchWikidataSitelinks: vi.fn(async () => ({}) as Record<string, string>),
resolveOsmIdentity: vi.fn(async () => null as { tags: Record<string, string>; osmUrl: string | null; matchedName: string } | null),
details: vi.fn(async () => ({ place: null })),
...over,
} as unknown as MapsService;
}
const cacheStub = () =>
({ get: vi.fn(() => null), put: vi.fn(async () => ({ photoUrl: '/x', filePath: '/x', attribution: null })) }) as unknown as PlacePhotoCacheService;
const make = (maps: MapsService) => new PlaceEnrichmentService(new DatabaseService(db as never), maps, cacheStub());
beforeEach(() => {
mockDbGet.mockReset();
mockDbGet.mockReturnValue(undefined);
mockDbRun.mockReset();
});
describe('identity resolution', () => {
it('ENRICH-070: looks up a Google place in OpenStreetMap, which is where its wiki tags live', async () => {
// Google's payload has no wikidata, no wikipedia and no wikimedia_commons —
// it never had. Before this, configuring an API key silently switched off
// the free half of the column.
const maps = mapsStub({
getMapsKey: vi.fn(() => 'key'),
details: vi.fn(async () => ({ place: { source: 'google', name: 'Hamburg Airport' } })),
resolveOsmIdentity: vi.fn(async () => ({
tags: { wikidata: 'Q27706', wikipedia: 'de:Flughafen Hamburg' },
osmUrl: 'https://www.openstreetmap.org/way/147122729',
matchedName: 'Hamburg Airport',
})),
fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Hamburg-Fuhlsbüttel' })),
fetchWikiExtractFor: vi.fn(async () => ({ ...EXTRACT, source: 'wikivoyage' as const })),
});
const out = await make(maps).enrich(1, GOOGLE_REQ);
expect(maps.resolveOsmIdentity).toHaveBeenCalledWith('Hamburg Airport', 53.6323, 10.0067, { lang: 'de' });
expect(out.description).toMatchObject({ source: 'wikivoyage', license: 'CC BY-SA 4.0' });
});
it('ENRICH-071: does not spend a lookup when the payload already carries tags', async () => {
const maps = mapsStub({
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })),
fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof' })),
fetchWikiExtractFor: vi.fn(async () => EXTRACT),
});
await make(maps).enrich(1, OSM_REQ);
expect(maps.resolveOsmIdentity).not.toHaveBeenCalled();
});
it('ENRICH-072: keeps OSM tags out of a Google payload', async () => {
// collectFacts and the OSM branch of collectDescription both gate on
// details.source. Merging the looked-up tags in would either be dead weight
// or would make a Google place claim to be an OpenStreetMap one.
const details = { source: 'google', name: 'Hamburg Airport', summary: null };
const maps = mapsStub({
getMapsKey: vi.fn(() => 'key'),
details: vi.fn(async () => ({ place: details })),
resolveOsmIdentity: vi.fn(async () => ({
tags: { wikidata: 'Q27706', cuisine: 'german' },
osmUrl: null,
matchedName: 'Hamburg Airport',
})),
});
const out = await make(maps).enrich(1, GOOGLE_REQ);
expect(details).toEqual({ source: 'google', name: 'Hamburg Airport', summary: null });
// Cuisine is an OSM fact and this is a Google place, so it stays out.
expect(out.facts.some((f) => f.kind === 'cuisine')).toBe(false);
});
});
describe('description source order', () => {
it('ENRICH-073: prefers a free article over Google, even with a key configured', async () => {
// This is the whole point of the reordering: Google used to be asked first
// and answered for almost nothing, so the column stayed empty while a
// perfectly good article sat one call away.
const maps = mapsStub({
getMapsKey: vi.fn(() => 'key'),
details: vi.fn(async () => ({ place: { source: 'google', wikidata: 'Q1097' } })),
fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof' })),
fetchWikiExtractFor: vi.fn(async () => EXTRACT),
fetchEditorialSummary: vi.fn(async () => 'A large railway station.'),
});
const out = await make(maps).enrich(1, { ...OSM_REQ, placeId: 'ChIJhbf' });
expect(out.description).toMatchObject({ source: 'wikipedia' });
// Not merely lower priority — the billed call is never made.
expect(maps.fetchEditorialSummary).not.toHaveBeenCalled();
});
it('ENRICH-074: still falls back to Google when nothing free describes the place', async () => {
const maps = mapsStub({
getMapsKey: vi.fn(() => 'key'),
details: vi.fn(async () => ({ place: { source: 'google', google_maps_url: 'https://maps.google/x' } })),
fetchEditorialSummary: vi.fn(async () => 'Casual chain for wood-fired pizza.'),
});
const out = await make(maps).enrich(1, { ...GOOGLE_REQ, name: "L'Osteria Rostock" });
expect(out.description).toMatchObject({
source: 'google',
text: 'Casual chain for wood-fired pizza.',
sourceUrl: 'https://maps.google/x',
license: null,
});
});
it('ENRICH-075: keeps the OpenStreetMap description ahead of everything', async () => {
const maps = mapsStub({
details: vi.fn(async () => ({
place: { source: 'openstreetmap', summary: ' Ein Bahnhof. ', osm_url: 'https://osm.org/r/1', wikidata: 'Q1097' },
})),
fetchWikiExtractFor: vi.fn(async () => EXTRACT),
});
const out = await make(maps).enrich(1, OSM_REQ);
expect(out.description).toMatchObject({ text: 'Ein Bahnhof.', source: 'osm', license: 'ODbL 1.0' });
expect(maps.fetchWikidataSitelinks).not.toHaveBeenCalled();
});
});
describe('article language choice', () => {
it('ENRICH-076: asks for the reader\'s language before English', async () => {
const maps = mapsStub({
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })),
fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Berlin Hauptbahnhof', enwiki: 'Berlin Hauptbahnhof' })),
fetchWikiExtractFor: vi.fn(async () => EXTRACT),
});
await make(maps).enrich(1, OSM_REQ);
const sites = (maps.fetchWikidataSitelinks as unknown as { mock: { calls: [string, string[]][] } }).mock.calls[0][1];
expect(sites.slice(0, 2)).toEqual(['dewikivoyage', 'dewiki']);
expect(sites).toContain('enwiki');
expect(maps.fetchWikiExtractFor).toHaveBeenCalledWith('wikipedia', 'de', 'Berlin Hauptbahnhof');
});
it('ENRICH-077: puts Wikivoyage ahead of Wikipedia in the same language', async () => {
// Wikivoyage describes a place for somebody about to go there; Wikipedia
// opens with area in square kilometres.
const maps = mapsStub({
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })),
fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Berlin', dewiki: 'Berlin Hauptbahnhof' })),
fetchWikiExtractFor: vi.fn(async () => ({ ...EXTRACT, source: 'wikivoyage' as const })),
});
await make(maps).enrich(1, OSM_REQ);
expect(maps.fetchWikiExtractFor).toHaveBeenNthCalledWith(1, 'wikivoyage', 'de', 'Berlin');
});
it('ENRICH-078: keeps the language the place itself named, ahead of English', async () => {
// A Korean reader looking at the Brandenburg Gate: no kowiki sitelink, but
// the OSM tag names a German article somebody curated for this place.
const maps = mapsStub({
details: vi.fn(async () => ({
place: { source: 'openstreetmap', wikidata: 'Q82425', wikipedia: 'de:Brandenburger Tor' },
})),
fetchWikidataSitelinks: vi.fn(async () => ({ dewiki: 'Brandenburger Tor', enwiki: 'Brandenburg Gate' })),
fetchWikiExtractFor: vi.fn(async () => EXTRACT),
});
await make(maps).enrich(1, { ...OSM_REQ, lang: 'ko' });
const sites = (maps.fetchWikidataSitelinks as unknown as { mock: { calls: [string, string[]][] } }).mock.calls[0][1];
expect(sites.indexOf('dewiki')).toBeLessThan(sites.indexOf('enwiki'));
});
it('ENRICH-079: falls back to the tag when the item has no usable sitelink', async () => {
const maps = mapsStub({
details: vi.fn(async () => ({
place: { source: 'openstreetmap', wikidata: 'Q1', wikipedia: 'de:Museum Ludwig' },
})),
fetchWikidataSitelinks: vi.fn(async () => ({})),
fetchWikiExtract: vi.fn(async () => EXTRACT),
});
const out = await make(maps).enrich(1, OSM_REQ);
expect(maps.fetchWikiExtract).toHaveBeenCalledWith('de:Museum Ludwig');
expect(out.description).toMatchObject({ source: 'wikipedia' });
});
it('ENRICH-080: tries the next site when one has no article', async () => {
const maps = mapsStub({
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikidata: 'Q1097' } })),
fetchWikidataSitelinks: vi.fn(async () => ({ dewikivoyage: 'Nowhere', dewiki: 'Berlin Hauptbahnhof' })),
fetchWikiExtractFor: vi
.fn()
.mockResolvedValueOnce(null)
.mockResolvedValueOnce(EXTRACT),
});
const out = await make(maps).enrich(1, OSM_REQ);
expect(maps.fetchWikiExtractFor).toHaveBeenCalledTimes(2);
expect(out.description).toMatchObject({ source: 'wikipedia' });
});
});
@@ -58,6 +58,9 @@ function mapsStub(over: Partial<Record<keyof MapsService, unknown>> = {}) {
fetchWikidataCandidates: vi.fn(async () => ({ candidates: [] as any[], commonsCategory: null as string | null })),
fetchCommonsFilesByName: vi.fn(async () => new Map<string, any>()),
fetchWikiLeadImageName: vi.fn(async () => null as string | null),
resolveOsmIdentity: vi.fn(async () => null as { tags: Record<string, string>; osmUrl: string | null; matchedName: string } | null),
fetchWikidataSitelinks: vi.fn(async () => ({}) as Record<string, string>),
fetchWikiExtractFor: vi.fn(async () => null as { text: string; sourceUrl: string; source: 'wikivoyage' | 'wikipedia' } | null),
details: vi.fn(async () => ({ place: null })),
...over,
} as unknown as MapsService;
@@ -443,7 +446,10 @@ describe('collectDescription', () => {
await make(maps, cacheStub()).enrich(1, OSM_REQ);
expect(maps.fetchWikiExtract).toHaveBeenCalledWith(null);
// Not a usable tag, so it counts as no identity at all: the lookup that
// finds one is tried instead, and the wiki itself is never asked.
expect(maps.resolveOsmIdentity).toHaveBeenCalled();
expect(maps.fetchWikiExtract).not.toHaveBeenCalled();
});
it('ENRICH-017: honours the details kill switch and the missing key', async () => {
@@ -477,12 +483,16 @@ describe('collectDescription', () => {
});
});
it('ENRICH-019: passes null on when the place carries no wiki tag', async () => {
it('ENRICH-019: looks the place up when it carries no wiki tag, and gives up quietly when that finds nothing', async () => {
// The kiosk case. A place with no tag and no OSM match genuinely has no
// description, and inventing one from a name search is how you confidently
// describe a different building.
const maps = mapsStub({ details: vi.fn(async () => ({ place: { name: 'Kiosk' } })) });
const out = await make(maps, cacheStub()).enrich(1, OSM_REQ);
expect(maps.fetchWikiExtract).toHaveBeenCalledWith(null);
expect(maps.resolveOsmIdentity).toHaveBeenCalledWith('Museum Ludwig', 50.9, 6.96, { lang: undefined });
expect(maps.fetchWikiExtract).not.toHaveBeenCalled();
expect(out.description).toBeNull();
});