mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-19 05:26:47 +00:00
feat(places): descriptions from the place's own website
Overture has no descriptions, and the encyclopaedias only describe the places that have an article, which is a fraction of a percent. What Overture does have is the address of the page the description is on, for 43 percent of places, and those pages publish a summary in JSON-LD or og:description precisely so machines can read it. The API fetches each page once and keeps the summary; every instance reads it from there. TREK never opens a business's website itself, which would mean one request per instance per place against small servers. In the ladder it sits below the encyclopaedias and above Google: an article about this exact place beats the operator's own copy, but for a restaurant or a shop the operator's summary is the only one that exists. Credited by host and linked back, with no licence claimed.
This commit is contained in:
@@ -169,6 +169,28 @@ describe('PlaceDetailsColumn', () => {
|
||||
expect(link.parentElement).toHaveTextContent('CC BY-SA 4.0')
|
||||
})
|
||||
|
||||
it('FE-PDC-010b: credits a website description by host', async () => {
|
||||
// The source labels used to be a ternary chain ending in 'Wikipedia', so a
|
||||
// restaurant's own marketing copy would have been credited to an
|
||||
// encyclopaedia. There is no fixed name for "the place's own website", and
|
||||
// the host is the honest one.
|
||||
placeEnrichment.mockResolvedValue({
|
||||
photos: [],
|
||||
facts: [],
|
||||
description: {
|
||||
text: 'Pizza in Rostock, seit 2015.',
|
||||
source: 'website',
|
||||
sourceUrl: 'https://www.losteria.net/rostock',
|
||||
license: null,
|
||||
},
|
||||
})
|
||||
renderColumn()
|
||||
|
||||
const link = await screen.findByRole('link', { name: /losteria\.net/ })
|
||||
expect(link).toHaveAttribute('href', 'https://www.losteria.net/rostock')
|
||||
expect(screen.queryByText(/Wikipedia/)).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-PDC-011: says so when the admin switched enrichment off', async () => {
|
||||
placeEnrichment.mockResolvedValue({ photos: [], description: null, facts: [], disabled: true })
|
||||
renderColumn()
|
||||
|
||||
@@ -379,6 +379,37 @@ function PhotoTile({
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Who the description came from, as the reader would name them.
|
||||
*
|
||||
* `website` has no fixed name — it is the place's own site — so it is credited
|
||||
* by host: "losteria.net" says where the text is from far better than a generic
|
||||
* word would. Deliberately not a ternary chain: the previous one fell through
|
||||
* to 'Wikipedia', which would have credited a restaurant's marketing copy to an
|
||||
* encyclopaedia.
|
||||
*/
|
||||
function descriptionSourceLabel(
|
||||
source: NonNullable<MapsPlaceEnrichmentResult['description']>['source'],
|
||||
sourceUrl: string | null,
|
||||
): string {
|
||||
switch (source) {
|
||||
case 'google':
|
||||
return 'Google'
|
||||
case 'osm':
|
||||
return 'OpenStreetMap'
|
||||
case 'wikivoyage':
|
||||
return 'Wikivoyage'
|
||||
case 'wikipedia':
|
||||
return 'Wikipedia'
|
||||
case 'website':
|
||||
try {
|
||||
return new URL(sourceUrl || '').hostname.replace(/^www\./, '')
|
||||
} catch {
|
||||
return 'Website'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sourceLabelFor(source: PlacePhotoCandidate['source']): string {
|
||||
if (source === 'google') return 'Google'
|
||||
if (source === 'wikipedia') return 'Wikipedia'
|
||||
@@ -659,14 +690,7 @@ function DescriptionBlock({
|
||||
}): React.ReactElement | null {
|
||||
if (!description) return null
|
||||
|
||||
const sourceLabel =
|
||||
description.source === 'google'
|
||||
? 'Google'
|
||||
: description.source === 'osm'
|
||||
? 'OpenStreetMap'
|
||||
: description.source === 'wikivoyage'
|
||||
? 'Wikivoyage'
|
||||
: 'Wikipedia'
|
||||
const sourceLabel = descriptionSourceLabel(description.source, description.sourceUrl)
|
||||
|
||||
// A description of the chain is not a description of this place, and the
|
||||
// heading is where that gets said. Without it the reader has no way to tell
|
||||
|
||||
@@ -116,8 +116,8 @@ export class BookingImportService {
|
||||
let found: { lat: number; lng: number } | null = null;
|
||||
try {
|
||||
for (const q of queries) {
|
||||
const hit = (await this.maps.searchNominatim(q, undefined, 'background'))[0];
|
||||
if (hit?.lat != null && hit?.lng != null) { found = { lat: hit.lat, lng: hit.lng }; break; }
|
||||
const hit = await this.maps.geocodeQuery(q);
|
||||
if (hit) { found = hit; break; }
|
||||
}
|
||||
} catch {
|
||||
// geocoding failure is non-fatal — the endpoint stays, and is warned about
|
||||
@@ -240,9 +240,8 @@ export class BookingImportService {
|
||||
].filter((q): q is string => !!q);
|
||||
|
||||
for (const q of queries) {
|
||||
const results = await this.maps.searchNominatim(q, undefined, 'background');
|
||||
const hit = results[0];
|
||||
if (hit?.lat != null && hit?.lng != null) {
|
||||
const hit = await this.maps.geocodeQuery(q);
|
||||
if (hit) {
|
||||
lat = hit.lat;
|
||||
lng = hit.lng;
|
||||
break;
|
||||
|
||||
@@ -695,6 +695,35 @@ export class MapsService {
|
||||
return this.resolveMapsKey(userId).key;
|
||||
}
|
||||
|
||||
/**
|
||||
* A coordinate for a name that came out of an import, from whichever source
|
||||
* has it.
|
||||
*
|
||||
* Booking imports geocode every venue and every uncoordinated endpoint in one
|
||||
* request loop, up to thirty sequential lookups. Every one of those used to be
|
||||
* a Nominatim call on the background lane, waiting out its throttle. The index
|
||||
* answers most of them without leaving our own infrastructure and without a
|
||||
* throttle at all; a street address it does not know still falls through to
|
||||
* Nominatim, which is the source that resolves addresses.
|
||||
*
|
||||
* Never throws: an import that cannot place a hotel still imports the hotel.
|
||||
*/
|
||||
async geocodeQuery(query: string): Promise<{ lat: number; lng: number } | null> {
|
||||
if (this.trekPlacesEnabled()) {
|
||||
try {
|
||||
const found = await trekPlacesSearch(query, { limit: 1 });
|
||||
const hit = found[0];
|
||||
if (hit && Number.isFinite(hit.lat) && Number.isFinite(hit.lng)) {
|
||||
return { lat: hit.lat, lng: hit.lng };
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
console.warn('TREK Places geocode failed, falling back:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
const hit = (await this.searchNominatim(query, undefined, 'background'))[0];
|
||||
return hit?.lat != null && hit?.lng != null ? { lat: hit.lat, lng: hit.lng } : null;
|
||||
}
|
||||
|
||||
// ── Nominatim search ───────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
type WikiIdentity,
|
||||
} from '../maps/maps.service';
|
||||
import { buildOsmDetails, isGooglePlaceId, parseWikipediaTag, rankCommonsCandidates, toWikiLang } from '../maps/maps.helpers';
|
||||
import { trekPlacesById } from '../maps/trek-places.client';
|
||||
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
|
||||
|
||||
/**
|
||||
@@ -570,6 +571,38 @@ export class PlaceEnrichmentService {
|
||||
|
||||
// ── Description ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The description the place publishes on its own site, served from the TREK
|
||||
* Places API rather than fetched here.
|
||||
*
|
||||
* TREK never opens the business's website itself: that would be one request
|
||||
* per instance per place, against small servers, from hundreds of homelabs.
|
||||
* The API fetches each page once, keeps the summary, and every instance reads
|
||||
* it from there. Only places carrying a GERS id can be looked up, which is
|
||||
* exactly the ones that came from the API in the first place.
|
||||
*/
|
||||
private async websiteDescription(placeId: string): Promise<PlaceDescription | null> {
|
||||
if (!placeId.startsWith('gers:')) return null;
|
||||
try {
|
||||
const place = await trekPlacesById(placeId.slice(5));
|
||||
const got = (place as { description?: { text?: string; sourceUrl?: string } } | null)?.description;
|
||||
const text = typeof got?.text === 'string' ? got.text.trim() : '';
|
||||
if (!text) return null;
|
||||
return {
|
||||
text,
|
||||
source: 'website',
|
||||
sourceUrl: typeof got?.sourceUrl === 'string' ? got.sourceUrl : null,
|
||||
// Not a licensed corpus: a quoted summary from the operator's own page,
|
||||
// credited and linked back. Saying "CC-something" here would be a claim
|
||||
// about terms nobody granted.
|
||||
license: null,
|
||||
};
|
||||
} catch {
|
||||
// The API being down must never cost the place its other sources.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async collectDescription(
|
||||
userId: number,
|
||||
placeId: string,
|
||||
@@ -599,6 +632,19 @@ export class PlaceEnrichmentService {
|
||||
return { text: extract.text, source: extract.source, sourceUrl: extract.sourceUrl, license: 'CC BY-SA 4.0' };
|
||||
}
|
||||
|
||||
// Then the place's own website, through the TREK Places API.
|
||||
//
|
||||
// Placed here and not higher on purpose: an encyclopaedia article reached
|
||||
// through the OSM `wikipedia` tag is about this exact place and was written
|
||||
// by someone with no stake in it, which beats marketing copy. But for the
|
||||
// ordinary case — a restaurant, a shop, a hotel, none of which will ever
|
||||
// have an article — the operator's own summary is the only description that
|
||||
// exists, and it is published in JSON-LD or og:description precisely so
|
||||
// machines can read it. That covers roughly 43 percent of places, where the
|
||||
// encyclopaedias cover a fraction of a percent.
|
||||
const fromSite = await this.websiteDescription(placeId);
|
||||
if (fromSite) return fromSite;
|
||||
|
||||
const apiKey = this.maps.getMapsKey(userId);
|
||||
if (apiKey && !this.maps.detailsDisabled() && isGooglePlaceId(placeId)) {
|
||||
const summary = await this.maps.fetchEditorialSummary(placeId, apiKey, req.lang);
|
||||
|
||||
@@ -35,6 +35,13 @@ vi.mock('../../../src/utils/ssrfGuard', () => ({
|
||||
|
||||
vi.mock('../../../src/config', () => ({ JWT_SECRET: 'test-secret', ENCRYPTION_KEY: '0'.repeat(64) }));
|
||||
|
||||
const { mockTrekPlacesById } = vi.hoisted(() => ({
|
||||
mockTrekPlacesById: vi.fn(async (_gers: string): Promise<unknown> => null),
|
||||
}));
|
||||
vi.mock('../../../src/nest/maps/trek-places.client', () => ({
|
||||
trekPlacesById: mockTrekPlacesById,
|
||||
}));
|
||||
|
||||
import { db } from '../../../src/db/database';
|
||||
import { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
import { PlaceEnrichmentService } from '../../../src/nest/place-enrichment/place-enrichment.service';
|
||||
@@ -81,6 +88,8 @@ beforeEach(() => {
|
||||
mockDbGet.mockReset();
|
||||
mockDbGet.mockReturnValue(undefined);
|
||||
mockDbRun.mockReset();
|
||||
mockTrekPlacesById.mockReset();
|
||||
mockTrekPlacesById.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
describe('identity resolution', () => {
|
||||
@@ -339,3 +348,70 @@ describe('filling Google gaps from the free sources', () => {
|
||||
});
|
||||
})
|
||||
|
||||
|
||||
describe("the description on the place's own website", () => {
|
||||
// The ordinary case, and the one nothing covered before: a restaurant has no
|
||||
// encyclopaedia article and never will, but it does publish a summary of
|
||||
// itself in JSON-LD for machines to read. 43 percent of Overture places carry
|
||||
// a website; a fraction of a percent carry a wiki tag.
|
||||
const GERS_REQ = {
|
||||
lat: 54.0879,
|
||||
lng: 12.1408,
|
||||
name: "L'Osteria",
|
||||
placeId: 'gers:abc-123',
|
||||
lang: 'de',
|
||||
};
|
||||
|
||||
it('ENRICH-090: quotes the site and credits it by URL', async () => {
|
||||
mockTrekPlacesById.mockResolvedValue({
|
||||
description: { text: 'Pizza in Rostock, seit 2015.', sourceUrl: 'https://losteria.net/rostock' },
|
||||
});
|
||||
|
||||
const out = await make(mapsStub()).enrich(1, GERS_REQ);
|
||||
|
||||
expect(mockTrekPlacesById).toHaveBeenCalledWith('abc-123');
|
||||
expect(out.description).toEqual({
|
||||
text: 'Pizza in Rostock, seit 2015.',
|
||||
source: 'website',
|
||||
sourceUrl: 'https://losteria.net/rostock',
|
||||
// No licence claim: the operator published a summary for machines, they
|
||||
// did not grant terms. Naming and linking the source is what is owed.
|
||||
license: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('ENRICH-091: leaves the encyclopaedias ahead of it', async () => {
|
||||
// An article about this exact place, written by someone with no stake in
|
||||
// it, beats the operator's own copy. The website only fills the gap the
|
||||
// encyclopaedias leave, which is almost every business.
|
||||
mockTrekPlacesById.mockResolvedValue({ description: { text: 'Unser Restaurant.', sourceUrl: 'https://x.de' } });
|
||||
const maps = mapsStub({
|
||||
details: vi.fn(async () => ({ place: { source: 'openstreetmap', wikipedia: 'de:Irgendwas' } })),
|
||||
fetchWikiExtract: vi.fn(async () => EXTRACT),
|
||||
});
|
||||
|
||||
const out = await make(maps).enrich(1, GERS_REQ);
|
||||
|
||||
expect(out.description?.source).toBe('wikipedia');
|
||||
});
|
||||
|
||||
it('ENRICH-092: asks only for places that came from the API', async () => {
|
||||
await make(mapsStub()).enrich(1, { ...GERS_REQ, placeId: 'ChIJsomethinggoogle' });
|
||||
expect(mockTrekPlacesById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ENRICH-093: a failing API costs the place nothing else', async () => {
|
||||
mockTrekPlacesById.mockRejectedValue(new Error('places api down'));
|
||||
const maps = mapsStub({
|
||||
details: vi.fn(async () => ({ place: { source: 'openstreetmap', name: "L'Osteria" } })),
|
||||
});
|
||||
|
||||
await expect(make(maps).enrich(1, GERS_REQ)).resolves.toMatchObject({ description: null });
|
||||
});
|
||||
|
||||
it('ENRICH-094: ignores an empty or whitespace-only summary', async () => {
|
||||
mockTrekPlacesById.mockResolvedValue({ description: { text: ' ', sourceUrl: 'https://x.de' } });
|
||||
const out = await make(mapsStub()).enrich(1, GERS_REQ);
|
||||
expect(out.description).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -131,7 +131,14 @@ export const placePhotoCandidateSchema = z.object({
|
||||
});
|
||||
export type PlacePhotoCandidate = z.infer<typeof placePhotoCandidateSchema>;
|
||||
|
||||
export const placeDescriptionSourceSchema = z.enum(['google', 'osm', 'wikivoyage', 'wikipedia']);
|
||||
export const placeDescriptionSourceSchema = z.enum([
|
||||
'google',
|
||||
'osm',
|
||||
'wikivoyage',
|
||||
'wikipedia',
|
||||
/** Quoted from the place's own site, via the TREK Places API. */
|
||||
'website',
|
||||
]);
|
||||
export type PlaceDescriptionSource = z.infer<typeof placeDescriptionSourceSchema>;
|
||||
|
||||
export const placeDescriptionSchema = z.object({
|
||||
|
||||
Reference in New Issue
Block a user