diff --git a/server/src/nest/integrations/dawarich-sync.service.ts b/server/src/nest/integrations/dawarich-sync.service.ts index 3dacbbf1f..216f6fbb4 100644 --- a/server/src/nest/integrations/dawarich-sync.service.ts +++ b/server/src/nest/integrations/dawarich-sync.service.ts @@ -13,7 +13,17 @@ import { logError, logInfo } from '../audit/audit-log.logger'; import { UnitOfWork } from '../database/unit-of-work'; import { JourneyRecordingPrepService, type PreparedJourney } from '../journey/journey-recording-prep.service'; import { DawarichClient, DawarichError, type DawarichCreds, type DawarichVisitRaw } from './dawarich.client'; -import { distanceMeters, localDateOf, normalizeVisit, syncWindow, visitHash } from './dawarich.helpers'; +import { + distanceMeters, + isPlaceholderName, + localDateOf, + nameFromGeocodedPoints, + normalizeVisit, + STAY_NAME_PENDING, + syncWindow, + UNNAMED_STAY, + visitHash, +} from './dawarich.helpers'; import { DawarichService } from './dawarich.service'; import { InjectRepository } from '@mikro-orm/nestjs'; import { Injectable } from '@nestjs/common'; @@ -27,6 +37,19 @@ import { const DAY_MS = 24 * 60 * 60 * 1000; +/** + * Stays one run reads points for to name them. A first sync of a long trip + * can hold hundreds without a name; the rest wait for the next run rather than + * holding this one open (see `stayNamer`). + */ +const MAX_STAY_NAME_LOOKUPS = 100; + +/** + * Names a stay from its points: the name, null when none of them was + * geocoded, undefined when they were not read this run. + */ +type StayNamer = (startedAt: string, endedAt: string) => Promise; + /** What one user's sync produced — plus whether it ran at all. */ export interface DawarichSyncOutcome { state: DawarichSyncState; @@ -281,11 +304,12 @@ export class DawarichSyncService { ): Promise<{ created: number; updated: number; missing: number }> { const { visits } = await this.client.listVisits(creds, from, to); + const nameStay = this.stayNamer(creds); const seenIds = new Set(); let created = 0; let updated = 0; for (const raw of visits) { - const outcome = await this.upsertVisit(userId, tripId, raw, trips); + const outcome = await this.upsertVisit(userId, tripId, raw, trips, nameStay); if (!outcome) continue; seenIds.add(outcome.sourceVisitId); if (outcome.change === 'created') created++; @@ -296,6 +320,48 @@ export class DawarichSyncService { return { created, updated, missing }; } + /** + * Reads a stay's points for a name, for one run. A run reads at most + * MAX_STAY_NAME_LOOKUPS stays, and stops reading after the first failure, + * since an instance that did not answer one will not answer the rest either. + * Either way the stay is left for the next run, not given up on. + */ + private stayNamer(creds: DawarichCreds): StayNamer { + let left = MAX_STAY_NAME_LOOKUPS; + return async (startedAt, endedAt) => { + if (left <= 0) return undefined; + left--; + try { + const points = await this.client.listGeocodedPoints(creds, new Date(startedAt), new Date(endedAt)); + return nameFromGeocodedPoints(points); + } catch { + left = 0; + return undefined; + } + }; + } + + /** + * What a stay is called. Its own name when Dawarich sent one. Otherwise the + * reverse-geocoded points recorded during it name it (#3105): Dawarich sends + * a placeholder for a visit without a place even when its points know the + * town. A name found earlier is kept while the visit is unchanged, and so is + * the finding that none of its points was geocoded; a stay whose points + * were not read yet keeps the placeholder, which the next run reads again. + */ + private async stayName( + visit: { name: string; named: boolean; startedAt: string; endedAt: string }, + hash: string, + existing: { name: string | null; source_hash: string | null } | null, + nameStay: StayNamer, + ): Promise { + if (visit.named) return visit.name; + if (existing?.name && existing.source_hash === hash && !isPlaceholderName(existing.name)) return existing.name; + const found = await nameStay(visit.startedAt, visit.endedAt); + if (found === undefined) return STAY_NAME_PENDING; + return found ?? UNNAMED_STAY; + } + /** * One visit into the user's suggestions: inserted when new, overwritten while * nobody acted on it, only its hash refreshed once accepted or dismissed. @@ -309,6 +375,7 @@ export class DawarichSyncService { tripId: number | null, raw: DawarichVisitRaw, trips: TripRow[], + nameStay: StayNamer, ): Promise<{ sourceVisitId: string; change: 'created' | 'updated' | 'unchanged' } | null> { const visit = normalizeVisit(raw); if (!visit) return null; @@ -331,7 +398,7 @@ export class DawarichSyncService { user_id: userId, source_visit_id: visit.sourceVisitId, trip_id: ownerTripId, - name: visit.name, + name: await this.stayName(visit, hash, null, nameStay), lat: visit.lat, lng: visit.lng, started_at: visit.startedAt, @@ -358,7 +425,7 @@ export class DawarichSyncService { // that includes the trip: a stay parked on a neighbour by an earlier // run moves to the trip whose dates hold it. await this.suggestions.updateNewVisit(existing.id, { - name: visit.name, + name: await this.stayName(visit, hash, existing, nameStay), lat: visit.lat, lng: visit.lng, started_at: visit.startedAt, @@ -424,10 +491,11 @@ export class DawarichSyncService { trips: TripRow[], ): Promise<{ created: number; updated: number; truncated: boolean }> { const { visits, truncated } = await this.client.listVisits(creds, from, to); + const nameStay = this.stayNamer(creds); let created = 0; let updated = 0; for (const raw of visits) { - const outcome = await this.upsertVisit(userId, null, raw, trips); + const outcome = await this.upsertVisit(userId, null, raw, trips, nameStay); if (outcome?.change === 'created') created++; else if (outcome?.change === 'updated') updated++; } diff --git a/server/src/nest/integrations/dawarich.client.ts b/server/src/nest/integrations/dawarich.client.ts index c858a1e08..f1c6c58da 100644 --- a/server/src/nest/integrations/dawarich.client.ts +++ b/server/src/nest/integrations/dawarich.client.ts @@ -47,6 +47,8 @@ export const VISITS_PAGE_SIZE = 500; export const TRACKS_PAGE_SIZE = 200; /** Points cap at 10000 upstream. A day of tracking is a few thousand. */ export const POINTS_PAGE_SIZE = 5000; +/** Points read to name one stay. Enough to get past a few not yet geocoded. */ +export const GEOCODED_POINTS_PAGE_SIZE = 50; /** * How many pages one call will walk before giving up and saying so. @@ -104,11 +106,26 @@ export interface DawarichVisitRaw { latitude: number | string | null; longitude: number | string | null; id: number | null; + /** Sent since 1.22, alongside the visit's own `name`. */ + name?: string | null; + /** Street and locality from the place's geodata, since 1.22. */ + address?: string | null; /** Requested upstream, absent at 1.14.4. Read defensively so it lights up on its own. */ country_code?: string | null; } | null; /** Requested upstream, absent at 1.14.4. */ updated_at?: string | null; + /** + * The weighted middle of the visit's own points, since 1.22. The only + * coordinate a visit without a place or an area carries. + */ + center?: { latitude: number | string | null; longitude: number | string | null } | null; +} + +/** The reverse-geocoded part of one entry of `GET /api/v1/points` (not slim). */ +export interface DawarichGeocodedPoint { + city?: string | null; + country_name?: string | null; } /** One GeoJSON feature from `GET /api/v1/tracks`. */ @@ -382,6 +399,23 @@ export class DawarichClient { return { points, truncated }; } + /** + * The first recorded points of a stay with what reverse geocoding made of + * them, to name a visit Dawarich sends without one (#3105). One small page: + * a stay's opening points name it as well as all of them would, and the full + * serializer carries geodata, so a whole day of it is no cheap answer. + */ + async listGeocodedPoints(creds: DawarichCreds, from: Date, to: Date): Promise { + const { data } = await this.get(creds, '/points', { + start_at: unixSeconds(from), + end_at: unixSeconds(to), + order: 'asc', + page: '1', + per_page: String(GEOCODED_POINTS_PAGE_SIZE), + }); + return Array.isArray(data) ? data : []; + } + /** * Countries and cities the recordings put the user in, for the Atlas. * diff --git a/server/src/nest/integrations/dawarich.helpers.ts b/server/src/nest/integrations/dawarich.helpers.ts index 36357ca2f..2c1dde500 100644 --- a/server/src/nest/integrations/dawarich.helpers.ts +++ b/server/src/nest/integrations/dawarich.helpers.ts @@ -1,5 +1,6 @@ import { DomainError } from '../common/domain-error'; import type { + DawarichGeocodedPoint, DawarichSlimPoint, DawarichTrackFeature, DawarichVisitRaw, @@ -134,6 +135,72 @@ export function toNumber(value: unknown): number | null { return null; } +/** What a stay is called when nothing names it. */ +export const UNNAMED_STAY = 'Unnamed stay'; + +/** + * The names Dawarich gives a visit or a place when it knows nothing about + * either (`Visit::DEFAULT_NAME`, `Place::DEFAULT_NAME`). Since 1.22 the + * visit's `name` is its display name, so one of these arrives in place of + * nothing, and taken as a name it hid the address the recording does hold + * (#3105). + */ +const DAWARICH_PLACEHOLDER_NAMES = new Set(['unknown location', 'suggested place']); + +/** A name worth showing: set, and not one of Dawarich's placeholders. */ +function realName(value: unknown): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + if (!trimmed || DAWARICH_PLACEHOLDER_NAMES.has(trimmed.toLowerCase())) return null; + return trimmed; +} + +/** + * The best name the visit itself carries: its own, its place's, then the + * place's address. Null when there is none of them. + */ +export function visitSourceName(visit: DawarichVisitRaw): string | null { + return realName(visit.name) ?? realName(visit.place?.name) ?? realName(visit.place?.address); +} + +/** + * What a stay is stored as while its points are still to be read for a name: + * Dawarich's own placeholder, which is what the source said, and which the + * next sync recognises and tries again. + */ +export const STAY_NAME_PENDING = 'Unknown Location'; + +/** True for a stored name that is one of Dawarich's placeholders, or none at all. */ +export function isPlaceholderName(name: string | null | undefined): boolean { + return realName(name) === null; +} + +/** + * A stay named from the reverse-geocoded points recorded during it: the city + * most of them fall in with its country ("Cinisi, Italy"), or the country + * alone when no point has a city. Null when none was geocoded. + */ +export function nameFromGeocodedPoints(points: readonly DawarichGeocodedPoint[]): string | null { + const cities = new Map(); + const countries = new Map(); + for (const point of points) { + const city = realName(point.city); + const country = realName(point.country_name); + if (city) { + const seen = cities.get(city); + if (seen) seen.count++; + else cities.set(city, { count: 1, country }); + } + if (country) countries.set(country, (countries.get(country) ?? 0) + 1); + } + let city: [string, { count: number; country: string | null }] | null = null; + for (const entry of cities) if (!city || entry[1].count > city[1].count) city = entry; + if (city) return city[1].country ? `${city[0]}, ${city[1].country}` : city[0]; + let country: [string, number] | null = null; + for (const entry of countries) if (!country || entry[1] > country[1]) country = entry; + return country ? country[0] : null; +} + /** * A visit reduced to what TREK stores. * @@ -144,6 +211,8 @@ export function toNumber(value: unknown): number | null { export function normalizeVisit(visit: DawarichVisitRaw): { sourceVisitId: string; name: string; + /** False when `name` is TREK's fallback because the source sent nothing that names the place. */ + named: boolean; lat: number | null; lng: number | null; startedAt: string; @@ -191,11 +260,18 @@ export function normalizeVisit(visit: DawarichVisitRaw): { : 0; const rawCode = visit.place?.country_code; + const name = visitSourceName(visit); + // A visit without a place or an area has no coordinate there; the middle of + // its own points, which newer builds send as `center`, is the next best one. + const placeLat = toNumber(visit.place?.latitude); + const placeLng = toNumber(visit.place?.longitude); + const fromPlace = placeLat !== null && placeLng !== null; return { sourceVisitId: String(visit.id), - name: (visit.name ?? '').trim() || 'Unnamed stay', - lat: toNumber(visit.place?.latitude), - lng: toNumber(visit.place?.longitude), + name: name ?? UNNAMED_STAY, + named: name !== null, + lat: fromPlace ? placeLat : toNumber(visit.center?.latitude), + lng: fromPlace ? placeLng : toNumber(visit.center?.longitude), startedAt: visit.started_at, endedAt, durationMinutes, diff --git a/server/tests/e2e/dawarich.e2e.test.ts b/server/tests/e2e/dawarich.e2e.test.ts index cf20090c1..b4b74abe2 100644 --- a/server/tests/e2e/dawarich.e2e.test.ts +++ b/server/tests/e2e/dawarich.e2e.test.ts @@ -78,6 +78,7 @@ function makeClientStub() { listVisits: boom('listVisits'), listTracks: boom('listTracks'), listPoints: boom('listPoints'), + listGeocodedPoints: boom('listGeocodedPoints'), listVisitedCities: boom('listVisitedCities'), findVisitsNear: boom('findVisitsNear'), }; diff --git a/server/tests/unit/nest/dawarich-sync.service.test.ts b/server/tests/unit/nest/dawarich-sync.service.test.ts index b08005804..1c21ba9cf 100644 --- a/server/tests/unit/nest/dawarich-sync.service.test.ts +++ b/server/tests/unit/nest/dawarich-sync.service.test.ts @@ -75,7 +75,8 @@ const CAPABILITIES: DawarichCapabilities = { }; const listVisits = vi.fn(); -const client = { listVisits } as unknown as DawarichClient; +const listGeocodedPoints = vi.fn(); +const client = { listVisits, listGeocodedPoints } as unknown as DawarichClient; /** Reads the same two columns the real service reads, so a row without a key is null. */ const getCredentials = vi.fn(async (userId: number): Promise => { @@ -282,6 +283,7 @@ beforeEach(async () => { t.clear(); vi.clearAllMocks(); probeCapabilities.mockResolvedValue(CAPABILITIES); + listGeocodedPoints.mockReset().mockResolvedValue([]); setAddonEnabled(testDb, 'dawarich', true); USER = createUser(testDb).user.id; @@ -296,6 +298,78 @@ afterAll(async () => { // ── Creating suggestions ───────────────────────────────────────────────────── +describe('DawarichSyncService, naming a stay Dawarich sends without a name (#3105)', () => { + const nameless = (id: number) => visit({ id, name: 'Unknown Location', place: null }); + + it('DAWARICH-SYNC-090: names it after the reverse-geocoded points recorded during it', async () => { + withVisits(nameless(901)); + listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]); + + await svc.syncUser(USER); + + expect((await only()).name).toBe('Cinisi, Italy'); + const [, from, to] = listGeocodedPoints.mock.calls[0] as [DawarichCreds, Date, Date]; + expect(from.toISOString()).toBe(`${VISIT_DAY}T09:00:00.000Z`); + expect(to.toISOString()).toBe(`${VISIT_DAY}T12:00:00.000Z`); + }); + + it('DAWARICH-SYNC-091: a name the visit carries itself needs no points read', async () => { + withVisits(visit({ id: 902, name: 'Hotel Adlon' })); + + await svc.syncUser(USER); + + expect((await only()).name).toBe('Hotel Adlon'); + expect(listGeocodedPoints).not.toHaveBeenCalled(); + }); + + it('DAWARICH-SYNC-092: a found name is kept on the next run without reading the points again', async () => { + withVisits(nameless(903)); + listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]); + await svc.syncUser(USER); + listGeocodedPoints.mockClear(); + + await svc.syncUser(USER); + + expect((await only()).name).toBe('Cinisi, Italy'); + expect(listGeocodedPoints).not.toHaveBeenCalled(); + }); + + it('DAWARICH-SYNC-093: points without a geocode settle on "Unnamed stay" until the visit changes', async () => { + withVisits(nameless(904)); + await svc.syncUser(USER); + expect((await only()).name).toBe('Unnamed stay'); + listGeocodedPoints.mockClear(); + + await svc.syncUser(USER); + + expect(listGeocodedPoints).not.toHaveBeenCalled(); + }); + + it('DAWARICH-SYNC-094: points that could not be read keep the placeholder and are read on the next run', async () => { + withVisits(nameless(905)); + listGeocodedPoints.mockRejectedValue(new DawarichError('unreachable', 'down')); + const result = await svc.syncUser(USER); + expect(result.state).toBe('ok'); + expect((await only()).name).toBe('Unknown Location'); + + listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]); + await svc.syncUser(USER); + + expect((await only()).name).toBe('Cinisi, Italy'); + }); + + it('DAWARICH-SYNC-095: after one failed read a run stops asking for the rest', async () => { + withVisits(nameless(906), nameless(907), nameless(908)); + listGeocodedPoints.mockRejectedValue(new DawarichError('unreachable', 'down')); + + await svc.syncUser(USER); + + // The first stay of each window is asked about; the rest wait for the next run. + expect(listGeocodedPoints).toHaveBeenCalledTimes(listVisits.mock.calls.length); + expect((await rows()).map((r) => r.name)).toEqual(['Unknown Location', 'Unknown Location', 'Unknown Location']); + }); +}); + describe('DawarichSyncService — new visits', () => { it('DAWARICH-SYNC-001: stores an unseen visit as a suggestion in state "new"', async () => { withVisits(visit({ id: 501, name: 'Hotel Adlon', confidence: 0.82, confidence_band: 'high' })); diff --git a/server/tests/unit/nest/dawarich.client.test.ts b/server/tests/unit/nest/dawarich.client.test.ts index 64f826e6d..38217c874 100644 --- a/server/tests/unit/nest/dawarich.client.test.ts +++ b/server/tests/unit/nest/dawarich.client.test.ts @@ -673,3 +673,26 @@ describe('DawarichClient — findVisitsNear', () => { await expect(client.findVisitsNear(CREDS, 0, 0, 100, 5)).resolves.toEqual([]); }); }); + +describe('DawarichClient, listGeocodedPoints', () => { + it('DAWARICH-CLIENT-100: reads one small page of full points, oldest first, to name a stay (#3105)', async () => { + const points = [{ city: 'Cinisi', country_name: 'Italy' }]; + answerWith(reply({ body: points })); + + const out = await client.listGeocodedPoints(CREDS, FROM, TO); + + const url = requestedUrl(0); + expect(url.pathname).toBe('/api/v1/points'); + expect(url.searchParams.get('start_at')).toBe(unixSeconds(FROM)); + expect(url.searchParams.get('end_at')).toBe(unixSeconds(TO)); + expect(url.searchParams.get('order')).toBe('asc'); + expect(url.searchParams.get('per_page')).toBe('50'); + expect(url.searchParams.get('slim')).toBeNull(); + expect(out).toEqual(points); + }); + + it('DAWARICH-CLIENT-101: an answer that is not a list names nothing', async () => { + answerWith(reply({ body: { error: 'nope' } })); + await expect(client.listGeocodedPoints(CREDS, FROM, TO)).resolves.toEqual([]); + }); +}); diff --git a/server/tests/unit/nest/dawarich.helpers.test.ts b/server/tests/unit/nest/dawarich.helpers.test.ts index 17dd79bad..540c199d5 100644 --- a/server/tests/unit/nest/dawarich.helpers.test.ts +++ b/server/tests/unit/nest/dawarich.helpers.test.ts @@ -25,6 +25,7 @@ import { localDateOf, mergeVisitedCountries, minutesBetween, + nameFromGeocodedPoints, normalizeVisit, offsetMinutesOf, parseTimeWindow, @@ -273,6 +274,7 @@ describe('normalizeVisit', () => { expect(normalizeVisit(BASE_VISIT)).toEqual({ sourceVisitId: '42', name: 'Cafe Kranzler', + named: true, lat: 52.5, lng: 13.4, startedAt: '2024-06-15T18:30:00+02:00', @@ -301,6 +303,7 @@ describe('normalizeVisit', () => { expect(normalizeVisit(sparse)).toEqual({ sourceVisitId: '9', name: 'Unnamed stay', + named: false, lat: null, lng: null, startedAt: '2024-06-15T18:30:00Z', @@ -423,6 +426,52 @@ describe('normalizeVisit', () => { expect(result?.durationMinutes).toBe(0); } }); + + it("DAWARICH-VISIT-018: Dawarich's placeholder names count as no name (#3105)", () => { + for (const name of ['Unknown Location', 'Suggested place', ' unknown location ']) { + const result = normalizeVisit(visit({ name, place: { latitude: 38.18, longitude: 13.14, id: null } })); + expect(result).toMatchObject({ name: 'Unnamed stay', named: false }); + } + }); + + it("DAWARICH-VISIT-019: a nameless visit takes its place's name, then the place's address", () => { + const place = { latitude: 52.5, longitude: 13.4, id: 7 }; + expect(normalizeVisit(visit({ name: 'Unknown Location', place: { ...place, name: 'Lido' } }))).toMatchObject({ + name: 'Lido', + named: true, + }); + expect( + normalizeVisit( + visit({ name: 'Suggested place', place: { ...place, name: 'Suggested place', address: 'Via Roma 1, Cinisi' } }), + ), + ).toMatchObject({ name: 'Via Roma 1, Cinisi', named: true }); + }); + + it('DAWARICH-VISIT-020: a visit without a place is placed at the middle of its points', () => { + const result = normalizeVisit(visit({ place: null, center: { latitude: '38.178824', longitude: 13.140629 } })); + expect(result?.lat).toBeCloseTo(38.178824, 6); + expect(result?.lng).toBeCloseTo(13.140629, 6); + }); +}); + +describe('nameFromGeocodedPoints', () => { + it('DAWARICH-VISIT-021: names a stay after the city most of its points fall in, with the country', () => { + expect( + nameFromGeocodedPoints([ + { city: null, country_name: null }, + { city: 'Terrasini', country_name: 'Italy' }, + { city: 'Cinisi', country_name: 'Italy' }, + { city: 'Cinisi', country_name: 'Italy' }, + ]), + ).toBe('Cinisi, Italy'); + }); + + it('DAWARICH-VISIT-022: falls back to the country, and to nothing when no point was geocoded', () => { + expect(nameFromGeocodedPoints([{ city: '', country_name: 'Italy' }, { country_name: null }])).toBe('Italy'); + expect(nameFromGeocodedPoints([{ city: 'Cinisi', country_name: null }])).toBe('Cinisi'); + expect(nameFromGeocodedPoints([{ city: null, country_name: null }])).toBeNull(); + expect(nameFromGeocodedPoints([])).toBeNull(); + }); }); // ── distanceMeters ───────────────────────────────────────────────────────────