fix(dawarich): name a stay from its geocoded points when Dawarich sends a placeholder

Since 1.22 Dawarich sends a visit's display name, which is "Unknown Location"
for a visit without a place. TREK showed that as the name, although the points
recorded during the stay are reverse geocoded. A stay without a real name now
takes its place's name or address, and otherwise the city most of its points
fall in. A visit without a place also takes the middle of its points as its
coordinate.

Closes #3105
This commit is contained in:
Maurice
2026-11-16 16:34:47 +01:00
parent eb56e77600
commit 04e0dc26e1
7 changed files with 334 additions and 9 deletions
@@ -13,7 +13,17 @@ import { logError, logInfo } from '../audit/audit-log.logger';
import { UnitOfWork } from '../database/unit-of-work';
import { JourneyRecordingPrepService, type PreparedJourney } from '../journey/journey-recording-prep.service';
import { DawarichClient, DawarichError, type DawarichCreds, type DawarichVisitRaw } from './dawarich.client';
import { distanceMeters, localDateOf, normalizeVisit, syncWindow, visitHash } from './dawarich.helpers';
import {
distanceMeters,
isPlaceholderName,
localDateOf,
nameFromGeocodedPoints,
normalizeVisit,
STAY_NAME_PENDING,
syncWindow,
UNNAMED_STAY,
visitHash,
} from './dawarich.helpers';
import { DawarichService } from './dawarich.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
@@ -27,6 +37,19 @@ import {
const DAY_MS = 24 * 60 * 60 * 1000;
/**
* Stays one run reads points for to name them. A first sync of a long trip
* can hold hundreds without a name; the rest wait for the next run rather than
* holding this one open (see `stayNamer`).
*/
const MAX_STAY_NAME_LOOKUPS = 100;
/**
* Names a stay from its points: the name, null when none of them was
* geocoded, undefined when they were not read this run.
*/
type StayNamer = (startedAt: string, endedAt: string) => Promise<string | null | undefined>;
/** What one user's sync produced — plus whether it ran at all. */
export interface DawarichSyncOutcome {
state: DawarichSyncState;
@@ -281,11 +304,12 @@ export class DawarichSyncService {
): Promise<{ created: number; updated: number; missing: number }> {
const { visits } = await this.client.listVisits(creds, from, to);
const nameStay = this.stayNamer(creds);
const seenIds = new Set<string>();
let created = 0;
let updated = 0;
for (const raw of visits) {
const outcome = await this.upsertVisit(userId, tripId, raw, trips);
const outcome = await this.upsertVisit(userId, tripId, raw, trips, nameStay);
if (!outcome) continue;
seenIds.add(outcome.sourceVisitId);
if (outcome.change === 'created') created++;
@@ -296,6 +320,48 @@ export class DawarichSyncService {
return { created, updated, missing };
}
/**
* Reads a stay's points for a name, for one run. A run reads at most
* MAX_STAY_NAME_LOOKUPS stays, and stops reading after the first failure,
* since an instance that did not answer one will not answer the rest either.
* Either way the stay is left for the next run, not given up on.
*/
private stayNamer(creds: DawarichCreds): StayNamer {
let left = MAX_STAY_NAME_LOOKUPS;
return async (startedAt, endedAt) => {
if (left <= 0) return undefined;
left--;
try {
const points = await this.client.listGeocodedPoints(creds, new Date(startedAt), new Date(endedAt));
return nameFromGeocodedPoints(points);
} catch {
left = 0;
return undefined;
}
};
}
/**
* What a stay is called. Its own name when Dawarich sent one. Otherwise the
* reverse-geocoded points recorded during it name it (#3105): Dawarich sends
* a placeholder for a visit without a place even when its points know the
* town. A name found earlier is kept while the visit is unchanged, and so is
* the finding that none of its points was geocoded; a stay whose points
* were not read yet keeps the placeholder, which the next run reads again.
*/
private async stayName(
visit: { name: string; named: boolean; startedAt: string; endedAt: string },
hash: string,
existing: { name: string | null; source_hash: string | null } | null,
nameStay: StayNamer,
): Promise<string> {
if (visit.named) return visit.name;
if (existing?.name && existing.source_hash === hash && !isPlaceholderName(existing.name)) return existing.name;
const found = await nameStay(visit.startedAt, visit.endedAt);
if (found === undefined) return STAY_NAME_PENDING;
return found ?? UNNAMED_STAY;
}
/**
* One visit into the user's suggestions: inserted when new, overwritten while
* nobody acted on it, only its hash refreshed once accepted or dismissed.
@@ -309,6 +375,7 @@ export class DawarichSyncService {
tripId: number | null,
raw: DawarichVisitRaw,
trips: TripRow[],
nameStay: StayNamer,
): Promise<{ sourceVisitId: string; change: 'created' | 'updated' | 'unchanged' } | null> {
const visit = normalizeVisit(raw);
if (!visit) return null;
@@ -331,7 +398,7 @@ export class DawarichSyncService {
user_id: userId,
source_visit_id: visit.sourceVisitId,
trip_id: ownerTripId,
name: visit.name,
name: await this.stayName(visit, hash, null, nameStay),
lat: visit.lat,
lng: visit.lng,
started_at: visit.startedAt,
@@ -358,7 +425,7 @@ export class DawarichSyncService {
// that includes the trip: a stay parked on a neighbour by an earlier
// run moves to the trip whose dates hold it.
await this.suggestions.updateNewVisit(existing.id, {
name: visit.name,
name: await this.stayName(visit, hash, existing, nameStay),
lat: visit.lat,
lng: visit.lng,
started_at: visit.startedAt,
@@ -424,10 +491,11 @@ export class DawarichSyncService {
trips: TripRow[],
): Promise<{ created: number; updated: number; truncated: boolean }> {
const { visits, truncated } = await this.client.listVisits(creds, from, to);
const nameStay = this.stayNamer(creds);
let created = 0;
let updated = 0;
for (const raw of visits) {
const outcome = await this.upsertVisit(userId, null, raw, trips);
const outcome = await this.upsertVisit(userId, null, raw, trips, nameStay);
if (outcome?.change === 'created') created++;
else if (outcome?.change === 'updated') updated++;
}
@@ -47,6 +47,8 @@ export const VISITS_PAGE_SIZE = 500;
export const TRACKS_PAGE_SIZE = 200;
/** Points cap at 10000 upstream. A day of tracking is a few thousand. */
export const POINTS_PAGE_SIZE = 5000;
/** Points read to name one stay. Enough to get past a few not yet geocoded. */
export const GEOCODED_POINTS_PAGE_SIZE = 50;
/**
* How many pages one call will walk before giving up and saying so.
@@ -104,11 +106,26 @@ export interface DawarichVisitRaw {
latitude: number | string | null;
longitude: number | string | null;
id: number | null;
/** Sent since 1.22, alongside the visit's own `name`. */
name?: string | null;
/** Street and locality from the place's geodata, since 1.22. */
address?: string | null;
/** Requested upstream, absent at 1.14.4. Read defensively so it lights up on its own. */
country_code?: string | null;
} | null;
/** Requested upstream, absent at 1.14.4. */
updated_at?: string | null;
/**
* The weighted middle of the visit's own points, since 1.22. The only
* coordinate a visit without a place or an area carries.
*/
center?: { latitude: number | string | null; longitude: number | string | null } | null;
}
/** The reverse-geocoded part of one entry of `GET /api/v1/points` (not slim). */
export interface DawarichGeocodedPoint {
city?: string | null;
country_name?: string | null;
}
/** One GeoJSON feature from `GET /api/v1/tracks`. */
@@ -382,6 +399,23 @@ export class DawarichClient {
return { points, truncated };
}
/**
* The first recorded points of a stay with what reverse geocoding made of
* them, to name a visit Dawarich sends without one (#3105). One small page:
* a stay's opening points name it as well as all of them would, and the full
* serializer carries geodata, so a whole day of it is no cheap answer.
*/
async listGeocodedPoints(creds: DawarichCreds, from: Date, to: Date): Promise<DawarichGeocodedPoint[]> {
const { data } = await this.get<DawarichGeocodedPoint[]>(creds, '/points', {
start_at: unixSeconds(from),
end_at: unixSeconds(to),
order: 'asc',
page: '1',
per_page: String(GEOCODED_POINTS_PAGE_SIZE),
});
return Array.isArray(data) ? data : [];
}
/**
* Countries and cities the recordings put the user in, for the Atlas.
*
@@ -1,5 +1,6 @@
import { DomainError } from '../common/domain-error';
import type {
DawarichGeocodedPoint,
DawarichSlimPoint,
DawarichTrackFeature,
DawarichVisitRaw,
@@ -134,6 +135,72 @@ export function toNumber(value: unknown): number | null {
return null;
}
/** What a stay is called when nothing names it. */
export const UNNAMED_STAY = 'Unnamed stay';
/**
* The names Dawarich gives a visit or a place when it knows nothing about
* either (`Visit::DEFAULT_NAME`, `Place::DEFAULT_NAME`). Since 1.22 the
* visit's `name` is its display name, so one of these arrives in place of
* nothing, and taken as a name it hid the address the recording does hold
* (#3105).
*/
const DAWARICH_PLACEHOLDER_NAMES = new Set(['unknown location', 'suggested place']);
/** A name worth showing: set, and not one of Dawarich's placeholders. */
function realName(value: unknown): string | null {
if (typeof value !== 'string') return null;
const trimmed = value.trim();
if (!trimmed || DAWARICH_PLACEHOLDER_NAMES.has(trimmed.toLowerCase())) return null;
return trimmed;
}
/**
* The best name the visit itself carries: its own, its place's, then the
* place's address. Null when there is none of them.
*/
export function visitSourceName(visit: DawarichVisitRaw): string | null {
return realName(visit.name) ?? realName(visit.place?.name) ?? realName(visit.place?.address);
}
/**
* What a stay is stored as while its points are still to be read for a name:
* Dawarich's own placeholder, which is what the source said, and which the
* next sync recognises and tries again.
*/
export const STAY_NAME_PENDING = 'Unknown Location';
/** True for a stored name that is one of Dawarich's placeholders, or none at all. */
export function isPlaceholderName(name: string | null | undefined): boolean {
return realName(name) === null;
}
/**
* A stay named from the reverse-geocoded points recorded during it: the city
* most of them fall in with its country ("Cinisi, Italy"), or the country
* alone when no point has a city. Null when none was geocoded.
*/
export function nameFromGeocodedPoints(points: readonly DawarichGeocodedPoint[]): string | null {
const cities = new Map<string, { count: number; country: string | null }>();
const countries = new Map<string, number>();
for (const point of points) {
const city = realName(point.city);
const country = realName(point.country_name);
if (city) {
const seen = cities.get(city);
if (seen) seen.count++;
else cities.set(city, { count: 1, country });
}
if (country) countries.set(country, (countries.get(country) ?? 0) + 1);
}
let city: [string, { count: number; country: string | null }] | null = null;
for (const entry of cities) if (!city || entry[1].count > city[1].count) city = entry;
if (city) return city[1].country ? `${city[0]}, ${city[1].country}` : city[0];
let country: [string, number] | null = null;
for (const entry of countries) if (!country || entry[1] > country[1]) country = entry;
return country ? country[0] : null;
}
/**
* A visit reduced to what TREK stores.
*
@@ -144,6 +211,8 @@ export function toNumber(value: unknown): number | null {
export function normalizeVisit(visit: DawarichVisitRaw): {
sourceVisitId: string;
name: string;
/** False when `name` is TREK's fallback because the source sent nothing that names the place. */
named: boolean;
lat: number | null;
lng: number | null;
startedAt: string;
@@ -191,11 +260,18 @@ export function normalizeVisit(visit: DawarichVisitRaw): {
: 0;
const rawCode = visit.place?.country_code;
const name = visitSourceName(visit);
// A visit without a place or an area has no coordinate there; the middle of
// its own points, which newer builds send as `center`, is the next best one.
const placeLat = toNumber(visit.place?.latitude);
const placeLng = toNumber(visit.place?.longitude);
const fromPlace = placeLat !== null && placeLng !== null;
return {
sourceVisitId: String(visit.id),
name: (visit.name ?? '').trim() || 'Unnamed stay',
lat: toNumber(visit.place?.latitude),
lng: toNumber(visit.place?.longitude),
name: name ?? UNNAMED_STAY,
named: name !== null,
lat: fromPlace ? placeLat : toNumber(visit.center?.latitude),
lng: fromPlace ? placeLng : toNumber(visit.center?.longitude),
startedAt: visit.started_at,
endedAt,
durationMinutes,
+1
View File
@@ -78,6 +78,7 @@ function makeClientStub() {
listVisits: boom('listVisits'),
listTracks: boom('listTracks'),
listPoints: boom('listPoints'),
listGeocodedPoints: boom('listGeocodedPoints'),
listVisitedCities: boom('listVisitedCities'),
findVisitsNear: boom('findVisitsNear'),
};
@@ -75,7 +75,8 @@ const CAPABILITIES: DawarichCapabilities = {
};
const listVisits = vi.fn();
const client = { listVisits } as unknown as DawarichClient;
const listGeocodedPoints = vi.fn();
const client = { listVisits, listGeocodedPoints } as unknown as DawarichClient;
/** Reads the same two columns the real service reads, so a row without a key is null. */
const getCredentials = vi.fn(async (userId: number): Promise<DawarichCreds | null> => {
@@ -282,6 +283,7 @@ beforeEach(async () => {
t.clear();
vi.clearAllMocks();
probeCapabilities.mockResolvedValue(CAPABILITIES);
listGeocodedPoints.mockReset().mockResolvedValue([]);
setAddonEnabled(testDb, 'dawarich', true);
USER = createUser(testDb).user.id;
@@ -296,6 +298,78 @@ afterAll(async () => {
// ── Creating suggestions ─────────────────────────────────────────────────────
describe('DawarichSyncService, naming a stay Dawarich sends without a name (#3105)', () => {
const nameless = (id: number) => visit({ id, name: 'Unknown Location', place: null });
it('DAWARICH-SYNC-090: names it after the reverse-geocoded points recorded during it', async () => {
withVisits(nameless(901));
listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]);
await svc.syncUser(USER);
expect((await only()).name).toBe('Cinisi, Italy');
const [, from, to] = listGeocodedPoints.mock.calls[0] as [DawarichCreds, Date, Date];
expect(from.toISOString()).toBe(`${VISIT_DAY}T09:00:00.000Z`);
expect(to.toISOString()).toBe(`${VISIT_DAY}T12:00:00.000Z`);
});
it('DAWARICH-SYNC-091: a name the visit carries itself needs no points read', async () => {
withVisits(visit({ id: 902, name: 'Hotel Adlon' }));
await svc.syncUser(USER);
expect((await only()).name).toBe('Hotel Adlon');
expect(listGeocodedPoints).not.toHaveBeenCalled();
});
it('DAWARICH-SYNC-092: a found name is kept on the next run without reading the points again', async () => {
withVisits(nameless(903));
listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]);
await svc.syncUser(USER);
listGeocodedPoints.mockClear();
await svc.syncUser(USER);
expect((await only()).name).toBe('Cinisi, Italy');
expect(listGeocodedPoints).not.toHaveBeenCalled();
});
it('DAWARICH-SYNC-093: points without a geocode settle on "Unnamed stay" until the visit changes', async () => {
withVisits(nameless(904));
await svc.syncUser(USER);
expect((await only()).name).toBe('Unnamed stay');
listGeocodedPoints.mockClear();
await svc.syncUser(USER);
expect(listGeocodedPoints).not.toHaveBeenCalled();
});
it('DAWARICH-SYNC-094: points that could not be read keep the placeholder and are read on the next run', async () => {
withVisits(nameless(905));
listGeocodedPoints.mockRejectedValue(new DawarichError('unreachable', 'down'));
const result = await svc.syncUser(USER);
expect(result.state).toBe('ok');
expect((await only()).name).toBe('Unknown Location');
listGeocodedPoints.mockResolvedValue([{ city: 'Cinisi', country_name: 'Italy' }]);
await svc.syncUser(USER);
expect((await only()).name).toBe('Cinisi, Italy');
});
it('DAWARICH-SYNC-095: after one failed read a run stops asking for the rest', async () => {
withVisits(nameless(906), nameless(907), nameless(908));
listGeocodedPoints.mockRejectedValue(new DawarichError('unreachable', 'down'));
await svc.syncUser(USER);
// The first stay of each window is asked about; the rest wait for the next run.
expect(listGeocodedPoints).toHaveBeenCalledTimes(listVisits.mock.calls.length);
expect((await rows()).map((r) => r.name)).toEqual(['Unknown Location', 'Unknown Location', 'Unknown Location']);
});
});
describe('DawarichSyncService — new visits', () => {
it('DAWARICH-SYNC-001: stores an unseen visit as a suggestion in state "new"', async () => {
withVisits(visit({ id: 501, name: 'Hotel Adlon', confidence: 0.82, confidence_band: 'high' }));
@@ -673,3 +673,26 @@ describe('DawarichClient — findVisitsNear', () => {
await expect(client.findVisitsNear(CREDS, 0, 0, 100, 5)).resolves.toEqual([]);
});
});
describe('DawarichClient, listGeocodedPoints', () => {
it('DAWARICH-CLIENT-100: reads one small page of full points, oldest first, to name a stay (#3105)', async () => {
const points = [{ city: 'Cinisi', country_name: 'Italy' }];
answerWith(reply({ body: points }));
const out = await client.listGeocodedPoints(CREDS, FROM, TO);
const url = requestedUrl(0);
expect(url.pathname).toBe('/api/v1/points');
expect(url.searchParams.get('start_at')).toBe(unixSeconds(FROM));
expect(url.searchParams.get('end_at')).toBe(unixSeconds(TO));
expect(url.searchParams.get('order')).toBe('asc');
expect(url.searchParams.get('per_page')).toBe('50');
expect(url.searchParams.get('slim')).toBeNull();
expect(out).toEqual(points);
});
it('DAWARICH-CLIENT-101: an answer that is not a list names nothing', async () => {
answerWith(reply({ body: { error: 'nope' } }));
await expect(client.listGeocodedPoints(CREDS, FROM, TO)).resolves.toEqual([]);
});
});
@@ -25,6 +25,7 @@ import {
localDateOf,
mergeVisitedCountries,
minutesBetween,
nameFromGeocodedPoints,
normalizeVisit,
offsetMinutesOf,
parseTimeWindow,
@@ -273,6 +274,7 @@ describe('normalizeVisit', () => {
expect(normalizeVisit(BASE_VISIT)).toEqual({
sourceVisitId: '42',
name: 'Cafe Kranzler',
named: true,
lat: 52.5,
lng: 13.4,
startedAt: '2024-06-15T18:30:00+02:00',
@@ -301,6 +303,7 @@ describe('normalizeVisit', () => {
expect(normalizeVisit(sparse)).toEqual({
sourceVisitId: '9',
name: 'Unnamed stay',
named: false,
lat: null,
lng: null,
startedAt: '2024-06-15T18:30:00Z',
@@ -423,6 +426,52 @@ describe('normalizeVisit', () => {
expect(result?.durationMinutes).toBe(0);
}
});
it("DAWARICH-VISIT-018: Dawarich's placeholder names count as no name (#3105)", () => {
for (const name of ['Unknown Location', 'Suggested place', ' unknown location ']) {
const result = normalizeVisit(visit({ name, place: { latitude: 38.18, longitude: 13.14, id: null } }));
expect(result).toMatchObject({ name: 'Unnamed stay', named: false });
}
});
it("DAWARICH-VISIT-019: a nameless visit takes its place's name, then the place's address", () => {
const place = { latitude: 52.5, longitude: 13.4, id: 7 };
expect(normalizeVisit(visit({ name: 'Unknown Location', place: { ...place, name: 'Lido' } }))).toMatchObject({
name: 'Lido',
named: true,
});
expect(
normalizeVisit(
visit({ name: 'Suggested place', place: { ...place, name: 'Suggested place', address: 'Via Roma 1, Cinisi' } }),
),
).toMatchObject({ name: 'Via Roma 1, Cinisi', named: true });
});
it('DAWARICH-VISIT-020: a visit without a place is placed at the middle of its points', () => {
const result = normalizeVisit(visit({ place: null, center: { latitude: '38.178824', longitude: 13.140629 } }));
expect(result?.lat).toBeCloseTo(38.178824, 6);
expect(result?.lng).toBeCloseTo(13.140629, 6);
});
});
describe('nameFromGeocodedPoints', () => {
it('DAWARICH-VISIT-021: names a stay after the city most of its points fall in, with the country', () => {
expect(
nameFromGeocodedPoints([
{ city: null, country_name: null },
{ city: 'Terrasini', country_name: 'Italy' },
{ city: 'Cinisi', country_name: 'Italy' },
{ city: 'Cinisi', country_name: 'Italy' },
]),
).toBe('Cinisi, Italy');
});
it('DAWARICH-VISIT-022: falls back to the country, and to nothing when no point was geocoded', () => {
expect(nameFromGeocodedPoints([{ city: '', country_name: 'Italy' }, { country_name: null }])).toBe('Italy');
expect(nameFromGeocodedPoints([{ city: 'Cinisi', country_name: null }])).toBe('Cinisi');
expect(nameFromGeocodedPoints([{ city: null, country_name: null }])).toBeNull();
expect(nameFromGeocodedPoints([])).toBeNull();
});
});
// ── distanceMeters ───────────────────────────────────────────────────────────