mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 19:40:34 +00:00
fix(dawarich): name a stay from its geocoded points when Dawarich sends a placeholder
Since 1.22 Dawarich sends a visit's display name, which is "Unknown Location" for a visit without a place. TREK showed that as the name, although the points recorded during the stay are reverse geocoded. A stay without a real name now takes its place's name or address, and otherwise the city most of its points fall in. A visit without a place also takes the middle of its points as its coordinate. Closes #3105
This commit is contained in:
@@ -13,7 +13,17 @@ import { logError, logInfo } from '../audit/audit-log.logger';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { JourneyRecordingPrepService, type PreparedJourney } from '../journey/journey-recording-prep.service';
|
||||
import { DawarichClient, DawarichError, type DawarichCreds, type DawarichVisitRaw } from './dawarich.client';
|
||||
import { distanceMeters, localDateOf, normalizeVisit, syncWindow, visitHash } from './dawarich.helpers';
|
||||
import {
|
||||
distanceMeters,
|
||||
isPlaceholderName,
|
||||
localDateOf,
|
||||
nameFromGeocodedPoints,
|
||||
normalizeVisit,
|
||||
STAY_NAME_PENDING,
|
||||
syncWindow,
|
||||
UNNAMED_STAY,
|
||||
visitHash,
|
||||
} from './dawarich.helpers';
|
||||
import { DawarichService } from './dawarich.service';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
@@ -27,6 +37,19 @@ import {
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Stays one run reads points for to name them. A first sync of a long trip
|
||||
* can hold hundreds without a name; the rest wait for the next run rather than
|
||||
* holding this one open (see `stayNamer`).
|
||||
*/
|
||||
const MAX_STAY_NAME_LOOKUPS = 100;
|
||||
|
||||
/**
|
||||
* Names a stay from its points: the name, null when none of them was
|
||||
* geocoded, undefined when they were not read this run.
|
||||
*/
|
||||
type StayNamer = (startedAt: string, endedAt: string) => Promise<string | null | undefined>;
|
||||
|
||||
/** What one user's sync produced — plus whether it ran at all. */
|
||||
export interface DawarichSyncOutcome {
|
||||
state: DawarichSyncState;
|
||||
@@ -281,11 +304,12 @@ export class DawarichSyncService {
|
||||
): Promise<{ created: number; updated: number; missing: number }> {
|
||||
const { visits } = await this.client.listVisits(creds, from, to);
|
||||
|
||||
const nameStay = this.stayNamer(creds);
|
||||
const seenIds = new Set<string>();
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
for (const raw of visits) {
|
||||
const outcome = await this.upsertVisit(userId, tripId, raw, trips);
|
||||
const outcome = await this.upsertVisit(userId, tripId, raw, trips, nameStay);
|
||||
if (!outcome) continue;
|
||||
seenIds.add(outcome.sourceVisitId);
|
||||
if (outcome.change === 'created') created++;
|
||||
@@ -296,6 +320,48 @@ export class DawarichSyncService {
|
||||
return { created, updated, missing };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a stay's points for a name, for one run. A run reads at most
|
||||
* MAX_STAY_NAME_LOOKUPS stays, and stops reading after the first failure,
|
||||
* since an instance that did not answer one will not answer the rest either.
|
||||
* Either way the stay is left for the next run, not given up on.
|
||||
*/
|
||||
private stayNamer(creds: DawarichCreds): StayNamer {
|
||||
let left = MAX_STAY_NAME_LOOKUPS;
|
||||
return async (startedAt, endedAt) => {
|
||||
if (left <= 0) return undefined;
|
||||
left--;
|
||||
try {
|
||||
const points = await this.client.listGeocodedPoints(creds, new Date(startedAt), new Date(endedAt));
|
||||
return nameFromGeocodedPoints(points);
|
||||
} catch {
|
||||
left = 0;
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* What a stay is called. Its own name when Dawarich sent one. Otherwise the
|
||||
* reverse-geocoded points recorded during it name it (#3105): Dawarich sends
|
||||
* a placeholder for a visit without a place even when its points know the
|
||||
* town. A name found earlier is kept while the visit is unchanged, and so is
|
||||
* the finding that none of its points was geocoded; a stay whose points
|
||||
* were not read yet keeps the placeholder, which the next run reads again.
|
||||
*/
|
||||
private async stayName(
|
||||
visit: { name: string; named: boolean; startedAt: string; endedAt: string },
|
||||
hash: string,
|
||||
existing: { name: string | null; source_hash: string | null } | null,
|
||||
nameStay: StayNamer,
|
||||
): Promise<string> {
|
||||
if (visit.named) return visit.name;
|
||||
if (existing?.name && existing.source_hash === hash && !isPlaceholderName(existing.name)) return existing.name;
|
||||
const found = await nameStay(visit.startedAt, visit.endedAt);
|
||||
if (found === undefined) return STAY_NAME_PENDING;
|
||||
return found ?? UNNAMED_STAY;
|
||||
}
|
||||
|
||||
/**
|
||||
* One visit into the user's suggestions: inserted when new, overwritten while
|
||||
* nobody acted on it, only its hash refreshed once accepted or dismissed.
|
||||
@@ -309,6 +375,7 @@ export class DawarichSyncService {
|
||||
tripId: number | null,
|
||||
raw: DawarichVisitRaw,
|
||||
trips: TripRow[],
|
||||
nameStay: StayNamer,
|
||||
): Promise<{ sourceVisitId: string; change: 'created' | 'updated' | 'unchanged' } | null> {
|
||||
const visit = normalizeVisit(raw);
|
||||
if (!visit) return null;
|
||||
@@ -331,7 +398,7 @@ export class DawarichSyncService {
|
||||
user_id: userId,
|
||||
source_visit_id: visit.sourceVisitId,
|
||||
trip_id: ownerTripId,
|
||||
name: visit.name,
|
||||
name: await this.stayName(visit, hash, null, nameStay),
|
||||
lat: visit.lat,
|
||||
lng: visit.lng,
|
||||
started_at: visit.startedAt,
|
||||
@@ -358,7 +425,7 @@ export class DawarichSyncService {
|
||||
// that includes the trip: a stay parked on a neighbour by an earlier
|
||||
// run moves to the trip whose dates hold it.
|
||||
await this.suggestions.updateNewVisit(existing.id, {
|
||||
name: visit.name,
|
||||
name: await this.stayName(visit, hash, existing, nameStay),
|
||||
lat: visit.lat,
|
||||
lng: visit.lng,
|
||||
started_at: visit.startedAt,
|
||||
@@ -424,10 +491,11 @@ export class DawarichSyncService {
|
||||
trips: TripRow[],
|
||||
): Promise<{ created: number; updated: number; truncated: boolean }> {
|
||||
const { visits, truncated } = await this.client.listVisits(creds, from, to);
|
||||
const nameStay = this.stayNamer(creds);
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
for (const raw of visits) {
|
||||
const outcome = await this.upsertVisit(userId, null, raw, trips);
|
||||
const outcome = await this.upsertVisit(userId, null, raw, trips, nameStay);
|
||||
if (outcome?.change === 'created') created++;
|
||||
else if (outcome?.change === 'updated') updated++;
|
||||
}
|
||||
|
||||
@@ -47,6 +47,8 @@ export const VISITS_PAGE_SIZE = 500;
|
||||
export const TRACKS_PAGE_SIZE = 200;
|
||||
/** Points cap at 10000 upstream. A day of tracking is a few thousand. */
|
||||
export const POINTS_PAGE_SIZE = 5000;
|
||||
/** Points read to name one stay. Enough to get past a few not yet geocoded. */
|
||||
export const GEOCODED_POINTS_PAGE_SIZE = 50;
|
||||
|
||||
/**
|
||||
* How many pages one call will walk before giving up and saying so.
|
||||
@@ -104,11 +106,26 @@ export interface DawarichVisitRaw {
|
||||
latitude: number | string | null;
|
||||
longitude: number | string | null;
|
||||
id: number | null;
|
||||
/** Sent since 1.22, alongside the visit's own `name`. */
|
||||
name?: string | null;
|
||||
/** Street and locality from the place's geodata, since 1.22. */
|
||||
address?: string | null;
|
||||
/** Requested upstream, absent at 1.14.4. Read defensively so it lights up on its own. */
|
||||
country_code?: string | null;
|
||||
} | null;
|
||||
/** Requested upstream, absent at 1.14.4. */
|
||||
updated_at?: string | null;
|
||||
/**
|
||||
* The weighted middle of the visit's own points, since 1.22. The only
|
||||
* coordinate a visit without a place or an area carries.
|
||||
*/
|
||||
center?: { latitude: number | string | null; longitude: number | string | null } | null;
|
||||
}
|
||||
|
||||
/** The reverse-geocoded part of one entry of `GET /api/v1/points` (not slim). */
|
||||
export interface DawarichGeocodedPoint {
|
||||
city?: string | null;
|
||||
country_name?: string | null;
|
||||
}
|
||||
|
||||
/** One GeoJSON feature from `GET /api/v1/tracks`. */
|
||||
@@ -382,6 +399,23 @@ export class DawarichClient {
|
||||
return { points, truncated };
|
||||
}
|
||||
|
||||
/**
|
||||
* The first recorded points of a stay with what reverse geocoding made of
|
||||
* them, to name a visit Dawarich sends without one (#3105). One small page:
|
||||
* a stay's opening points name it as well as all of them would, and the full
|
||||
* serializer carries geodata, so a whole day of it is no cheap answer.
|
||||
*/
|
||||
async listGeocodedPoints(creds: DawarichCreds, from: Date, to: Date): Promise<DawarichGeocodedPoint[]> {
|
||||
const { data } = await this.get<DawarichGeocodedPoint[]>(creds, '/points', {
|
||||
start_at: unixSeconds(from),
|
||||
end_at: unixSeconds(to),
|
||||
order: 'asc',
|
||||
page: '1',
|
||||
per_page: String(GEOCODED_POINTS_PAGE_SIZE),
|
||||
});
|
||||
return Array.isArray(data) ? data : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Countries and cities the recordings put the user in, for the Atlas.
|
||||
*
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import type {
|
||||
DawarichGeocodedPoint,
|
||||
DawarichSlimPoint,
|
||||
DawarichTrackFeature,
|
||||
DawarichVisitRaw,
|
||||
@@ -134,6 +135,72 @@ export function toNumber(value: unknown): number | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** What a stay is called when nothing names it. */
|
||||
export const UNNAMED_STAY = 'Unnamed stay';
|
||||
|
||||
/**
|
||||
* The names Dawarich gives a visit or a place when it knows nothing about
|
||||
* either (`Visit::DEFAULT_NAME`, `Place::DEFAULT_NAME`). Since 1.22 the
|
||||
* visit's `name` is its display name, so one of these arrives in place of
|
||||
* nothing, and taken as a name it hid the address the recording does hold
|
||||
* (#3105).
|
||||
*/
|
||||
const DAWARICH_PLACEHOLDER_NAMES = new Set(['unknown location', 'suggested place']);
|
||||
|
||||
/** A name worth showing: set, and not one of Dawarich's placeholders. */
|
||||
function realName(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed || DAWARICH_PLACEHOLDER_NAMES.has(trimmed.toLowerCase())) return null;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* The best name the visit itself carries: its own, its place's, then the
|
||||
* place's address. Null when there is none of them.
|
||||
*/
|
||||
export function visitSourceName(visit: DawarichVisitRaw): string | null {
|
||||
return realName(visit.name) ?? realName(visit.place?.name) ?? realName(visit.place?.address);
|
||||
}
|
||||
|
||||
/**
|
||||
* What a stay is stored as while its points are still to be read for a name:
|
||||
* Dawarich's own placeholder, which is what the source said, and which the
|
||||
* next sync recognises and tries again.
|
||||
*/
|
||||
export const STAY_NAME_PENDING = 'Unknown Location';
|
||||
|
||||
/** True for a stored name that is one of Dawarich's placeholders, or none at all. */
|
||||
export function isPlaceholderName(name: string | null | undefined): boolean {
|
||||
return realName(name) === null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A stay named from the reverse-geocoded points recorded during it: the city
|
||||
* most of them fall in with its country ("Cinisi, Italy"), or the country
|
||||
* alone when no point has a city. Null when none was geocoded.
|
||||
*/
|
||||
export function nameFromGeocodedPoints(points: readonly DawarichGeocodedPoint[]): string | null {
|
||||
const cities = new Map<string, { count: number; country: string | null }>();
|
||||
const countries = new Map<string, number>();
|
||||
for (const point of points) {
|
||||
const city = realName(point.city);
|
||||
const country = realName(point.country_name);
|
||||
if (city) {
|
||||
const seen = cities.get(city);
|
||||
if (seen) seen.count++;
|
||||
else cities.set(city, { count: 1, country });
|
||||
}
|
||||
if (country) countries.set(country, (countries.get(country) ?? 0) + 1);
|
||||
}
|
||||
let city: [string, { count: number; country: string | null }] | null = null;
|
||||
for (const entry of cities) if (!city || entry[1].count > city[1].count) city = entry;
|
||||
if (city) return city[1].country ? `${city[0]}, ${city[1].country}` : city[0];
|
||||
let country: [string, number] | null = null;
|
||||
for (const entry of countries) if (!country || entry[1] > country[1]) country = entry;
|
||||
return country ? country[0] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A visit reduced to what TREK stores.
|
||||
*
|
||||
@@ -144,6 +211,8 @@ export function toNumber(value: unknown): number | null {
|
||||
export function normalizeVisit(visit: DawarichVisitRaw): {
|
||||
sourceVisitId: string;
|
||||
name: string;
|
||||
/** False when `name` is TREK's fallback because the source sent nothing that names the place. */
|
||||
named: boolean;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
startedAt: string;
|
||||
@@ -191,11 +260,18 @@ export function normalizeVisit(visit: DawarichVisitRaw): {
|
||||
: 0;
|
||||
|
||||
const rawCode = visit.place?.country_code;
|
||||
const name = visitSourceName(visit);
|
||||
// A visit without a place or an area has no coordinate there; the middle of
|
||||
// its own points, which newer builds send as `center`, is the next best one.
|
||||
const placeLat = toNumber(visit.place?.latitude);
|
||||
const placeLng = toNumber(visit.place?.longitude);
|
||||
const fromPlace = placeLat !== null && placeLng !== null;
|
||||
return {
|
||||
sourceVisitId: String(visit.id),
|
||||
name: (visit.name ?? '').trim() || 'Unnamed stay',
|
||||
lat: toNumber(visit.place?.latitude),
|
||||
lng: toNumber(visit.place?.longitude),
|
||||
name: name ?? UNNAMED_STAY,
|
||||
named: name !== null,
|
||||
lat: fromPlace ? placeLat : toNumber(visit.center?.latitude),
|
||||
lng: fromPlace ? placeLng : toNumber(visit.center?.longitude),
|
||||
startedAt: visit.started_at,
|
||||
endedAt,
|
||||
durationMinutes,
|
||||
|
||||
Reference in New Issue
Block a user