feat(maps): count how much routing an instance actually does

Every route is calculated in the browser against the public FOSSGIS hosts, so the
server never sees a routing request and nobody has a number: not how many a real trip
costs, not how big they are, not how much already goes to a self-hosted engine.
Deciding whether TREK could run its own router without that number is guessing at the
bill, and the counters have to be collecting before the question comes up.

Counters, not a log. A row is a day, a profile, a request kind and an engine kind with
totals on it: how many requests, how many waypoints, roughly how far, how many were
refused. No query, no coordinate, no route, no user, no trip. Nothing leaves the
instance: its own database holds them and its own admin reads them at
GET /api/route-usage/summary.

The client tallies in memory and flushes in batches — on a timer, past a threshold and
when the page is hidden — because a counter that cost a request of its own would
measure the load by adding to it. Routing all funnels through one fetch in
RouteCalculator, so that is the only place instrumented.

On by default and switchable off with route_usage_enabled, unlike the shadow log next
door: that one stores what people typed, this one stores four integers about a day.
This commit is contained in:
Maurice
2026-09-09 14:44:32 +02:00
committed by Maurice
parent 88129e4546
commit d30b916f9c
14 changed files with 830 additions and 6 deletions
+31
View File
@@ -4483,6 +4483,37 @@ function runMigrations(db: Database.Database): void {
)`);
db.exec('CREATE INDEX IF NOT EXISTS idx_collab_links_trip ON collab_links(trip_id)');
},
/**
* Route usage counters — how much routing this instance really does.
*
* Daily aggregates, not a log: one row per day, profile, surface and engine
* kind, carrying totals. No query, no coordinate, no route, no user, no trip.
* The question they answer is whether TREK could host a router itself, and
* that needs volume, not itineraries.
*
* The table is created regardless of the switch, like the shadow log above:
* the switch decides whether rows are written, and a schema that appears only
* when a feature is on is a schema that differs between installs.
*
* Appended LAST: the array is index-addressed against schema_version.
*/
() => {
db.exec(`
CREATE TABLE IF NOT EXISTS route_usage_daily (
day TEXT NOT NULL,
profile TEXT NOT NULL,
surface TEXT NOT NULL,
self_hosted INTEGER NOT NULL,
requests INTEGER NOT NULL DEFAULT 0,
waypoints INTEGER NOT NULL DEFAULT 0,
km REAL NOT NULL DEFAULT 0,
failed INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (day, profile, surface, self_hosted)
)
`);
// Retention deletes by day, and the summary reads the newest days first.
db.exec('CREATE INDEX IF NOT EXISTS idx_route_usage_day ON route_usage_daily(day)');
},
];
if (currentVersion < migrations.length) {
+2 -1
View File
@@ -22,6 +22,7 @@ import { MapsModule } from './maps/maps.module';
import { GeoModule } from './geo/geo.module';
import { PlaceEnrichmentModule } from './place-enrichment/place-enrichment.module';
import { PlaceShadowModule } from './place-shadow/place-shadow.module';
import { RouteUsageModule } from './route-usage/route-usage.module';
import { CategoriesModule } from './categories/categories.module';
import { TagsModule } from './tags/tags.module';
import { NotificationsModule } from './notifications/notifications.module';
@@ -78,7 +79,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
* migrated.
*/
@Module({
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
providers: [
// Default-deny: a route is authenticated unless it carries @Public() or
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
@@ -0,0 +1,43 @@
import { Body, Controller, Delete, Get, HttpCode, Post, UseGuards } from '@nestjs/common';
import type { RouteUsageReportResult, RouteUsageSummaryResult } from '@trek/shared';
import { AdminGuard } from '../auth/admin.guard';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { RouteUsageReportDto } from './route-usage.dto';
import { RouteUsageService } from './route-usage.service';
/**
* /api/route-usage — how much routing this instance does.
*
* Reporting is open to any signed-in user, because every signed-in user is the one
* generating the load; reading is admin only, because the totals are everybody's
* routing and the person entitled to see that is the person running the instance.
* Nothing here leaves the instance on its own.
*/
@Controller('api/route-usage')
@UseGuards(JwtAuthGuard)
export class RouteUsageController {
constructor(private readonly usage: RouteUsageService) {}
/**
* 200 with `{ recorded: false }` when counting is off, rather than a 403. The
* client flushes and moves on; a rejection would be a permanent error in the
* console of every install that switched the counters off.
*/
@Post('report')
@HttpCode(200)
report(@Body() body: RouteUsageReportDto): RouteUsageReportResult {
return { recorded: this.usage.record(body) };
}
@Get('summary')
@UseGuards(JwtAuthGuard, AdminGuard)
summary(): RouteUsageSummaryResult {
return this.usage.summary();
}
@Delete()
@UseGuards(JwtAuthGuard, AdminGuard)
clear(): { removed: number } {
return { removed: this.usage.clear() };
}
}
@@ -0,0 +1,9 @@
import { createZodDto } from 'nestjs-zod';
import { routeUsageReportRequestSchema } from '@trek/shared';
/**
* Zod-pipe wrapper for a batch of counters. Same pattern as every other write
* endpoint: the contract lives in @trek/shared, the class exists so a controller
* parameter can be typed with it.
*/
export class RouteUsageReportDto extends createZodDto(routeUsageReportRequestSchema) {}
@@ -0,0 +1,38 @@
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
import { logError, logInfo } from '../audit/audit-log.logger';
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
import { RouteUsageService } from './route-usage.service';
/**
* Nightly retention for the routing counters.
*
* Runs even when counting is switched off, on purpose: switching it off should also
* let what it already counted age out, rather than freezing the rows in place until
* somebody remembers to press the wipe button.
*
* 3:45 AM, just after the shadow log's own sweep and clear of the 3:00 idempotency
* purge and the backup window.
*/
@Injectable()
export class RouteUsageRetentionJob implements OnApplicationBootstrap {
constructor(
private readonly usage: RouteUsageService,
private readonly registrar: CronRegistrarService,
) {}
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
this.registrar.register('route-usage-retention', '45 3 * * *', () => this.tick());
}
tick(): void {
try {
const removed = this.usage.purgeExpired();
if (removed > 0) {
logInfo(`Route usage retention: removed ${removed} expired row(s)`);
}
} catch (err: unknown) {
logError(`Route usage retention: ${err instanceof Error ? err.message : err}`);
}
}
}
@@ -0,0 +1,17 @@
import { Module } from '@nestjs/common';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { RouteUsageController } from './route-usage.controller';
import { RouteUsageRetentionJob } from './route-usage.job';
import { RouteUsageService } from './route-usage.service';
/**
* Routing usage counters. Registered in AppModule; exports the service so an admin
* surface can read the summary without going through HTTP.
*/
@Module({
imports: [SchedulingModule],
controllers: [RouteUsageController],
providers: [RouteUsageService, RouteUsageRetentionJob],
exports: [RouteUsageService],
})
export class RouteUsageModule {}
@@ -0,0 +1,185 @@
import { Injectable } from '@nestjs/common';
import type {
RouteUsageDayRow,
RouteUsageProfile,
RouteUsageReportRequest,
RouteUsageSummaryResult,
RouteUsageSurface,
} from '@trek/shared';
import { DatabaseService } from '../database/database.service';
/** Days a counted day survives. Aggregates are tiny, so this is a year and a bit. */
export const RETENTION_DAYS = 400;
/** Days returned in the summary's own series, newest first. */
export const SERIES_DAYS = 90;
interface DbRow {
day: string;
profile: string;
surface: string;
self_hosted: number;
requests: number;
waypoints: number;
km: number;
failed: number;
}
/**
* Route usage counters.
*
* All routing happens in the browser, so the server never sees a route request and
* cannot count one. The client keeps a small tally and posts it in batches; this
* adds those batches onto the day's row.
*
* On by default, unlike the shadow log next door. That log stores what people typed,
* so silence is the safe default; this stores four integers about a day with nothing
* in them that could belong to anybody, and it is worthless unless it is already
* running by the time the question comes up. An operator who wants none of it sets
* `route_usage_enabled` to `false`.
*/
@Injectable()
export class RouteUsageService {
constructor(private readonly db: DatabaseService) {}
enabled(): boolean {
const row = this.db.get<{ value: string }>(
"SELECT value FROM app_settings WHERE key = 'route_usage_enabled'",
);
// Absent means on: the counters have to be collecting before anyone thinks to
// look for them, and there is nothing here to protect.
return row?.value !== 'false';
}
/**
* Adds a batch onto today's rows. Returns whether anything was written, so a
* switched-off instance answers 200 rather than an error the client would log on
* every flush.
*/
record(report: RouteUsageReportRequest): boolean {
if (!this.enabled()) return false;
// One transaction for the batch: a flush is a handful of rows, and a partial
// one would leave a day counted twice on the client's next retry.
this.db.transaction(() => {
for (const entry of report.entries) {
this.db.run(
`INSERT INTO route_usage_daily (day, profile, surface, self_hosted, requests, waypoints, km, failed)
VALUES (date('now'), ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(day, profile, surface, self_hosted) DO UPDATE SET
requests = requests + excluded.requests,
waypoints = waypoints + excluded.waypoints,
km = km + excluded.km,
failed = failed + excluded.failed`,
entry.profile,
entry.surface,
entry.selfHosted ? 1 : 0,
entry.requests,
entry.waypoints,
Math.round(entry.km),
entry.failed,
);
}
});
return true;
}
rows(): RouteUsageDayRow[] {
const rows = this.db.all<DbRow>(
'SELECT * FROM route_usage_daily ORDER BY day DESC, profile, surface',
);
return rows.map((r) => ({
day: r.day,
profile: r.profile as RouteUsageProfile,
surface: r.surface as RouteUsageSurface,
selfHosted: r.self_hosted === 1,
requests: r.requests,
waypoints: r.waypoints,
km: r.km,
failed: r.failed,
}));
}
summary(): RouteUsageSummaryResult {
const rows = this.rows();
const enabled = this.enabled();
const empty: RouteUsageSummaryResult = {
enabled,
retentionDays: RETENTION_DAYS,
daysCovered: 0,
firstDay: null,
lastDay: null,
totalRequests: 0,
totalFailed: 0,
requestsPerDay: 0,
busiestDay: null,
busiestDayRequests: 0,
waypointsPerRequest: 0,
kmPerRequest: 0,
byProfile: [],
bySurface: [],
selfHostedShare: 0,
days: [],
};
if (!rows.length) return empty;
const perDay = new Map<string, number>();
const perProfile = new Map<RouteUsageProfile, number>();
const perSurface = new Map<RouteUsageSurface, number>();
let requests = 0;
let failed = 0;
let waypoints = 0;
let km = 0;
let selfHosted = 0;
for (const row of rows) {
requests += row.requests;
failed += row.failed;
waypoints += row.waypoints;
km += row.km;
if (row.selfHosted) selfHosted += row.requests;
perDay.set(row.day, (perDay.get(row.day) ?? 0) + row.requests);
perProfile.set(row.profile, (perProfile.get(row.profile) ?? 0) + row.requests);
perSurface.set(row.surface, (perSurface.get(row.surface) ?? 0) + row.requests);
}
// Days come back newest first, so the last one is the earliest counted day.
const days = [...perDay.entries()].map(([day, count]) => ({ day, requests: count }));
const busiest = days.reduce((a, b) => (b.requests > a.requests ? b : a), days[0]);
const ratio = (part: number) => (requests > 0 ? Math.round((part / requests) * 1000) / 1000 : 0);
const per = (total: number) => (requests > 0 ? Math.round((total / requests) * 100) / 100 : 0);
return {
enabled,
retentionDays: RETENTION_DAYS,
daysCovered: perDay.size,
firstDay: rows[rows.length - 1].day,
lastDay: rows[0].day,
totalRequests: requests,
totalFailed: failed,
requestsPerDay: perDay.size > 0 ? Math.round((requests / perDay.size) * 10) / 10 : 0,
busiestDay: busiest.day,
busiestDayRequests: busiest.requests,
waypointsPerRequest: per(waypoints),
kmPerRequest: per(km),
byProfile: [...perProfile.entries()].map(([profile, count]) => ({ profile, requests: count })),
bySurface: [...perSurface.entries()].map(([surface, count]) => ({ surface, requests: count })),
selfHostedShare: ratio(selfHosted),
days: days.slice(0, SERIES_DAYS),
};
}
/** Removes days past the retention window. Returns how many rows went. */
purgeExpired(): number {
const result = this.db.run(
`DELETE FROM route_usage_daily WHERE day < date('now', ?)`,
`-${RETENTION_DAYS} days`,
);
return result.changes ?? 0;
}
/** Wipes every counter. The admin's own "start over". */
clear(): number {
const result = this.db.run('DELETE FROM route_usage_daily');
return result.changes ?? 0;
}
}