fix(tours): keep the Places layers off the search breaker and keep partial way answers

The ways and routes layers now count failures on a breaker of their own, so
busy or slow layer answers no longer send place search to the fallback for a
minute, and a layer's 404 no longer clears the count of search failures.

When a later batch of /v1/ways fails, the batches that answered are kept and
only the ids still open go to the Overpass mirrors.

The wiki, the compose file and the chart now say that the way ids of a planned
tour and the horse routes box go to the TREK Places API.

server/src/nest/maps/trek-places-layers.service.ts

@@ -6,7 +6,7 @@ import {
   type TrekPlacesMiss,
   type TrekPlacesRoutes,
   type TrekPlacesRouteType,
-  type TrekWayTags,
+  type TrekWaysAnswer,
 } from './trek-places.client';
 import { Injectable } from '@nestjs/common';
 
@@ -37,8 +37,16 @@ const WHY: Record<TrekPlacesMiss, string> = {
 export class TrekPlacesLayersService {
   private readonly told = new Map<Layer, TrekPlacesMiss>();
 
-  wayTags(ids: readonly number[]): Promise<TrekPlacesLayerRead<TrekWayTags[]>> {
-    return this.read('ways', () => trekPlacesWays(ids));
+  /**
+   * An answer that stopped part way is still an answer: the caller keeps what
+   * came and asks elsewhere for the rest. Why it stopped is said like a miss.
+   */
+  wayTags(ids: readonly number[]): Promise<TrekPlacesLayerRead<TrekWaysAnswer>> {
+    return this.read(
+      'ways',
+      () => trekPlacesWays(ids),
+      (answer) => (answer.complete ? null : { failure: answer.failure }),
+    );
   }
 
   routes(
@@ -49,10 +57,16 @@ export class TrekPlacesLayersService {
     return this.read('routes', () => trekPlacesRoutes(bbox, type, limits));
   }
 
-  private async read<T>(layer: Layer, ask: () => Promise<T>): Promise<TrekPlacesLayerRead<T>> {
+  private async read<T>(
+    layer: Layer,
+    ask: () => Promise<T>,
+    shortOf: (value: T) => { failure: unknown } | null = () => null,
+  ): Promise<TrekPlacesLayerRead<T>> {
     try {
       const value = await ask();
-      this.told.delete(layer);
+      const short = shortOf(value);
+      if (short) this.tell(layer, trekPlacesMiss(short.failure), short.failure);
+      else this.told.delete(layer);
       return { ok: true, value };
     } catch (err: unknown) {
       const miss = trekPlacesMiss(err);

server/src/nest/maps/trek-places.client.ts

@@ -138,8 +138,23 @@ function instanceToken(): string {
  */
 const BREAKER_FAILURES_BEFORE_OPEN = 4;
 const BREAKER_COOLDOWN_MS = 60_000;
-let breakerFailures = 0;
-let breakerOpenUntil = 0;
+
+interface Breaker {
+  failures: number;
+  openUntil: number;
+}
+
+/**
+ * Place search (autocomplete, full search, the category pills, the offline
+ * download, the lookup by id) shares one breaker. The OpenStreetMap layers
+ * Tours reads have their own: the service answers a heavy layer query with a
+ * 503 when it interrupts it, and a view of routes may run its full 20 s, so a
+ * few busy layer answers would otherwise send every keystroke to the fallback
+ * for a minute although the service is there. The other way round, a layer's
+ * 404 for "not built yet" would clear the count of real search failures.
+ */
+const searchBreaker: Breaker = { failures: 0, openUntil: 0 };
+const layersBreaker: Breaker = { failures: 0, openUntil: 0 };
 
 /**
  * Whether a failure says anything about reachability.
@@ -158,8 +173,10 @@ function countsAgainstReachability(err: unknown): boolean {
 
 /** Test seam: the breaker is process state, so a suite has to be able to clear it. */
 export function resetTrekPlacesBreaker(): void {
-  breakerFailures = 0;
-  breakerOpenUntil = 0;
+  for (const breaker of [searchBreaker, layersBreaker]) {
+    breaker.failures = 0;
+    breaker.openUntil = 0;
+  }
 }
 
 const SWITCHED_OFF_MESSAGE = 'TREK Places API is switched off on this instance';
@@ -186,8 +203,9 @@ export function trekPlacesBaseUrl(): string {
 async function getJson<T>(
   path: string,
   params: Record<string, string | number | undefined>,
-  opts: { maxBytes?: number; timeoutMs?: number } = {},
+  opts: { maxBytes?: number; timeoutMs?: number; breaker?: Breaker } = {},
 ): Promise<T> {
+  const breaker = opts.breaker ?? searchBreaker;
   // TREK_PLACES_ENABLED=false is the operator's word that nothing leaves for
   // the index. Every caller checks it before asking, and this is the one place
   // every request passes, so a caller that did not is still stopped here. Ahead
@@ -201,7 +219,7 @@ async function getJson<T>(
     if (v !== undefined && v !== null && v !== '') url.searchParams.set(k, String(v));
   }
 
-  if (Date.now() < breakerOpenUntil) {
+  if (Date.now() < breaker.openUntil) {
     throw new Error('TREK Places API unreachable, not retrying yet');
   }
 
@@ -225,21 +243,21 @@ async function getJson<T>(
     const text = await readCapped(res, opts.maxBytes ?? MAX_BYTES);
     const parsed = JSON.parse(text) as T;
     // An answer of any kind means the service is there.
-    breakerFailures = 0;
+    breaker.failures = 0;
     return parsed;
   } catch (err: unknown) {
     if (countsAgainstReachability(err)) {
-      breakerFailures += 1;
-      if (breakerFailures >= BREAKER_FAILURES_BEFORE_OPEN) {
-        breakerOpenUntil = Date.now() + BREAKER_COOLDOWN_MS;
+      breaker.failures += 1;
+      if (breaker.failures >= BREAKER_FAILURES_BEFORE_OPEN) {
+        breaker.openUntil = Date.now() + BREAKER_COOLDOWN_MS;
         // Back to the threshold rather than to zero: when the cooldown ends,
         // one more failure re-opens the breaker instead of buying another four
         // full timeouts.
-        breakerFailures = BREAKER_FAILURES_BEFORE_OPEN - 1;
+        breaker.failures = BREAKER_FAILURES_BEFORE_OPEN - 1;
       }
     } else {
       // The service answered. A 404 is an answer.
-      breakerFailures = 0;
+      breaker.failures = 0;
     }
     throw err;
   } finally {
@@ -744,7 +762,11 @@ const elementsAnswerSchema = z.object({ elements: z.array(z.unknown()) });
  * having no grade.
  */
 async function trekPlacesWaysPart(ids: readonly number[]): Promise<TrekWayTags[]> {
-  const body = await getJson<unknown>('/v1/ways', { ids: ids.join(',') }, { timeoutMs: WAYS_TIMEOUT_MS });
+  const body = await getJson<unknown>(
+    '/v1/ways',
+    { ids: ids.join(',') },
+    { timeoutMs: WAYS_TIMEOUT_MS, breaker: layersBreaker },
+  );
   const answer = elementsAnswerSchema.safeParse(body);
   if (!answer.success) throw new Error('TREK Places API answered /v1/ways without elements');
   const asked = new Set(ids);
@@ -752,19 +774,36 @@ async function trekPlacesWaysPart(ids: readonly number[]): Promise<TrekWayTags[]
 }
 
 /**
- * The tags of OpenStreetMap ways by id, from the ways layer. Asked in batches
- * of WAYS_MAX_IDS, one after another; the first batch that fails fails the
- * whole call, so the caller never mistakes half an answer for all of it.
+ * What the ways layer answered for a tour: every batch (`complete`), or the
+ * batches before the one that failed, with the ids that batch and the ones
+ * after it held, and why it failed. The caller asks another source for those
+ * ids only, never again for the ways already answered.
  */
-export async function trekPlacesWays(ids: readonly number[]): Promise<TrekWayTags[]> {
+export type TrekWaysAnswer =
+  | { complete: true; ways: TrekWayTags[] }
+  | { complete: false; ways: TrekWayTags[]; unanswered: number[]; failure: unknown };
+
+/**
+ * The tags of OpenStreetMap ways by id, from the ways layer. Asked in batches
+ * of WAYS_MAX_IDS, one after another. When the first batch fails the call
+ * throws, as the layer gave nothing. When a later one fails, nothing more is
+ * asked (a service that just failed would mostly cost one more deadline per
+ * batch) and the answer says which ids are still open.
+ */
+export async function trekPlacesWays(ids: readonly number[]): Promise<TrekWaysAnswer> {
   const wanted = [...new Set(ids)].filter((id) => osmIdSchema.safeParse(id).success);
-  const parts = Array.from({ length: Math.ceil(wanted.length / WAYS_MAX_IDS) }, (_, index) =>
-    wanted.slice(index * WAYS_MAX_IDS, (index + 1) * WAYS_MAX_IDS),
-  );
-  return parts.reduce<Promise<TrekWayTags[]>>(
-    async (done, part) => [...(await done), ...(await trekPlacesWaysPart(part))],
-    Promise.resolve([]),
-  );
+  const starts = Array.from({ length: Math.ceil(wanted.length / WAYS_MAX_IDS) }, (_, index) => index * WAYS_MAX_IDS);
+  return starts.reduce<Promise<TrekWaysAnswer>>(async (previous, start) => {
+    const done = await previous;
+    if (!done.complete) return done;
+    try {
+      const part = await trekPlacesWaysPart(wanted.slice(start, start + WAYS_MAX_IDS));
+      return { complete: true, ways: [...done.ways, ...part] };
+    } catch (failure: unknown) {
+      if (start === 0) throw failure;
+      return { complete: false, ways: done.ways, unanswered: wanted.slice(start), failure };
+    }
+  }, Promise.resolve<TrekWaysAnswer>({ complete: true, ways: [] }));
 }
 
 /**
@@ -786,7 +825,7 @@ export async function trekPlacesRoutes(
       limitRelations: clampedInt(limits.relations, 1, ROUTES_MAX_RELATIONS),
       limitWays: clampedInt(limits.ways, 1, ROUTES_MAX_WAYS),
     },
-    { maxBytes: ROUTES_MAX_BYTES, timeoutMs: ROUTES_TIMEOUT_MS },
+    { maxBytes: ROUTES_MAX_BYTES, timeoutMs: ROUTES_TIMEOUT_MS, breaker: layersBreaker },
   );
   const answer = elementsAnswerSchema.extend({ truncated: z.unknown() }).safeParse(body);
   if (!answer.success) throw new Error('TREK Places API answered /v1/routes without elements');

server/src/nest/tours/tour-trail-grade.service.ts

@@ -104,17 +104,20 @@ export class TourTrailGradeService {
 
   /**
    * Reads the grades of the ways not kept yet into what is kept: from the ways
-   * layer in one go, else from the mirrors part by part. Gives back the ways
-   * no source answered.
+   * layer first, then from the mirrors part by part for the ways the layer
+   * did not answer (all of them when it gave nothing, only the rest when it
+   * stopped part way). Gives back the ways no source answered.
    */
   private async read(ids: number[], now: number): Promise<number[]> {
     const layer = await this.layers.wayTags(ids);
+    let rest = ids;
     if (layer.ok) {
-      this.keepAll(ids, layer.value, now);
-      return [];
+      const open = new Set(layer.value.complete ? [] : layer.value.unanswered);
+      this.keepAll(ids.filter((id) => !open.has(id)), layer.value.ways, now);
+      rest = ids.filter((id) => open.has(id));
     }
-    const parts = Array.from({ length: Math.ceil(ids.length / WAYS_PER_QUERY) }, (_, index) =>
-      ids.slice(index * WAYS_PER_QUERY, (index + 1) * WAYS_PER_QUERY),
+    const parts = Array.from({ length: Math.ceil(rest.length / WAYS_PER_QUERY) }, (_, index) =>
+      rest.slice(index * WAYS_PER_QUERY, (index + 1) * WAYS_PER_QUERY),
     );
     // One question after another, so a long tour does not flood the shared mirrors.
     return parts.reduce<Promise<number[]>>(async (done, part) => {