diff --git a/charts/trek/values.yaml b/charts/trek/values.yaml index 4286d2cc1..473cb8153 100644 --- a/charts/trek/values.yaml +++ b/charts/trek/values.yaml @@ -102,8 +102,8 @@ env: # lower value hangs up on answers that were still coming. Raise it for a slow self-hosted Overpass instance. # Defaults to 25000. # TREK_PLACES_ENABLED: "false" - # Place search (suggestions, full search, category buttons, Road trip search, import geocoding, offline downloads) - # asks the TREK Places API at https://places.liketrek.com by default, so the pod makes outbound requests to that + # Place search (suggestions, full search, category buttons, Road trip search, import geocoding, offline downloads, + # and the OpenStreetMap ways and horse routes Tours reads) asks the TREK Places API at https://places.liketrek.com by default, so the pod makes outbound requests to that # host. Set "false" to search the way it did before 4.3.0 (OpenStreetMap, or your Google/Amap key) and send # nothing there. There is no switch in the admin panel on purpose: whether searches leave the instance is decided # here. Defaults to on. diff --git a/docker-compose.yml b/docker-compose.yml index 73bbd6939..c9a1c598a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: # - LLM_MAX_OUTPUT_TOKENS=8192 # Optional. Longest answer (tokens) one AI-parsing call may ask for. Raise it when a document with many bookings comes back cut off; keep prompt plus cap inside your model's context. Whole number from 256 to 1000000. (default: 4096 for OpenAI-compatible, 8192 for Anthropic) # - TREK_PLUGINS_IGNORE_TREK_RANGE=true # Optional. Lets plugins whose declared TREK version range predates this build, and plugins that declare none, be installed and activated anyway. Every gate warns instead of refusing; the plugin-API version gate and the Discover compatibility verdict stay hard. Off unless set. (default: off) # - UNSPLASH_ACCESS_KEY= # Optional. Unsplash Access Key for trip-cover / place-image search. Without one, TREK uses Unsplash's unauthenticated endpoint, which some datacenter/VPS IPs are blocked from (#1449). Free key at unsplash.com/developers. Can also be set per-admin in Admin > Settings; the env var overrides that. -# - TREK_PLACES_ENABLED=false # Optional. Place search (suggestions, full search, category buttons, Road trip search, import geocoding, offline downloads) asks the TREK Places API at https://places.liketrek.com by default, so the server makes outbound requests to that host. Set false to search the way it did before 4.3.0 (OpenStreetMap, or your Google/Amap key) and send nothing there. No switch in the admin panel on purpose: whether searches leave the instance is decided here. (default: true) +# - TREK_PLACES_ENABLED=false # Optional. Place search (suggestions, full search, category buttons, Road trip search, import geocoding, offline downloads, and the OpenStreetMap ways and horse routes Tours reads) asks the TREK Places API at https://places.liketrek.com by default, so the server makes outbound requests to that host. Set false to search the way it did before 4.3.0 (OpenStreetMap, or your Google/Amap key) and send nothing there. No switch in the admin panel on purpose: whether searches leave the instance is decided here. (default: true) # - TREK_PLACES_URL=https://places.liketrek.com # Optional. Base URL of a copy of the Places API you run yourself; it has to answer the same /v1 API. Read from configuration rather than typed by a user, so an address on your LAN or Docker network works without ALLOW_INTERNAL_NETWORK. (default: https://places.liketrek.com) # - PLACES_API_KEY= # Optional. Google Maps API key (Places API (New) enabled) for place search, photos, details and Google transit. Overrides the key set in Admin > Settings > API Keys, whose field then shows this variable read-only. # - AMAP_API_KEY= # Optional. Amap (高德地图) Web 服务 key for place search and geocoding inside mainland China. Overrides the key set in Admin > Settings > API Keys. A Web 端 (JS API) key is a different credential and is rejected with INVALID_USER_KEY. diff --git a/server/src/nest/maps/trek-places-layers.service.ts b/server/src/nest/maps/trek-places-layers.service.ts index 2f78e2de8..6c38d4b27 100644 --- a/server/src/nest/maps/trek-places-layers.service.ts +++ b/server/src/nest/maps/trek-places-layers.service.ts @@ -6,7 +6,7 @@ import { type TrekPlacesMiss, type TrekPlacesRoutes, type TrekPlacesRouteType, - type TrekWayTags, + type TrekWaysAnswer, } from './trek-places.client'; import { Injectable } from '@nestjs/common'; @@ -37,8 +37,16 @@ const WHY: Record = { export class TrekPlacesLayersService { private readonly told = new Map(); - wayTags(ids: readonly number[]): Promise> { - return this.read('ways', () => trekPlacesWays(ids)); + /** + * An answer that stopped part way is still an answer: the caller keeps what + * came and asks elsewhere for the rest. Why it stopped is said like a miss. + */ + wayTags(ids: readonly number[]): Promise> { + return this.read( + 'ways', + () => trekPlacesWays(ids), + (answer) => (answer.complete ? null : { failure: answer.failure }), + ); } routes( @@ -49,10 +57,16 @@ export class TrekPlacesLayersService { return this.read('routes', () => trekPlacesRoutes(bbox, type, limits)); } - private async read(layer: Layer, ask: () => Promise): Promise> { + private async read( + layer: Layer, + ask: () => Promise, + shortOf: (value: T) => { failure: unknown } | null = () => null, + ): Promise> { try { const value = await ask(); - this.told.delete(layer); + const short = shortOf(value); + if (short) this.tell(layer, trekPlacesMiss(short.failure), short.failure); + else this.told.delete(layer); return { ok: true, value }; } catch (err: unknown) { const miss = trekPlacesMiss(err); diff --git a/server/src/nest/maps/trek-places.client.ts b/server/src/nest/maps/trek-places.client.ts index a029b86e1..254bec585 100644 --- a/server/src/nest/maps/trek-places.client.ts +++ b/server/src/nest/maps/trek-places.client.ts @@ -138,8 +138,23 @@ function instanceToken(): string { */ const BREAKER_FAILURES_BEFORE_OPEN = 4; const BREAKER_COOLDOWN_MS = 60_000; -let breakerFailures = 0; -let breakerOpenUntil = 0; + +interface Breaker { + failures: number; + openUntil: number; +} + +/** + * Place search (autocomplete, full search, the category pills, the offline + * download, the lookup by id) shares one breaker. The OpenStreetMap layers + * Tours reads have their own: the service answers a heavy layer query with a + * 503 when it interrupts it, and a view of routes may run its full 20 s, so a + * few busy layer answers would otherwise send every keystroke to the fallback + * for a minute although the service is there. The other way round, a layer's + * 404 for "not built yet" would clear the count of real search failures. + */ +const searchBreaker: Breaker = { failures: 0, openUntil: 0 }; +const layersBreaker: Breaker = { failures: 0, openUntil: 0 }; /** * Whether a failure says anything about reachability. @@ -158,8 +173,10 @@ function countsAgainstReachability(err: unknown): boolean { /** Test seam: the breaker is process state, so a suite has to be able to clear it. */ export function resetTrekPlacesBreaker(): void { - breakerFailures = 0; - breakerOpenUntil = 0; + for (const breaker of [searchBreaker, layersBreaker]) { + breaker.failures = 0; + breaker.openUntil = 0; + } } const SWITCHED_OFF_MESSAGE = 'TREK Places API is switched off on this instance'; @@ -186,8 +203,9 @@ export function trekPlacesBaseUrl(): string { async function getJson( path: string, params: Record, - opts: { maxBytes?: number; timeoutMs?: number } = {}, + opts: { maxBytes?: number; timeoutMs?: number; breaker?: Breaker } = {}, ): Promise { + const breaker = opts.breaker ?? searchBreaker; // TREK_PLACES_ENABLED=false is the operator's word that nothing leaves for // the index. Every caller checks it before asking, and this is the one place // every request passes, so a caller that did not is still stopped here. Ahead @@ -201,7 +219,7 @@ async function getJson( if (v !== undefined && v !== null && v !== '') url.searchParams.set(k, String(v)); } - if (Date.now() < breakerOpenUntil) { + if (Date.now() < breaker.openUntil) { throw new Error('TREK Places API unreachable, not retrying yet'); } @@ -225,21 +243,21 @@ async function getJson( const text = await readCapped(res, opts.maxBytes ?? MAX_BYTES); const parsed = JSON.parse(text) as T; // An answer of any kind means the service is there. - breakerFailures = 0; + breaker.failures = 0; return parsed; } catch (err: unknown) { if (countsAgainstReachability(err)) { - breakerFailures += 1; - if (breakerFailures >= BREAKER_FAILURES_BEFORE_OPEN) { - breakerOpenUntil = Date.now() + BREAKER_COOLDOWN_MS; + breaker.failures += 1; + if (breaker.failures >= BREAKER_FAILURES_BEFORE_OPEN) { + breaker.openUntil = Date.now() + BREAKER_COOLDOWN_MS; // Back to the threshold rather than to zero: when the cooldown ends, // one more failure re-opens the breaker instead of buying another four // full timeouts. - breakerFailures = BREAKER_FAILURES_BEFORE_OPEN - 1; + breaker.failures = BREAKER_FAILURES_BEFORE_OPEN - 1; } } else { // The service answered. A 404 is an answer. - breakerFailures = 0; + breaker.failures = 0; } throw err; } finally { @@ -744,7 +762,11 @@ const elementsAnswerSchema = z.object({ elements: z.array(z.unknown()) }); * having no grade. */ async function trekPlacesWaysPart(ids: readonly number[]): Promise { - const body = await getJson('/v1/ways', { ids: ids.join(',') }, { timeoutMs: WAYS_TIMEOUT_MS }); + const body = await getJson( + '/v1/ways', + { ids: ids.join(',') }, + { timeoutMs: WAYS_TIMEOUT_MS, breaker: layersBreaker }, + ); const answer = elementsAnswerSchema.safeParse(body); if (!answer.success) throw new Error('TREK Places API answered /v1/ways without elements'); const asked = new Set(ids); @@ -752,19 +774,36 @@ async function trekPlacesWaysPart(ids: readonly number[]): Promise { +export type TrekWaysAnswer = + | { complete: true; ways: TrekWayTags[] } + | { complete: false; ways: TrekWayTags[]; unanswered: number[]; failure: unknown }; + +/** + * The tags of OpenStreetMap ways by id, from the ways layer. Asked in batches + * of WAYS_MAX_IDS, one after another. When the first batch fails the call + * throws, as the layer gave nothing. When a later one fails, nothing more is + * asked (a service that just failed would mostly cost one more deadline per + * batch) and the answer says which ids are still open. + */ +export async function trekPlacesWays(ids: readonly number[]): Promise { const wanted = [...new Set(ids)].filter((id) => osmIdSchema.safeParse(id).success); - const parts = Array.from({ length: Math.ceil(wanted.length / WAYS_MAX_IDS) }, (_, index) => - wanted.slice(index * WAYS_MAX_IDS, (index + 1) * WAYS_MAX_IDS), - ); - return parts.reduce>( - async (done, part) => [...(await done), ...(await trekPlacesWaysPart(part))], - Promise.resolve([]), - ); + const starts = Array.from({ length: Math.ceil(wanted.length / WAYS_MAX_IDS) }, (_, index) => index * WAYS_MAX_IDS); + return starts.reduce>(async (previous, start) => { + const done = await previous; + if (!done.complete) return done; + try { + const part = await trekPlacesWaysPart(wanted.slice(start, start + WAYS_MAX_IDS)); + return { complete: true, ways: [...done.ways, ...part] }; + } catch (failure: unknown) { + if (start === 0) throw failure; + return { complete: false, ways: done.ways, unanswered: wanted.slice(start), failure }; + } + }, Promise.resolve({ complete: true, ways: [] })); } /** @@ -786,7 +825,7 @@ export async function trekPlacesRoutes( limitRelations: clampedInt(limits.relations, 1, ROUTES_MAX_RELATIONS), limitWays: clampedInt(limits.ways, 1, ROUTES_MAX_WAYS), }, - { maxBytes: ROUTES_MAX_BYTES, timeoutMs: ROUTES_TIMEOUT_MS }, + { maxBytes: ROUTES_MAX_BYTES, timeoutMs: ROUTES_TIMEOUT_MS, breaker: layersBreaker }, ); const answer = elementsAnswerSchema.extend({ truncated: z.unknown() }).safeParse(body); if (!answer.success) throw new Error('TREK Places API answered /v1/routes without elements'); diff --git a/server/src/nest/tours/tour-trail-grade.service.ts b/server/src/nest/tours/tour-trail-grade.service.ts index a171ad4d7..e9ea29aec 100644 --- a/server/src/nest/tours/tour-trail-grade.service.ts +++ b/server/src/nest/tours/tour-trail-grade.service.ts @@ -104,17 +104,20 @@ export class TourTrailGradeService { /** * Reads the grades of the ways not kept yet into what is kept: from the ways - * layer in one go, else from the mirrors part by part. Gives back the ways - * no source answered. + * layer first, then from the mirrors part by part for the ways the layer + * did not answer (all of them when it gave nothing, only the rest when it + * stopped part way). Gives back the ways no source answered. */ private async read(ids: number[], now: number): Promise { const layer = await this.layers.wayTags(ids); + let rest = ids; if (layer.ok) { - this.keepAll(ids, layer.value, now); - return []; + const open = new Set(layer.value.complete ? [] : layer.value.unanswered); + this.keepAll(ids.filter((id) => !open.has(id)), layer.value.ways, now); + rest = ids.filter((id) => open.has(id)); } - const parts = Array.from({ length: Math.ceil(ids.length / WAYS_PER_QUERY) }, (_, index) => - ids.slice(index * WAYS_PER_QUERY, (index + 1) * WAYS_PER_QUERY), + const parts = Array.from({ length: Math.ceil(rest.length / WAYS_PER_QUERY) }, (_, index) => + rest.slice(index * WAYS_PER_QUERY, (index + 1) * WAYS_PER_QUERY), ); // One question after another, so a long tour does not flood the shared mirrors. return parts.reduce>(async (done, part) => { diff --git a/server/tests/e2e/tours-bike.e2e.test.ts b/server/tests/e2e/tours-bike.e2e.test.ts index 947c31fc9..3956634ab 100644 --- a/server/tests/e2e/tours-bike.e2e.test.ts +++ b/server/tests/e2e/tours-bike.e2e.test.ts @@ -325,7 +325,10 @@ describe('Bike tours e2e (real guards + temp SQLite)', () => { overpassQuery.mockReset(); placesWayTags.mockResolvedValueOnce({ ok: true, - value: [{ type: 'way', id: 21, tags: { 'mtb:scale': '1', surface: 'gravel', highway: 'track' } }], + value: { + complete: true, + ways: [{ type: 'way', id: 21, tags: { 'mtb:scale': '1', surface: 'gravel', highway: 'track' } }], + }, }); const res = await ask({ way_ids: [21, 22] }, VIEWER); expect(res.status).toBe(200); diff --git a/server/tests/unit/nest/tour-trail-grade.service.test.ts b/server/tests/unit/nest/tour-trail-grade.service.test.ts index 9e47632fc..1abbb401e 100644 --- a/server/tests/unit/nest/tour-trail-grade.service.test.ts +++ b/server/tests/unit/nest/tour-trail-grade.service.test.ts @@ -5,7 +5,7 @@ import { Test } from '@nestjs/testing'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -// TOUR-GRADE-001 to TOUR-GRADE-016 +// TOUR-GRADE-001 to TOUR-GRADE-017 const query = vi.fn<(q: string, caller: string) => Promise>(); const wayTags = vi.fn(); @@ -158,10 +158,13 @@ describe('TourTrailGradeService (BIK-05)', () => { it('TOUR-GRADE-011: reads the ways from the TREK Places ways layer first and asks no mirror', async () => { wayTags.mockResolvedValue({ ok: true, - value: [ - { type: 'way', id: 1, tags: { 'mtb:scale': '2', sac_scale: 'hiking', surface: 'gravel', highway: 'path' } }, - { type: 'way', id: 2, tags: { highway: 'track' } }, - ], + value: { + complete: true, + ways: [ + { type: 'way', id: 1, tags: { 'mtb:scale': '2', sac_scale: 'hiking', surface: 'gravel', highway: 'path' } }, + { type: 'way', id: 2, tags: { highway: 'track' } }, + ], + }, }); const answer = await service.grades({ way_ids: [1, 2, 3, 1] }); expect(wayTags).toHaveBeenCalledWith([1, 2, 3]); @@ -184,7 +187,10 @@ describe('TourTrailGradeService (BIK-05)', () => { }); it('TOUR-GRADE-012: hands a long tour to the layer whole, which asks it in batches itself', async () => { - wayTags.mockResolvedValue({ ok: true, value: [{ type: 'way', id: 900, tags: { 'mtb:scale': '5' } }] }); + wayTags.mockResolvedValue({ + ok: true, + value: { complete: true, ways: [{ type: 'way', id: 900, tags: { 'mtb:scale': '5' } }] }, + }); const ids = Array.from({ length: 900 }, (_, index) => index + 1); const answer = await service.grades({ way_ids: ids }); expect(wayTags).toHaveBeenCalledTimes(1); @@ -215,7 +221,10 @@ describe('TourTrailGradeService (BIK-05)', () => { it('TOUR-GRADE-015: keeps what the layer answered a day, a way it left out as having no grade', async () => { vi.useFakeTimers(); try { - wayTags.mockResolvedValue({ ok: true, value: [{ type: 'way', id: 1, tags: { 'mtb:scale': '3' } }] }); + wayTags.mockResolvedValue({ + ok: true, + value: { complete: true, ways: [{ type: 'way', id: 1, tags: { 'mtb:scale': '3' } }] }, + }); expect(await service.grades({ way_ids: [1, 2] })).toMatchObject({ unanswered: [] }); expect(await service.grades({ way_ids: [2, 1] })).toMatchObject({ ways: [{ id: 1 }], unanswered: [] }); expect(wayTags).toHaveBeenCalledTimes(1); @@ -232,8 +241,36 @@ describe('TourTrailGradeService (BIK-05)', () => { wayTags.mockResolvedValue({ ok: true, // Typed rows from the client, but read through the same check as the mirrors' answer. - value: [{ type: 'way', id: 0, tags: { 'mtb:scale': '1' } }], + value: { complete: true, ways: [{ type: 'way', id: 0, tags: { 'mtb:scale': '1' } }] }, }); expect(await service.grades({ way_ids: [1] })).toEqual({ ways: [], unanswered: [] }); }); + + it('TOUR-GRADE-017: a layer that stopped part way keeps what it answered and asks the mirrors only for the rest', async () => { + const ids = Array.from({ length: 1000 }, (_, index) => index + 1); + const rest = ids.slice(500); + wayTags.mockResolvedValue({ + ok: true, + value: { + complete: false, + ways: [{ type: 'way', id: 7, tags: { 'mtb:scale': '2' } }], + unanswered: rest, + failure: new Error('TREK Places API 503'), + }, + }); + query.mockResolvedValueOnce([way(600, { surface: 'gravel' })]).mockRejectedValueOnce(new Error('all mirrors down')); + const answer = await service.grades({ way_ids: ids }); + // 500 ways left in parts of 400: two questions, none of them naming a way the layer answered. + expect(query).toHaveBeenCalledTimes(2); + expect(query.mock.calls[0]?.[0]).toBe(`[out:json][timeout:25];way(id:${rest.slice(0, 400).join(',')});out tags;`); + expect(query.mock.calls[1]?.[0]).toBe(`[out:json][timeout:25];way(id:${rest.slice(400).join(',')});out tags;`); + expect(answer.ways.map((grade) => grade.id)).toEqual([7, 600]); + expect(answer.unanswered).toEqual(rest.slice(400)); + // The ways the layer answered are kept: asked again, nothing goes out for them. + query.mockClear(); + wayTags.mockClear(); + await service.grades({ way_ids: ids.slice(0, 500) }); + expect(wayTags).not.toHaveBeenCalled(); + expect(query).not.toHaveBeenCalled(); + }); }); diff --git a/server/tests/unit/nest/trek-places-layers.service.test.ts b/server/tests/unit/nest/trek-places-layers.service.test.ts index 006aa7315..2a53a948c 100644 --- a/server/tests/unit/nest/trek-places-layers.service.test.ts +++ b/server/tests/unit/nest/trek-places-layers.service.test.ts @@ -11,7 +11,7 @@ import { Test } from '@nestjs/testing'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -// TREK-LAYERS-001 to TREK-LAYERS-006 +// TREK-LAYERS-001 to TREK-LAYERS-007 vi.mock('../../../src/nest/audit/audit-log.logger', () => ({ LOG_LEVEL: 'error', @@ -54,11 +54,11 @@ beforeEach(async () => { describe('TrekPlacesLayersService', () => { it('TREK-LAYERS-001: hands on what a layer answered, asked as the caller asked', async () => { - ways.mockResolvedValue([{ type: 'way', id: 1, tags: { highway: 'path' } }]); + ways.mockResolvedValue({ complete: true, ways: [{ type: 'way', id: 1, tags: { highway: 'path' } }] }); routes.mockResolvedValue({ elements: [], truncated: true }); expect(await service.wayTags([1, 2])).toEqual({ ok: true, - value: [{ type: 'way', id: 1, tags: { highway: 'path' } }], + value: { complete: true, ways: [{ type: 'way', id: 1, tags: { highway: 'path' } }] }, }); expect(await service.routes(BOX, 'horse', CAPS)).toEqual({ ok: true, value: { elements: [], truncated: true } }); expect(ways).toHaveBeenCalledWith([1, 2]); @@ -103,9 +103,12 @@ describe('TrekPlacesLayersService', () => { }); it('TREK-LAYERS-005: after a layer answered, the same failure is said again', async () => { - ways.mockRejectedValueOnce(down()).mockResolvedValueOnce([]).mockRejectedValueOnce(down()); + ways + .mockRejectedValueOnce(down()) + .mockResolvedValueOnce({ complete: true, ways: [] }) + .mockRejectedValueOnce(down()); await service.wayTags([1]); - expect(await service.wayTags([1])).toEqual({ ok: true, value: [] }); + expect(await service.wayTags([1])).toEqual({ ok: true, value: { complete: true, ways: [] } }); await service.wayTags([1]); expect(warn).toHaveBeenCalledTimes(2); }); @@ -119,4 +122,25 @@ describe('TrekPlacesLayersService', () => { await service.routes(BOX, 'horse', CAPS); expect(info.mock.calls.map(([line]) => line.includes('/v1/ways'))).toEqual([true, false]); }); + + it('TREK-LAYERS-007: an answer that stopped part way is handed on, and why it stopped is said once', async () => { + const partial = { + complete: false as const, + ways: [{ type: 'way' as const, id: 1, tags: {} }], + unanswered: [501, 502], + failure: down(), + }; + ways.mockResolvedValue(partial); + expect(await service.wayTags([1, 501, 502])).toEqual({ ok: true, value: partial }); + await service.wayTags([1, 501, 502]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain('/v1/ways'); + expect(warn.mock.calls[0]?.[0]).toContain('TREK Places API 503'); + // A whole answer in between clears it, so the next stop is said again. + ways.mockResolvedValueOnce({ complete: true, ways: [] }); + await service.wayTags([1]); + await service.wayTags([1, 501, 502]); + expect(warn).toHaveBeenCalledTimes(2); + expect(info).not.toHaveBeenCalled(); + }); }); diff --git a/server/tests/unit/nest/trek-places.client.test.ts b/server/tests/unit/nest/trek-places.client.test.ts index 5f46ee948..d1d6cc815 100644 --- a/server/tests/unit/nest/trek-places.client.test.ts +++ b/server/tests/unit/nest/trek-places.client.test.ts @@ -878,7 +878,7 @@ describe('indexRowTerm (#2663)', () => { }); }); -// TREK-PLACES-WAYS-01 to 05, TREK-PLACES-ROUTES-01 to 05, TREK-PLACES-MISS-01 +// TREK-PLACES-WAYS-01 to 06, TREK-PLACES-ROUTES-01 to 05, TREK-PLACES-MISS-01, TREK-PLACES-BREAKER-01 to 03 /** A fetch answering each call with the next of `answers`, the last one again once they run out. */ function stubFetchSequence(answers: { body: unknown; status?: number }[]) { @@ -911,17 +911,20 @@ describe('trekPlacesWays', () => { missing: [13], keys: ['highway', 'mtb:scale'], }); - const ways = await trekPlacesWays([11, 12, 13]); + const answer = await trekPlacesWays([11, 12, 13]); const url = new URL(calls[0].url); expect(url.pathname).toBe('/v1/ways'); expect(url.searchParams.get('ids')).toBe('11,12,13'); expect((calls[0].init?.headers as Record)['X-TREK-Instance']).toMatch(/^trek-/); expect(calls[0].init?.signal).toBeInstanceOf(AbortSignal); // A way without tags reads as one without tags, never as a malformed row. - expect(ways).toEqual([ - { type: 'way', id: 11, tags: { 'mtb:scale': '2', highway: 'path' } }, - { type: 'way', id: 12, tags: {} }, - ]); + expect(answer).toEqual({ + complete: true, + ways: [ + { type: 'way', id: 11, tags: { 'mtb:scale': '2', highway: 'path' } }, + { type: 'way', id: 12, tags: {} }, + ], + }); }); it('TREK-PLACES-WAYS-02: asks in batches of 500, one after another, each id once, and nothing for no id', async () => { @@ -932,8 +935,8 @@ describe('trekPlacesWays', () => { expect(askedIds(calls[0])?.startsWith('1,2,3,')).toBe(true); expect(askedIds(calls[2])?.endsWith(',1200')).toBe(true); calls = []; - await expect(trekPlacesWays([])).resolves.toEqual([]); - await expect(trekPlacesWays([0, -1])).resolves.toEqual([]); + await expect(trekPlacesWays([])).resolves.toEqual({ complete: true, ways: [] }); + await expect(trekPlacesWays([0, -1])).resolves.toEqual({ complete: true, ways: [] }); expect(calls).toHaveLength(0); }); @@ -947,7 +950,7 @@ describe('trekPlacesWays', () => { 'row', ], }); - expect(await trekPlacesWays([1, 2, 3])).toEqual([{ type: 'way', id: 1, tags: { surface: 'gravel' } }]); + expect((await trekPlacesWays([1, 2, 3])).ways).toEqual([{ type: 'way', id: 1, tags: { surface: 'gravel' } }]); }); it('TREK-PLACES-WAYS-04: an answer without elements fails rather than reading as ways without tags', async () => { @@ -957,12 +960,27 @@ describe('trekPlacesWays', () => { await expect(trekPlacesWays([1])).rejects.toThrow('without elements'); }); - it('TREK-PLACES-WAYS-05: a batch that fails fails the call and asks no further, and the off switch asks nothing', async () => { - stubFetchSequence([{ body: { elements: [] } }, { body: {}, status: 503 }, { body: { elements: [] } }]); + it('TREK-PLACES-WAYS-05: a later batch that fails stops there and names the ids still open', async () => { + stubFetchSequence([ + { body: { elements: [{ type: 'way', id: 2, tags: { surface: 'gravel' } }] } }, + { body: { detail: 'busy' }, status: 503 }, + { body: { elements: [] } }, + ]); + const ids = Array.from({ length: 1400 }, (_, index) => index + 1); + const answer = await trekPlacesWays(ids); + expect(calls).toHaveLength(2); + if (answer.complete) throw new Error('expected an answer that stopped part way'); + expect(answer.ways).toEqual([{ type: 'way', id: 2, tags: { surface: 'gravel' } }]); + expect(answer.unanswered).toEqual(ids.slice(500)); + expect(trekPlacesMiss(answer.failure)).toBe('failed'); + }); + + it('TREK-PLACES-WAYS-06: a first batch that fails fails the call and asks no further, and the off switch asks nothing', async () => { + stubFetchSequence([{ body: {}, status: 503 }, { body: { elements: [] } }]); const ids = Array.from({ length: 1400 }, (_, index) => index + 1); const failure = await trekPlacesWays(ids).catch((err: unknown) => err); expect(trekPlacesMiss(failure)).toBe('failed'); - expect(calls).toHaveLength(2); + expect(calls).toHaveLength(1); stubFetch({}, 404); expect(trekPlacesMiss(await trekPlacesWays([1]).catch((err: unknown) => err))).toBe('not_built'); env.maps.trekPlacesEnabled = false; @@ -1064,3 +1082,39 @@ describe('trekPlacesMiss', () => { expect(trekPlacesMiss('nope')).toBe('failed'); }); }); + +describe('the layers breaker', () => { + const VIEW = { south: 47.1, west: 11.2, north: 47.2, east: 11.3 }; + const CAPS = { relations: 60, ways: 4000 }; + + it('TREK-PLACES-BREAKER-01: busy layer answers never stop place search', async () => { + stubFetch({ detail: 'busy' }, 503); + for (let i = 0; i < 4; i++) { + await expect(trekPlacesRoutes(VIEW, 'horse', CAPS)).rejects.toThrow(/503/); + } + stubFetch({ query: 'x', results: [PLACE] }); + await expect(trekPlacesSearch('x')).resolves.toHaveLength(1); + expect(calls).toHaveLength(1); + }); + + it('TREK-PLACES-BREAKER-02: the layers stop dialling on their own after a few failures', async () => { + stubFetch({ detail: 'busy' }, 503); + for (let i = 0; i < 4; i++) await expect(trekPlacesWays([1])).rejects.toThrow(/503/); + const dialled = calls.length; + await expect(trekPlacesRoutes(VIEW, 'horse', CAPS)).rejects.toThrow('unreachable, not retrying yet'); + await expect(trekPlacesWays([1])).rejects.toThrow('unreachable, not retrying yet'); + expect(calls).toHaveLength(dialled); + }); + + it('TREK-PLACES-BREAKER-03: a layer not built yet does not clear the count of search failures', async () => { + stubFetch({}, 503); + for (let i = 0; i < 3; i++) await expect(trekPlacesSearch('x')).rejects.toThrow(/503/); + stubFetch({ detail: 'not built' }, 404); + await expect(trekPlacesWays([1])).rejects.toThrow(/404/); + stubFetch({}, 503); + await expect(trekPlacesSearch('x')).rejects.toThrow(/503/); + calls = []; + await expect(trekPlacesSearch('x')).rejects.toThrow('unreachable, not retrying yet'); + expect(calls).toHaveLength(0); + }); +}); diff --git a/wiki/Environment-Variables.md b/wiki/Environment-Variables.md index 64706e726..c0cc02619 100644 --- a/wiki/Environment-Variables.md +++ b/wiki/Environment-Variables.md @@ -383,7 +383,7 @@ TREK's own place index, the [TREK Places API](TREK-Places-API), answers the sugg | Variable | Description | Default | |-----------------------|-------------|---------| -| `TREK_PLACES_ENABLED` | Set to `false` to stop asking the index. Search then works as it did before 4.3.0: OpenStreetMap on an install without a key, the keyed provider (Google or Amap) when one is set, the Overpass mirrors for the category buttons, and no places downloaded for offline search. It is a boolean switch like the others on this page: `false`, `0`, `off` and `no` in any casing switch it off, a value that is not boolean-like aborts startup, and unset or blank leaves the index on. | on | +| `TREK_PLACES_ENABLED` | Set to `false` to stop asking the index. Search then works as it did before 4.3.0: OpenStreetMap on an install without a key, the keyed provider (Google or Amap) when one is set, the Overpass mirrors for the category buttons and for the OpenStreetMap ways and horse routes Tours reads, and no places downloaded for offline search. It is a boolean switch like the others on this page: `false`, `0`, `off` and `no` in any casing switch it off, a value that is not boolean-like aborts startup, and unset or blank leaves the index on. | on | | `TREK_PLACES_URL` | Base URL of a copy of the service you run yourself; it has to answer the same `/v1` API. Unset or blank uses the public service. A trailing slash is stripped, and a value that is not a full URL aborts startup. It is configuration rather than user input and is not run through the SSRF guard, so an address on your LAN or Docker network works without `ALLOW_INTERNAL_NETWORK`. | `https://places.liketrek.com` | On Helm both go under `env:` in `values.yaml`. The chart passes `TREK_PLACES_ENABLED` through whenever it is set at all, so an unquoted `false` or `--set env.TREK_PLACES_ENABLED=false` reaches the container as well. diff --git a/wiki/TREK-Places-API.md b/wiki/TREK-Places-API.md index c0b2c9d7c..cf23fa333 100644 --- a/wiki/TREK-Places-API.md +++ b/wiki/TREK-Places-API.md @@ -74,6 +74,8 @@ Every request goes out from your TREK server, never from a browser, so the servi | The trip's area as a box | Preparing a trip for offline use | | The index id of one place | Opening a place picked from the index | | A venue name | Booking imports and route imports | +| The OpenStreetMap way ids a planned tour runs on | Tours trail grades, surfaces and kinds of way | +| The map view as a box, grown to a 0.05 degree grid | Tours horse routes | | An `X-TREK-Instance` header | Every request | The header carries an opaque token derived from your instance's own address, so the service can rate-limit each instance on its own instead of everyone behind one IP. Set `APP_URL` (or `ALLOWED_ORIGINS`): without either, the token is derived from the port alone, and two unconfigured installs on the same port share one rate-limit bucket. @@ -127,7 +129,7 @@ Defaults and details for each are on [Environment Variables](Environment-Variabl ## When the service cannot be reached -Nothing breaks. Every call has a short deadline, 3.5 seconds for anything somebody is waiting on and 20 seconds for an offline download, and every call site falls back to what TREK did before the index existed. After four calls in a row that got no answer or a server error, TREK stops asking for a minute and takes the fallback straight away, so an instance that cannot reach the service does not pay the deadline on every keystroke. The first call after that minute tries again, and one answer resets the count. +Nothing breaks. Every call has a short deadline, 3.5 seconds for a search, 10 seconds per batch of tour ways, 20 seconds for an offline download or a view of routes, and every call site falls back to what TREK did before the index existed. After four calls in a row that got no answer or a server error, TREK stops asking for a minute and takes the fallback straight away, so an instance that cannot reach the service does not pay the deadline on every keystroke. The first call after that minute tries again, and one answer resets the count. Place search and the ways and routes of Tours are counted apart, so a busy answer to a heavy tour never pauses the search. When a tour's ways stop answering part way, Tours keeps the batches that came and asks the Overpass mirrors only for the rest. | What you see | What it usually means | |---|---| @@ -135,6 +137,7 @@ Nothing breaks. Every call has a short deadline, 3.5 seconds for anything somebo | `TREK Places search failed, falling back: …` in the server log | The same, or the service could not be reached. The text after the colon says which. | | `TREK Places API unreachable, not retrying yet` in the server log | Four calls in a row got no answer and TREK is waiting out the minute. Check that the server can reach `places.liketrek.com` (or your `TREK_PLACES_URL`) and your proxy settings. | | `TREK Places area lookup failed: …` in the server log | A trip's surroundings could not be downloaded for offline use. The trip itself still syncs. | +| `[TrekPlaces] /v1/ways: …; asking the Overpass mirrors instead` (or `/v1/routes`) in the server log | Logged once per reason: the layer is switched off, not built yet on the service, or did not answer. Tours keeps using Overpass. | | The category buttons take seconds | They are being answered by Overpass, because the index had nothing for that area or is switched off. | ## See also diff --git a/wiki/Updating.md b/wiki/Updating.md index 77a495cf8..2e5478d72 100644 --- a/wiki/Updating.md +++ b/wiki/Updating.md @@ -120,6 +120,10 @@ Two things change with 4.3.1. Neither needs a manual step on Docker, but they ar - **Booked nights lead their day.** A migration moves every booked night to the head of its check-in day, behind only a stop whose own time is at or before the check-in, and the stops without an hour follow it: a hotel booked for ten in the morning is reached at ten in the [Road-Trip](Road-Trip) view, not at a quarter past twelve behind a whole day of untimed stops. A night booked for the end of a driving day heads that day too when its stops carry no time of their own, so such a day now reads hotel first. A via point drawn on the road into that hotel from the stop that is now last is removed, because that leg no longer exists in the day, the same way a drag would remove it; the other vias follow their stops. Under **Days** nothing changes. To put a hotel back at the end, drag it down the day in the Road Trip rail, or give the first stop after it a start time at or before the check-in, which brings the untimed stops behind that one along; a time on the last stop alone leaves the hotel in the middle of the day. See [Accommodations](Accommodations#on-the-route). - **Source installs need a build toolchain, or `--ignore-scripts`.** The server runs better-sqlite3 13, which ships its binaries inside the package, so nothing is compiled any more; the Docker image and the Helm chart are unaffected. On an install that runs `npm ci` itself, outside Docker, a lockfile-driven `npm ci` still runs node-gyp against those binaries and stops without Python 3 and a C++ toolchain (`build-essential` on Debian/Ubuntu, the Xcode Command Line Tools on macOS, the Visual Studio Build Tools on Windows; npm/cli#9837). Either keep those installed, as [Development-environment](Development-environment) lists them, or run `npm ci --ignore-scripts`, which is what the image does. +## Upgrading to 5.0 + +- **Tours asks the TREK Places API first.** The grades, surfaces and kinds of way under a planned tour and the horse routes around the map are read from `https://places.liketrek.com`, and the Overpass mirrors are asked only when it has no answer. An `OVERPASS_URL` pointed at an internal Overpass therefore no longer keeps these lookups inside your network: set `TREK_PLACES_ENABLED=false`, or point `TREK_PLACES_URL` at a copy you run yourself. [TREK-Places-API](TREK-Places-API#what-leaves-your-instance) lists what is sent. + ## Encryption Key Note If you are upgrading from a version that predates the dedicated `ENCRYPTION_KEY` (i.e. you have no `ENCRYPTION_KEY` environment variable set), TREK automatically falls back to `./data/.jwt_secret` on startup and immediately promotes it to `./data/.encryption_key`. No manual steps are required — the transition is handled at first boot after the upgrade.