fix(llm-parse): LLM_MAX_OUTPUT_TOKENS sets the answer cap, a cut-off answer says so

The OpenAI-compatible client always asked for 4096 tokens, and a document
whose bookings needed more came back as "no reservations" with nothing that
pointed at the cap. LLM_MAX_OUTPUT_TOKENS now sets the cap for every
provider; unset keeps 4096 and 8192 so a small local model's context still
fits. A finish_reason of "length" is reported like Anthropic's max_tokens
stop, naming the limit and the variable that raises it.

Closes #2677

server/.env.example

@@ -84,6 +84,7 @@ DEMO_MODE=false # Demo mode: resets data hourly
 # KITINERARY_EXTRACTOR_PATH= # Full path to the kitinerary-extractor binary. Unset, TREK searches /usr/lib/*/libexec/kf6/ and then PATH; the official image pins it. Set it to run a newer extractor than your distribution packages — the version in use is in the startup log and in GET /api/admin/system-info. (default: auto-detected)
 
 # LLM_TIMEOUT_MS=900000 # Ceiling (ms) for one AI-parsing call before it is abandoned. One value for every provider, and the underlying HTTP client is held to it too. Generous by default so heavier parsing work fits without a code change; lower it on a cloud provider to fail fast. (default: 900000)
+# LLM_MAX_OUTPUT_TOKENS=8192 # Longest answer (tokens) one AI-parsing call may ask for. Raise it when a document with many bookings comes back cut off; keep prompt plus cap inside your model's context. (default: 4096 for OpenAI-compatible, 8192 for Anthropic)
 
 # VAPID_PUBLIC_KEY= # Web Push: the public half of the key pair browsers subscribe with (base64url, 65 bytes). Leave both halves unset and TREK generates a pair on first start and keeps it in the database, so it travels with every backup. Set both to use a pair of your own, e.g. printed by `npx web-push generate-vapid-keys`. Changing the pair later means every device has to turn push on again.
 # VAPID_PRIVATE_KEY= # Web Push: the private half (base64url, 32 bytes). Only used together with VAPID_PUBLIC_KEY; a pair whose halves do not belong together, or only one half, turns push off with an error in the log until the pair is fixed. A pair stored in the database is not used in its place, and every device stays registered.

server/src/app-config/README.md

@@ -165,7 +165,7 @@ ALLOWED_ORIGINS, UNSPLASH_ACCESS_KEY, WEBAUTHN_*, TZ, ADMIN_EMAIL,
 TREK_DB_PRE_MIGRATE_SNAPSHOT(_KEEP) (read when a migration run starts; the
 legacy-upgrade suites set it per file),
 ADMIN_PASSWORD, IDEMPOTENCY_TTL_SECONDS, MCP_RATE_LIMIT (request-path check),
-LLM_TIMEOUT_MS, NOMINATIM_URL, VAPID_* (resolved per use by the Web Push key
+LLM_TIMEOUT_MS, LLM_MAX_OUTPUT_TOKENS, NOMINATIM_URL, VAPID_* (resolved per use by the Web Push key
 service, which the notification suites build without the container).
 
 ## Exemptions — raw `process.env` stays

server/src/app-config/derive.ts

@@ -295,6 +295,13 @@ export function deriveIntegrations(raw: RawEnv) {
      * rejects a fractional value.
      */
     llmTimeoutMs: Math.floor(positiveNumberOr(raw.LLM_TIMEOUT_MS, 900_000)),
+    /**
+     * How long one AI-parsing answer may get. Unset leaves each client the cap it
+     * has always sent (4096 to OpenAI-compatible servers, 8192 to Anthropic): a
+     * bigger default would make the server of a small local model refuse every
+     * request whose prompt plus cap overruns its context (#2677).
+     */
+    llmMaxOutputTokens: positiveIntOr(raw.LLM_MAX_OUTPUT_TOKENS, 0) || undefined,
     // TRANSIT_API_URL, KITINERARY_EXTRACTOR_PATH and PATH are boot-stable and
     // read only by Nest providers, so the transitConfig and kitineraryConfig
     // tokens own them (boot-derive.ts).

server/src/app-config/env.schema.ts

@@ -144,6 +144,8 @@ export const envSchema = z.object({
   // 2^31-1 makes Node clamp the delay to 1 ms and do the same. Both refuse at
   // boot rather than degrading into a timeout of zero.
   LLM_TIMEOUT_MS: integer(1, 2_147_483_647, 'must be a whole number of milliseconds between 1 and 2147483647'),
+  // Below 256 not even one reservation fits, so a value that small is a typo, not a choice.
+  LLM_MAX_OUTPUT_TOKENS: integer(256, 1_000_000, 'must be a whole number of tokens between 256 and 1000000'),
   KITINERARY_EXTRACTOR_PATH: anyString,
   // The OS search path. Not configuration anybody sets for TREK — it is here so
   // the kitinerary probe can resolve its binary to an absolute path itself

server/src/nest/llm-parse/clients/anthropic.client.ts

@@ -5,7 +5,8 @@ import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.in
 import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
 import { UnreadableLlmResponse } from './openai-compatible.client';
 
-const MAX_TOKENS = 8192;
+/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
+const DEFAULT_MAX_TOKENS = 8192;
 const ANTHROPIC_VERSION = '2023-06-01';
 
 /**
@@ -43,9 +44,10 @@ export class AnthropicClient implements LlmExtractionClient {
       text: input.text ? `${userText}\n\n${input.text}` : userText,
     });
 
+    const maxTokens = readEnv().integrations.llmMaxOutputTokens ?? DEFAULT_MAX_TOKENS;
     const body = {
       model: input.model,
-      max_tokens: MAX_TOKENS,
+      max_tokens: maxTokens,
       system: input.prompt,
       tools: [
         {
@@ -101,7 +103,7 @@ export class AnthropicClient implements LlmExtractionClient {
     // log: the #2375 symptom, on the provider the dropdown offers first.
     if (data.stop_reason === 'max_tokens') {
       throw new UnreadableLlmResponse(
-        `the answer was cut off at the ${MAX_TOKENS}-token limit — the document is too long to extract in one pass`,
+        `the answer was cut off at the ${maxTokens}-token limit — raise LLM_MAX_OUTPUT_TOKENS or import fewer bookings per document`,
       );
     }
 

server/src/nest/llm-parse/clients/openai-compatible.client.ts

@@ -5,7 +5,8 @@ import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.in
 import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
 import { isNuExtractModel, buildNuExtractUserText, nuExtractToKiReservations } from './nuextract';
 
-const MAX_TOKENS = 4096;
+/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
+const DEFAULT_MAX_TOKENS = 4096;
 
 /**
  * A response the client could not read at all: prose where JSON was asked for,
@@ -106,10 +107,11 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
     // local server (Ollama/vLLM/llama.cpp), but newer OpenAI models reject it
     // with a 400 and demand `max_completion_tokens`. Start with the broadly
     // supported spelling and swap on that specific rejection (#1760).
+    const maxTokens = readEnv().integrations.llmMaxOutputTokens ?? DEFAULT_MAX_TOKENS;
     const buildBody = (shape: RequestShape) => {
       const baseBody = {
         model: input.model,
-        [shape.tokenParam]: MAX_TOKENS,
+        [shape.tokenParam]: maxTokens,
         // Extraction is a deterministic task — Ollama defaults to 0.7, which makes
         // small models (NuExtract) drop fields or return empty. Pin to 0, and only
         // leave it out once a server has explicitly rejected the parameter (#2262).
@@ -193,8 +195,16 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
     console.debug(`[DEBUG] LLM answered with response_format=${sentFormat(shape, nuextract)}`);
 
     const data = (await res.json()) as {
-      choices?: { message?: { content?: string } }[];
+      choices?: { message?: { content?: string }; finish_reason?: string }[];
     };
+    // A run that hits the cap stops mid-list, and the fragment either reads as
+    // nothing or not at all: "no reservations" for a document full of them, with
+    // no hint that a setting would fix it (#2677). Same rule as the Anthropic client.
+    if (data.choices?.[0]?.finish_reason === 'length') {
+      throw new UnreadableLlmResponse(
+        `the answer was cut off at the ${maxTokens}-token limit — raise LLM_MAX_OUTPUT_TOKENS or import fewer bookings per document`,
+      );
+    }
     const content = data.choices?.[0]?.message?.content;
     return nuextract ? parseNuExtract(content) : parseReservations(content, rootKey);
   }

server/tests/unit/app-config/derive.test.ts

@@ -248,6 +248,12 @@ describe('deriveIntegrations', () => {
     expect(deriveIntegrations({ LLM_TIMEOUT_MS: '-1' }).llmTimeoutMs).toBe(900_000);
     expect(deriveIntegrations({ LLM_TIMEOUT_MS: '60000.5' }).llmTimeoutMs).toBe(60_000);
   });
+
+  it('leaves the LLM answer cap to each client unless one is set (#2677)', () => {
+    expect(deriveIntegrations({}).llmMaxOutputTokens).toBeUndefined();
+    expect(deriveIntegrations({ LLM_MAX_OUTPUT_TOKENS: '' }).llmMaxOutputTokens).toBeUndefined();
+    expect(deriveIntegrations({ LLM_MAX_OUTPUT_TOKENS: '16384' }).llmMaxOutputTokens).toBe(16384);
+  });
 });
 
 describe('deriveTransit (transitConfig token)', () => {

server/tests/unit/app-config/validate.test.ts

@@ -74,6 +74,14 @@ describe('validateEnvAtBoot', () => {
     expect(() => validateEnvAtBoot({ LLM_TIMEOUT_MS: '' })).not.toThrow();
   });
 
+  it('refuses an LLM answer cap that is no whole token count or too small to hold a booking (#2677)', () => {
+    expect(() => validateEnvAtBoot({ LLM_MAX_OUTPUT_TOKENS: '8192.5' })).toThrow();
+    expect(() => validateEnvAtBoot({ LLM_MAX_OUTPUT_TOKENS: '100' })).toThrow();
+    expect(() => validateEnvAtBoot({ LLM_MAX_OUTPUT_TOKENS: 'lots' })).toThrow();
+    expect(() => validateEnvAtBoot({ LLM_MAX_OUTPUT_TOKENS: '16384' })).not.toThrow();
+    expect(() => validateEnvAtBoot({ LLM_MAX_OUTPUT_TOKENS: '' })).not.toThrow();
+  });
+
   it('refuses an ALLOW_LINK_LOCAL_IPS entry that is not a listable link-local address', () => {
     // A metadata address would be dropped silently at runtime; saying so at boot
     // is the difference between an OIDC login that works and a puzzling one.

server/tests/unit/nest/llm-parse/clients.test.ts

@@ -129,6 +129,29 @@ describe('OpenAiCompatibleClient', () => {
     }
   });
 
+  it('caps the answer at 4096 tokens unless LLM_MAX_OUTPUT_TOKENS says otherwise (#2677)', async () => {
+    const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
+    await new OpenAiCompatibleClient().extract(baseInput);
+    expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).max_tokens).toBe(4096);
+
+    vi.stubEnv('LLM_MAX_OUTPUT_TOKENS', '16384');
+    try {
+      await new OpenAiCompatibleClient().extract(baseInput);
+      expect(JSON.parse(fetchFn.mock.calls[1][1].body as string).max_tokens).toBe(16384);
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
+  it('throws when the answer was cut off at the token cap instead of reading it as empty (#2677)', async () => {
+    mockFetch(() =>
+      jsonResponse({ choices: [{ message: { content: '{"reservations":[{"@type":"Flig' }, finish_reason: 'length' }] }),
+    );
+    await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(
+      /cut off at the 4096-token limit — raise LLM_MAX_OUTPUT_TOKENS/,
+    );
+  });
+
   it('reads a single reservation the model answered on its own', async () => {
     mockFetch(() =>
       jsonResponse({ choices: [{ message: { content: '{"@type":"LodgingReservation","reservationNumber":"733"}' } }] }),
@@ -695,6 +718,22 @@ describe('AnthropicClient', () => {
     await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/cut off at the 8192-token limit/);
   });
 
+  it('sends the cap LLM_MAX_OUTPUT_TOKENS sets and names it when the answer hits it (#2677)', async () => {
+    const fetchFn = mockFetch(() =>
+      jsonResponse({
+        stop_reason: 'max_tokens',
+        content: [{ type: 'tool_use', name: 'emit_reservations', input: {} }],
+      }),
+    );
+    vi.stubEnv('LLM_MAX_OUTPUT_TOKENS', '20000');
+    try {
+      await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/cut off at the 20000-token limit/);
+      expect(JSON.parse(fetchFn.mock.calls[0][1].body as string).max_tokens).toBe(20000);
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
   it('throws when the forced tool was never called (#2375)', async () => {
     mockFetch(() =>
       jsonResponse({