fix(llm-parse): LLM_MAX_OUTPUT_TOKENS sets the answer cap, a cut-off answer says so

The OpenAI-compatible client always asked for 4096 tokens, and a document
whose bookings needed more came back as "no reservations" with nothing that
pointed at the cap. LLM_MAX_OUTPUT_TOKENS now sets the cap for every
provider; unset keeps 4096 and 8192 so a small local model's context still
fits. A finish_reason of "length" is reported like Anthropic's max_tokens
stop, naming the limit and the variable that raises it.

Closes #2677

server/src/app-config/README.md

@@ -165,7 +165,7 @@ ALLOWED_ORIGINS, UNSPLASH_ACCESS_KEY, WEBAUTHN_*, TZ, ADMIN_EMAIL,
 TREK_DB_PRE_MIGRATE_SNAPSHOT(_KEEP) (read when a migration run starts; the
 legacy-upgrade suites set it per file),
 ADMIN_PASSWORD, IDEMPOTENCY_TTL_SECONDS, MCP_RATE_LIMIT (request-path check),
-LLM_TIMEOUT_MS, NOMINATIM_URL, VAPID_* (resolved per use by the Web Push key
+LLM_TIMEOUT_MS, LLM_MAX_OUTPUT_TOKENS, NOMINATIM_URL, VAPID_* (resolved per use by the Web Push key
 service, which the notification suites build without the container).
 
 ## Exemptions — raw `process.env` stays

server/src/app-config/derive.ts

@@ -295,6 +295,13 @@ export function deriveIntegrations(raw: RawEnv) {
      * rejects a fractional value.
      */
     llmTimeoutMs: Math.floor(positiveNumberOr(raw.LLM_TIMEOUT_MS, 900_000)),
+    /**
+     * How long one AI-parsing answer may get. Unset leaves each client the cap it
+     * has always sent (4096 to OpenAI-compatible servers, 8192 to Anthropic): a
+     * bigger default would make the server of a small local model refuse every
+     * request whose prompt plus cap overruns its context (#2677).
+     */
+    llmMaxOutputTokens: positiveIntOr(raw.LLM_MAX_OUTPUT_TOKENS, 0) || undefined,
     // TRANSIT_API_URL, KITINERARY_EXTRACTOR_PATH and PATH are boot-stable and
     // read only by Nest providers, so the transitConfig and kitineraryConfig
     // tokens own them (boot-derive.ts).

server/src/app-config/env.schema.ts

@@ -144,6 +144,8 @@ export const envSchema = z.object({
   // 2^31-1 makes Node clamp the delay to 1 ms and do the same. Both refuse at
   // boot rather than degrading into a timeout of zero.
   LLM_TIMEOUT_MS: integer(1, 2_147_483_647, 'must be a whole number of milliseconds between 1 and 2147483647'),
+  // Below 256 not even one reservation fits, so a value that small is a typo, not a choice.
+  LLM_MAX_OUTPUT_TOKENS: integer(256, 1_000_000, 'must be a whole number of tokens between 256 and 1000000'),
   KITINERARY_EXTRACTOR_PATH: anyString,
   // The OS search path. Not configuration anybody sets for TREK — it is here so
   // the kitinerary probe can resolve its binary to an absolute path itself

server/src/nest/llm-parse/clients/anthropic.client.ts

@@ -5,7 +5,8 @@ import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.in
 import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
 import { UnreadableLlmResponse } from './openai-compatible.client';
 
-const MAX_TOKENS = 8192;
+/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
+const DEFAULT_MAX_TOKENS = 8192;
 const ANTHROPIC_VERSION = '2023-06-01';
 
 /**
@@ -43,9 +44,10 @@ export class AnthropicClient implements LlmExtractionClient {
       text: input.text ? `${userText}\n\n${input.text}` : userText,
     });
 
+    const maxTokens = readEnv().integrations.llmMaxOutputTokens ?? DEFAULT_MAX_TOKENS;
     const body = {
       model: input.model,
-      max_tokens: MAX_TOKENS,
+      max_tokens: maxTokens,
       system: input.prompt,
       tools: [
         {
@@ -101,7 +103,7 @@ export class AnthropicClient implements LlmExtractionClient {
     // log: the #2375 symptom, on the provider the dropdown offers first.
     if (data.stop_reason === 'max_tokens') {
       throw new UnreadableLlmResponse(
-        `the answer was cut off at the ${MAX_TOKENS}-token limit — the document is too long to extract in one pass`,
+        `the answer was cut off at the ${maxTokens}-token limit — raise LLM_MAX_OUTPUT_TOKENS or import fewer bookings per document`,
       );
     }
 

server/src/nest/llm-parse/clients/openai-compatible.client.ts

@@ -5,7 +5,8 @@ import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.in
 import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
 import { isNuExtractModel, buildNuExtractUserText, nuExtractToKiReservations } from './nuextract';
 
-const MAX_TOKENS = 4096;
+/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
+const DEFAULT_MAX_TOKENS = 4096;
 
 /**
  * A response the client could not read at all: prose where JSON was asked for,
@@ -106,10 +107,11 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
     // local server (Ollama/vLLM/llama.cpp), but newer OpenAI models reject it
     // with a 400 and demand `max_completion_tokens`. Start with the broadly
     // supported spelling and swap on that specific rejection (#1760).
+    const maxTokens = readEnv().integrations.llmMaxOutputTokens ?? DEFAULT_MAX_TOKENS;
     const buildBody = (shape: RequestShape) => {
       const baseBody = {
         model: input.model,
-        [shape.tokenParam]: MAX_TOKENS,
+        [shape.tokenParam]: maxTokens,
         // Extraction is a deterministic task — Ollama defaults to 0.7, which makes
         // small models (NuExtract) drop fields or return empty. Pin to 0, and only
         // leave it out once a server has explicitly rejected the parameter (#2262).
@@ -193,8 +195,16 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
     console.debug(`[DEBUG] LLM answered with response_format=${sentFormat(shape, nuextract)}`);
 
     const data = (await res.json()) as {
-      choices?: { message?: { content?: string } }[];
+      choices?: { message?: { content?: string }; finish_reason?: string }[];
     };
+    // A run that hits the cap stops mid-list, and the fragment either reads as
+    // nothing or not at all: "no reservations" for a document full of them, with
+    // no hint that a setting would fix it (#2677). Same rule as the Anthropic client.
+    if (data.choices?.[0]?.finish_reason === 'length') {
+      throw new UnreadableLlmResponse(
+        `the answer was cut off at the ${maxTokens}-token limit — raise LLM_MAX_OUTPUT_TOKENS or import fewer bookings per document`,
+      );
+    }
     const content = data.choices?.[0]?.message?.content;
     return nuextract ? parseNuExtract(content) : parseReservations(content, rootKey);
   }