From 019c66874455ef9d9c8020e6cd7abbd03fe1a935 Mon Sep 17 00:00:00 2001 From: Maurice Date: Fri, 30 Oct 2026 21:22:36 +0100 Subject: [PATCH] fix(llm-parse): ask Anthropic models for the extraction tool instead of forcing it Claude Sonnet 5.5, Opus 5.5 and Fable 5.1 reject tool_choice type "tool" and "any" with a 400, so every booking import through the Anthropic provider failed on them. The request now sends tool_choice auto with parallel tool use off, which every Anthropic model and Messages-compatible gateway accepts, and the system prompt tells the model to answer through the emit tool exactly once. A model that still answers in text is reported as before. Closes #2902 --- .../llm-parse/clients/anthropic.client.ts | 21 +++++++++++++------ .../tests/unit/nest/llm-parse/clients.test.ts | 14 ++++++++----- wiki/AI-Booking-Import.md | 2 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/server/src/nest/llm-parse/clients/anthropic.client.ts b/server/src/nest/llm-parse/clients/anthropic.client.ts index 2cc789495..2c909b973 100644 --- a/server/src/nest/llm-parse/clients/anthropic.client.ts +++ b/server/src/nest/llm-parse/clients/anthropic.client.ts @@ -10,10 +10,13 @@ const DEFAULT_MAX_TOKENS = 8192; const ANTHROPIC_VERSION = '2023-06-01'; /** - * Anthropic Messages API client. Structured output via forced tool-use: a single + * Anthropic Messages API client. Structured output via tool use: a single * `emit_` tool (`emit_reservations` by default) whose `input_schema` is - * the caller's schema, with `tool_choice` forcing it — the documented, reliable - * way to get structured JSON. + * the caller's schema, which the system prompt asks the model to answer with. + * The tool is not forced through `tool_choice`: the current models (Claude + * Sonnet 5.5, Opus 5.5, Fable 5.1) refuse `{ type: 'tool' }` and `{ type: 'any' }` + * with a 400, while `auto` is accepted by every model and by Messages-compatible + * gateways (#2902). A model that answers in text instead is reported as such below. * PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs), photos * as `image` blocks. * Raw fetch (no SDK) to match the codebase's HTTP style. @@ -48,7 +51,7 @@ export class AnthropicClient implements LlmExtractionClient { const body = { model: input.model, max_tokens: maxTokens, - system: input.prompt, + system: `${input.prompt}\n\n${toolInstruction(toolName)}`, tools: [ { name: toolName, @@ -56,7 +59,8 @@ export class AnthropicClient implements LlmExtractionClient { input_schema: input.jsonSchema, }, ], - tool_choice: { type: 'tool', name: toolName }, + // At most one call: the whole list goes into it. + tool_choice: { type: 'auto', disable_parallel_tool_use: true }, messages: [{ role: 'user', content }], }; @@ -97,7 +101,7 @@ export class AnthropicClient implements LlmExtractionClient { throw new Error('Anthropic declined to process this document'); } // A run that hits the cap stops mid-tool-call, so what arrives is a fragment - // of the list or nothing at all — and a forced tool that was not called left + // of the list or nothing at all — and a tool that was not called left // no list either. Neither is "this document holds no booking", but both came // back as [] and reached the person as an empty preview with nothing in the // log: the #2375 symptom, on the provider the dropdown offers first. @@ -114,3 +118,8 @@ export class AnthropicClient implements LlmExtractionClient { } const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.'; + +/** What stands in for the forced tool call: the answer goes through the tool, once, and nowhere else. */ +function toolInstruction(toolName: string): string { + return `Answer only by calling the ${toolName} tool, exactly once, with everything you extracted. Call it with an empty list when the document holds nothing to extract. Do not answer in text.`; +} diff --git a/server/tests/unit/nest/llm-parse/clients.test.ts b/server/tests/unit/nest/llm-parse/clients.test.ts index ee2b87cf9..b80acb0cc 100644 --- a/server/tests/unit/nest/llm-parse/clients.test.ts +++ b/server/tests/unit/nest/llm-parse/clients.test.ts @@ -576,7 +576,7 @@ describe('AnthropicClient', () => { expect(fetchFn.mock.calls[0][0]).toBe('http://gateway.lan:4000/v1/messages'); expect(fetchFn.mock.calls[1][0]).toBe('https://api.anthropic.com/v1/messages'); const body = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string); - expect(body.tool_choice).toMatchObject({ type: 'tool', name: 'emit_reservations' }); + expect(body.tool_choice).toEqual({ type: 'auto', disable_parallel_tool_use: true }); }); it('tells a prompt too long for a gateway apart from other refusals', async () => { @@ -632,7 +632,7 @@ describe('AnthropicClient', () => { expect(blockOf(1).type).toBe('document'); }); - it('forces the emit_reservations tool and reads its input', async () => { + it('asks for the emit_reservations tool without forcing it and reads its input (#2902)', async () => { const fetchFn = mockFetch(() => jsonResponse({ stop_reason: 'tool_use', @@ -644,8 +644,12 @@ describe('AnthropicClient', () => { const out = await new AnthropicClient().extract(baseInput); expect(out).toEqual([{ '@type': 'LodgingReservation' }]); const body = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string); - expect(body.tool_choice).toEqual({ type: 'tool', name: 'emit_reservations' }); + // Claude Sonnet 5.5, Opus 5.5 and Fable 5.1 refuse a forced tool_choice with a 400; + // auto with one call at most is accepted by every model, the prompt names the tool. + expect(body.tool_choice).toEqual({ type: 'auto', disable_parallel_tool_use: true }); expect(body.tools[0].name).toBe('emit_reservations'); + expect(body.system.startsWith(baseInput.prompt)).toBe(true); + expect(body.system).toContain('Answer only by calling the emit_reservations tool, exactly once'); }); /* @@ -702,7 +706,7 @@ describe('AnthropicClient', () => { }); /* - * A forced tool that was never called, and a run that stopped at the cap with a + * A tool that was never called, and a run that stopped at the cap with a * fragment of one, both came back as [] — so a long voucher reached the person * as "no reservations found", with an empty warnings array and nothing in the * log. That is the #2375 symptom on the provider the dropdown offers first, so @@ -734,7 +738,7 @@ describe('AnthropicClient', () => { } }); - it('throws when the forced tool was never called (#2375)', async () => { + it('throws when the tool was never called (#2375)', async () => { mockFetch(() => jsonResponse({ stop_reason: 'end_turn', diff --git a/wiki/AI-Booking-Import.md b/wiki/AI-Booking-Import.md index 4411f9835..6867825e5 100644 --- a/wiki/AI-Booking-Import.md +++ b/wiki/AI-Booking-Import.md @@ -84,7 +84,7 @@ The **AI Parsing** tab starts with **Reads bookings with**, which shows the conn The form for an **Own connection** has these fields: - **Provider**: **Local (Ollama)**, **OpenAI** or **Anthropic**. -- **Base URL**: empty means the provider's own address: `http://localhost:11434/v1` for a local Ollama server, `https://api.openai.com/v1` for OpenAI and `https://api.anthropic.com` for Anthropic. Any OpenAI-compatible endpoint goes here. For Anthropic it can name a gateway that speaks the Anthropic Messages API; give it without `/v1`, since TREK adds `/v1/messages`, and the gateway has to accept PDF document blocks and forced tool calls. Picking another provider empties the field, so an address meant for one never receives another provider's key. +- **Base URL**: empty means the provider's own address: `http://localhost:11434/v1` for a local Ollama server, `https://api.openai.com/v1` for OpenAI and `https://api.anthropic.com` for Anthropic. Any OpenAI-compatible endpoint goes here. For Anthropic it can name a gateway that speaks the Anthropic Messages API; give it without `/v1`, since TREK adds `/v1/messages`, and the gateway has to accept PDF document blocks and tool calls. Picking another provider empties the field, so an address meant for one never receives another provider's key. - **API key**: optional for a local server (the field says *Often not needed*), required for the cloud providers. It is stored **encrypted** and shown masked (`••••••••`) once saved; leaving it unchanged keeps the stored key. - **Model**: the model id, for example `qwen3.5:4b`, `gpt-4o` or `claude-opus-4-8`.