From c1c2bbf1467808f5843bae12d84e0046212ef3ea Mon Sep 17 00:00:00 2001 From: Maurice Date: Sun, 25 Oct 2026 16:49:46 +0100 Subject: [PATCH] fix(assistant): replay an Anthropic answer that thought with its thinking blocks Sonnet 5.5 and Opus 5.5 think by default and write what they say between tool calls into thinking blocks, which they want back unchanged with the rest of the turn. The client dropped those blocks (and their signatures) and rebuilt the turn from its text and calls, so after a tool result the next step came back empty and the assistant gave up with empty_answer. A turn that thought is now replayed block by block like one that searched the web; the cache mark before the question skips a thinking block, and replayed blocks are copies so a mark never sticks to the stored answer. Closes #2789 --- server/src/db/json-columns.ts | 2 +- .../providers/anthropic-messages.client.ts | 35 +++++++-- .../ai-connections/provider-adapters.test.ts | 73 +++++++++++++++++++ 3 files changed, 102 insertions(+), 8 deletions(-) diff --git a/server/src/db/json-columns.ts b/server/src/db/json-columns.ts index 206cc0d7f..461176e93 100644 --- a/server/src/db/json-columns.ts +++ b/server/src/db/json-columns.ts @@ -370,7 +370,7 @@ const aiProviderStateSchema = z.discriminatedUnion('adapter', [ adapter: z.literal('openai-responses'), items: z.array(z.record(z.string(), z.unknown())), }), - // Anthropic's content blocks of an answer that searched the web, replayed with their search results. + // Anthropic's content blocks of an answer that searched the web or thought, replayed as they came. z.object({ adapter: z.literal('anthropic-messages'), blocks: z.array(z.record(z.string(), z.unknown())), diff --git a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts index de819da33..9f991828d 100644 --- a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts +++ b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts @@ -86,9 +86,11 @@ export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage * A block of an answer as it goes back to the provider: as it came, but a text * block without the citations it carried, which the provider checks against * the search results and which the history needs no more than their words. + * Always a copy: the request marks its blocks for the cache in place, and a + * mark left on the stored answer would go out again on every later step. */ function replayed(block: ContentBlock): ContentBlock { - return block.type === 'text' ? { type: 'text', text: asText(block.text) } : block; + return block.type === 'text' ? { type: 'text', text: asText(block.text) } : { ...block }; } /** Anthropic's own web search, a server tool; a few searches a step keep the turn short. */ @@ -114,10 +116,17 @@ export function markHistoryForCache(messages: AnthropicMessage[], breakpoints: n last.block.cache_control = CACHED; let question = blocks.length - 1; while (question >= 0 && !(blocks[question]?.role === 'user' && blocks[question]?.block.type === 'text')) question--; - const before = question > 0 ? blocks[question - 1] : undefined; + // A replayed thinking block goes back exactly as it came, so the mark sits on the block before it. + let mark = question - 1; + while (mark >= 0 && isThinking(blocks[mark]?.block)) mark--; + const before = mark >= 0 ? blocks[mark] : undefined; if (breakpoints >= 2 && before && before !== last) before.block.cache_control = CACHED; } +function isThinking(block: ContentBlock | undefined): boolean { + return block?.type === 'thinking' || block?.type === 'redacted_thinking'; +} + const STOP_REASONS: Record = { tool_use: 'tool-calls', max_tokens: 'length' }; /** @@ -207,9 +216,10 @@ interface OpenBlock { /** * What one streamed Messages API turn has said so far, fed one event at a - * time. It keeps every content block as it came, so an answer that searched - * the web can be replayed with its search and its results (state()); an - * answer that did not is rebuilt from its text and calls as before. + * time. It keeps every content block as it came, thinking blocks with their + * signature included, so an answer that searched the web or thought can be + * replayed as it was (state()); any other answer is rebuilt from its text and + * calls as before. */ class AnthropicTurn { inputTokens = 0; @@ -246,6 +256,10 @@ class AnthropicTurn { if (delta.type === 'text_delta' && asText(delta.text)) { if (open) open.block.text = asText(open.block.text) + asText(delta.text); yield { type: 'text-delta', delta: asText(delta.text) }; + } else if (delta.type === 'thinking_delta' && open) { + open.block.thinking = asText(open.block.thinking) + asText(delta.thinking); + } else if (delta.type === 'signature_delta' && open) { + open.block.signature = asText(open.block.signature) + asText(delta.signature); } else if (delta.type === 'input_json_delta' && open) { open.json += asText(delta.partial_json); } else if (delta.type === 'citations_delta') { @@ -285,10 +299,17 @@ class AnthropicTurn { } } - /** The blocks to replay, for an answer that searched the web; null for one rebuilt from its text and calls. */ + /** + * The blocks to replay, for an answer that searched the web or thought; null + * for one rebuilt from its text and calls. A model that thinks (Sonnet 5.5, + * Opus 5.5) wants its thinking blocks back unchanged with the rest of the + * turn, and writes what it says between tool calls into them: rebuilt from + * text and calls alone, the next step after a tool came back empty (#2789). + */ state(): AiProviderState | null { - if (!this.searched) return null; const blocks = [...this.blocks.entries()].sort(([a], [b]) => a - b).map(([, open]) => open.block); + const thought = blocks.some(isThinking); + if (!this.searched && !thought) return null; return { adapter: 'anthropic-messages', blocks }; } } diff --git a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts index 060870df6..138814e7a 100644 --- a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts +++ b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts @@ -295,6 +295,79 @@ describe('AnthropicMessagesClient', () => { expect(err).not.toBeInstanceOf(AiProviderError); expect((err as Error).name).toBe('AbortError'); }); + + it('ANTH-008: an answer that thought is replayed with its thinking blocks as they came, never marked for the cache (#2789)', async () => { + const thinking = { type: 'thinking', thinking: '', signature: 'sig-a1b2' }; + const call = { type: 'tool_use', id: 'toolu_02', name: 'list_accommodations', input: {} }; + const http = fetchAnswering( + streamResponse([ + { type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-a1b2' } }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { type: 'tool_use', id: 'toolu_02', name: 'list_accommodations' }, + }, + { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '{}' } }, + { type: 'content_block_stop', index: 1 }, + { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 30 } }, + { type: 'message_stop' }, + ]), + ); + const events = await run( + new AnthropicMessagesClient('claude-sonnet-5-5', 'k', http.fetcher).stream(REQUEST, signal()), + ); + // The thinking says nothing on screen; the call and the blocks to replay come back. + expect(events.filter((e) => e.type === 'text-delta')).toEqual([]); + expect(events).toContainEqual({ + type: 'provider-state', + state: { adapter: 'anthropic-messages', blocks: [thinking, call] }, + }); + + const next = toAnthropicMessages([ + { role: 'user', text: 'Where are we staying?' }, + { + role: 'assistant', + text: '', + toolCalls: [{ id: 'toolu_02', name: 'list_accommodations', args: {} }], + providerState: { adapter: 'anthropic-messages', blocks: [thinking, call] }, + }, + { role: 'tool', toolCallId: 'toolu_02', name: 'list_accommodations', output: '[]' }, + { role: 'user', text: 'And in Kyoto?' }, + ]); + expect(next[1]).toEqual({ role: 'assistant', content: [thinking, call] }); + markHistoryForCache(next, 2); + expect(next[1]?.content[0]).toEqual(thinking); + + // An answer that ended on a thinking block: the mark before the question skips it. + const ended = toAnthropicMessages([ + { role: 'user', text: 'Hi' }, + { + role: 'assistant', + text: '', + toolCalls: [], + providerState: { adapter: 'anthropic-messages', blocks: [thinking] }, + }, + { role: 'user', text: 'Still there?' }, + ]); + markHistoryForCache(ended, 2); + expect(ended[1]?.content[0]).toEqual(thinking); + expect(ended[0]?.content[0]?.cache_control).toEqual({ type: 'ephemeral' }); + + // The request is marked, the stored answer is not: a mark left on it would go out on every later step. + const stored = { adapter: 'anthropic-messages' as const, blocks: [{ ...thinking }, { ...call }] }; + const marked = toAnthropicMessages([ + { role: 'user', text: 'Hi' }, + { role: 'assistant', text: '', toolCalls: [], providerState: stored }, + { role: 'user', text: 'Go on' }, + ]); + markHistoryForCache(marked, 2); + expect(marked[1]?.content[1]?.cache_control).toEqual({ type: 'ephemeral' }); + expect(stored.blocks[1]).not.toHaveProperty('cache_control'); + }); }); describe('OpenAiChatClient', () => {