diff --git a/server/src/db/json-columns.ts b/server/src/db/json-columns.ts index 206cc0d7f..461176e93 100644 --- a/server/src/db/json-columns.ts +++ b/server/src/db/json-columns.ts @@ -370,7 +370,7 @@ const aiProviderStateSchema = z.discriminatedUnion('adapter', [ adapter: z.literal('openai-responses'), items: z.array(z.record(z.string(), z.unknown())), }), - // Anthropic's content blocks of an answer that searched the web, replayed with their search results. + // Anthropic's content blocks of an answer that searched the web or thought, replayed as they came. z.object({ adapter: z.literal('anthropic-messages'), blocks: z.array(z.record(z.string(), z.unknown())), diff --git a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts index de819da33..9f991828d 100644 --- a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts +++ b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts @@ -86,9 +86,11 @@ export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage * A block of an answer as it goes back to the provider: as it came, but a text * block without the citations it carried, which the provider checks against * the search results and which the history needs no more than their words. + * Always a copy: the request marks its blocks for the cache in place, and a + * mark left on the stored answer would go out again on every later step. */ function replayed(block: ContentBlock): ContentBlock { - return block.type === 'text' ? { type: 'text', text: asText(block.text) } : block; + return block.type === 'text' ? { type: 'text', text: asText(block.text) } : { ...block }; } /** Anthropic's own web search, a server tool; a few searches a step keep the turn short. */ @@ -114,10 +116,17 @@ export function markHistoryForCache(messages: AnthropicMessage[], breakpoints: n last.block.cache_control = CACHED; let question = blocks.length - 1; while (question >= 0 && !(blocks[question]?.role === 'user' && blocks[question]?.block.type === 'text')) question--; - const before = question > 0 ? blocks[question - 1] : undefined; + // A replayed thinking block goes back exactly as it came, so the mark sits on the block before it. + let mark = question - 1; + while (mark >= 0 && isThinking(blocks[mark]?.block)) mark--; + const before = mark >= 0 ? blocks[mark] : undefined; if (breakpoints >= 2 && before && before !== last) before.block.cache_control = CACHED; } +function isThinking(block: ContentBlock | undefined): boolean { + return block?.type === 'thinking' || block?.type === 'redacted_thinking'; +} + const STOP_REASONS: Record = { tool_use: 'tool-calls', max_tokens: 'length' }; /** @@ -207,9 +216,10 @@ interface OpenBlock { /** * What one streamed Messages API turn has said so far, fed one event at a - * time. It keeps every content block as it came, so an answer that searched - * the web can be replayed with its search and its results (state()); an - * answer that did not is rebuilt from its text and calls as before. + * time. It keeps every content block as it came, thinking blocks with their + * signature included, so an answer that searched the web or thought can be + * replayed as it was (state()); any other answer is rebuilt from its text and + * calls as before. */ class AnthropicTurn { inputTokens = 0; @@ -246,6 +256,10 @@ class AnthropicTurn { if (delta.type === 'text_delta' && asText(delta.text)) { if (open) open.block.text = asText(open.block.text) + asText(delta.text); yield { type: 'text-delta', delta: asText(delta.text) }; + } else if (delta.type === 'thinking_delta' && open) { + open.block.thinking = asText(open.block.thinking) + asText(delta.thinking); + } else if (delta.type === 'signature_delta' && open) { + open.block.signature = asText(open.block.signature) + asText(delta.signature); } else if (delta.type === 'input_json_delta' && open) { open.json += asText(delta.partial_json); } else if (delta.type === 'citations_delta') { @@ -285,10 +299,17 @@ class AnthropicTurn { } } - /** The blocks to replay, for an answer that searched the web; null for one rebuilt from its text and calls. */ + /** + * The blocks to replay, for an answer that searched the web or thought; null + * for one rebuilt from its text and calls. A model that thinks (Sonnet 5.5, + * Opus 5.5) wants its thinking blocks back unchanged with the rest of the + * turn, and writes what it says between tool calls into them: rebuilt from + * text and calls alone, the next step after a tool came back empty (#2789). + */ state(): AiProviderState | null { - if (!this.searched) return null; const blocks = [...this.blocks.entries()].sort(([a], [b]) => a - b).map(([, open]) => open.block); + const thought = blocks.some(isThinking); + if (!this.searched && !thought) return null; return { adapter: 'anthropic-messages', blocks }; } } diff --git a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts index 060870df6..138814e7a 100644 --- a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts +++ b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts @@ -295,6 +295,79 @@ describe('AnthropicMessagesClient', () => { expect(err).not.toBeInstanceOf(AiProviderError); expect((err as Error).name).toBe('AbortError'); }); + + it('ANTH-008: an answer that thought is replayed with its thinking blocks as they came, never marked for the cache (#2789)', async () => { + const thinking = { type: 'thinking', thinking: '', signature: 'sig-a1b2' }; + const call = { type: 'tool_use', id: 'toolu_02', name: 'list_accommodations', input: {} }; + const http = fetchAnswering( + streamResponse([ + { type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-a1b2' } }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { type: 'tool_use', id: 'toolu_02', name: 'list_accommodations' }, + }, + { type: 'content_block_delta', index: 1, delta: { type: 'input_json_delta', partial_json: '{}' } }, + { type: 'content_block_stop', index: 1 }, + { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 30 } }, + { type: 'message_stop' }, + ]), + ); + const events = await run( + new AnthropicMessagesClient('claude-sonnet-5-5', 'k', http.fetcher).stream(REQUEST, signal()), + ); + // The thinking says nothing on screen; the call and the blocks to replay come back. + expect(events.filter((e) => e.type === 'text-delta')).toEqual([]); + expect(events).toContainEqual({ + type: 'provider-state', + state: { adapter: 'anthropic-messages', blocks: [thinking, call] }, + }); + + const next = toAnthropicMessages([ + { role: 'user', text: 'Where are we staying?' }, + { + role: 'assistant', + text: '', + toolCalls: [{ id: 'toolu_02', name: 'list_accommodations', args: {} }], + providerState: { adapter: 'anthropic-messages', blocks: [thinking, call] }, + }, + { role: 'tool', toolCallId: 'toolu_02', name: 'list_accommodations', output: '[]' }, + { role: 'user', text: 'And in Kyoto?' }, + ]); + expect(next[1]).toEqual({ role: 'assistant', content: [thinking, call] }); + markHistoryForCache(next, 2); + expect(next[1]?.content[0]).toEqual(thinking); + + // An answer that ended on a thinking block: the mark before the question skips it. + const ended = toAnthropicMessages([ + { role: 'user', text: 'Hi' }, + { + role: 'assistant', + text: '', + toolCalls: [], + providerState: { adapter: 'anthropic-messages', blocks: [thinking] }, + }, + { role: 'user', text: 'Still there?' }, + ]); + markHistoryForCache(ended, 2); + expect(ended[1]?.content[0]).toEqual(thinking); + expect(ended[0]?.content[0]?.cache_control).toEqual({ type: 'ephemeral' }); + + // The request is marked, the stored answer is not: a mark left on it would go out on every later step. + const stored = { adapter: 'anthropic-messages' as const, blocks: [{ ...thinking }, { ...call }] }; + const marked = toAnthropicMessages([ + { role: 'user', text: 'Hi' }, + { role: 'assistant', text: '', toolCalls: [], providerState: stored }, + { role: 'user', text: 'Go on' }, + ]); + markHistoryForCache(marked, 2); + expect(marked[1]?.content[1]?.cache_control).toEqual({ type: 'ephemeral' }); + expect(stored.blocks[1]).not.toHaveProperty('cache_control'); + }); }); describe('OpenAiChatClient', () => {