From fed564e7c0880b23da3cfaf298da9347521c3b64 Mon Sep 17 00:00:00 2001 From: Maurice Date: Tue, 27 Oct 2026 14:53:13 +0100 Subject: [PATCH] fix(assistant): answer the model's own calls of server tools when a thread is replayed remember_for_trip runs at once during a turn, and read_attached_file and keep_attached_file are answered by the server, but all three are calls the model made. Rebuilding a thread from its stored steps left them unanswered, and ChatGPT, the OpenAI reasoning models and Anthropic replay their own record of each step, call included, so the provider refused the conversation with a 400: the continuation after applied changes failed, and every later message of the thread with it. A step that kept a note also sent an empty user message. Every call the model made is now answered as in the live turn, the attached file's outcome as the answer of its call; threads that broke this way work again on their next message. A refusal's log line names the refused field's path, never its message. Closes #2845 --- .../ai-connections/providers/provider-http.ts | 14 +- .../src/nest/assistant/assistant-history.ts | 60 ++---- .../assistant/assistant-tools.constants.ts | 12 ++ .../ai-connections/provider-adapters.test.ts | 40 ++++ .../nest/assistant/assistant-history.test.ts | 196 ++++++++++++++++-- 5 files changed, 262 insertions(+), 60 deletions(-) diff --git a/server/src/nest/ai-connections/providers/provider-http.ts b/server/src/nest/ai-connections/providers/provider-http.ts index 13a657cb1..b1542772a 100644 --- a/server/src/nest/ai-connections/providers/provider-http.ts +++ b/server/src/nest/ai-connections/providers/provider-http.ts @@ -59,11 +59,21 @@ export async function refusalOf(kind: AiConnectionKind, res: Response): Promise< // refusal the provider's own code already explains (a plan that does not cover it) keeps that code. const image = namesImageInput(res.status, body) && providerFailure(kind, res.status, named).code === 'generic'; const code = image ? IMAGE_INPUT_REFUSED : named; - // Status and code only: an error body can quote the request, and the request holds the trip. - logWarn(`[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}`); + // Status, code and the refused field's path only: an error message can quote the request, and the + // request holds the trip. The path alone says which part of a replayed conversation the provider refused. + const param = refusedParamOf(body); + logWarn( + `[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}${param ? ` at ${param}` : ''}`, + ); return providerFailure(kind, res.status, code); } +/** The field an error body names as refused (OpenAI's `param`, such as `input[12]`), when it is a bare path. */ +function refusedParamOf(body: unknown): string | undefined { + const param = asRecord(asRecord(body).error).param; + return typeof param === 'string' && /^[\w.[\]-]{1,80}$/.test(param) ? param : undefined; +} + /** One SSE data payload as a JSON object; null for anything else (a `[DONE]`, a keepalive). */ export function parseEventData(data: string): Record | null { try { diff --git a/server/src/nest/assistant/assistant-history.ts b/server/src/nest/assistant/assistant-history.ts index bf3c619fe..ec3a47d19 100644 --- a/server/src/nest/assistant/assistant-history.ts +++ b/server/src/nest/assistant/assistant-history.ts @@ -9,6 +9,7 @@ import { fitToSize } from './assistant-output'; import { ADD_FOUND_PLACE_TOOL, APPROVAL_TTL_MS, + ASSISTANT_MODEL_CALLED_SERVER_TOOLS, ASSISTANT_SERVER_TOOLS, IMPORT_BOOKING_TOOL, IMPORT_TOUR_FILE_TOOL, @@ -190,13 +191,6 @@ function earlierCut(text: string, earlier: boolean): string { : text; } -/** - * The model's own calls on attached files: told after the step that made - * them, so the history ends on the user's side when the user's decision on - * them goes on. - */ -const ATTACHED_FILE_TOOLS: ReadonlySet = new Set([READ_ATTACHED_FILE_TOOL, KEEP_ATTACHED_FILE_TOOL]); - /** What became of keeping an attached file in Files, as its stored part says. */ function keptFileLine(part: ToolCallPart): string { const file = quoted(part.args.name); @@ -234,30 +228,31 @@ function attachedReadLine( } /** - * What the model is told of its reads of attached files in one step: the - * text of each file the user let it read, the first time the history holds - * it (`told` keeps track), and what became of a read the user did not allow. - * `texts` are the shared files' texts, by attachment id; one no longer there - * was swept with its file. + * The answer a replayed call gets: what became of an attached file (the text + * of a file the user let it read, the first time the history holds it, `told` + * keeping track; `texts` are the shared files' texts by attachment id), "not + * applied" for a held change that never ran, the stored result otherwise. */ -function attachedReadNote( - calls: readonly ToolCallPart[], +function replayedOutput( + part: ToolCallPart, texts: ReadonlyMap, told: Set, earlier: boolean, ): string { - return calls - .map((part) => - part.toolName === KEEP_ATTACHED_FILE_TOOL ? keptFileLine(part) : attachedReadLine(part, texts, told, earlier), - ) - .join('\n\n'); + if (part.toolName === KEEP_ATTACHED_FILE_TOOL) return keptFileLine(part); + if (part.toolName === READ_ATTACHED_FILE_TOOL) return attachedReadLine(part, texts, told, earlier); + if (part.result === undefined) return NOT_APPLIED; + return outputText(earlier ? fitToSize(part.result, EARLIER_TOOL_OUTPUT_CHARS) : part.result); } /** * The thread as the provider reads it, oldest first, from its first user * message on. A step's tool calls are followed by their answers; a held * change that was never applied answers that it was not, since every call - * must be answered. `providerIds` maps a held call's own id to the id the + * must be answered. That includes the server's tools the model called itself + * (a trip note, an attached file): the providers that replay their own record + * of a step refuse it with a call left unanswered, and every later message of + * the thread with it. `providerIds` maps a held call's own id to the id the * provider gave it. */ export function toChatHistory( @@ -282,42 +277,31 @@ export function toChatHistory( (part): part is Extract => part.type === 'tool-call' && part.toolName !== ASSISTANT_WEB_SEARCH_STEP, ); - const reads = allCalls.filter((part) => ATTACHED_FILE_TOOLS.has(part.toolName)); - const serverCalls = allCalls.filter( - (part) => ASSISTANT_SERVER_TOOLS.has(part.toolName) && !ATTACHED_FILE_TOOLS.has(part.toolName), - ); + const serverStep = (part: ToolCallPart) => + ASSISTANT_SERVER_TOOLS.has(part.toolName) && !ASSISTANT_MODEL_CALLED_SERVER_TOOLS.has(part.toolName); // Told as the user's side of the conversation: the history then ends on it // when a decision on these bookings starts the next answer. - if (serverCalls.length > 0) history.push({ role: 'user', text: serverStepsNote(serverCalls) }); - const calls = allCalls.filter((part) => !ASSISTANT_SERVER_TOOLS.has(part.toolName)); + const note = serverStepsNote(allCalls.filter(serverStep)); + if (note) history.push({ role: 'user', text: note }); + const calls = allCalls.filter((part) => !serverStep(part)); const toolCalls: AiToolCall[] = calls.map((part) => ({ id: providerIds.get(part.toolCallId) ?? part.toolCallId, name: part.toolName, args: part.args, })); const text = textOf(parts); - // Told after the step that read the files, so the history ends on the user's side when a consent goes on. - const readNote = reads.length > 0 ? attachedReadNote(reads, attachedTexts, told, earlier) : ''; - if (!text && toolCalls.length === 0) { - if (readNote) history.push({ role: 'user', text: readNote }); - return; - } + if (!text && toolCalls.length === 0) return; const providerState = decodeJson(ASSISTANT_PROVIDER_STATE, row.provider_state, `message ${row.id}`); history.push({ role: 'assistant', text, toolCalls, providerState }); calls.forEach((part, index) => { - const output = - part.result === undefined - ? NOT_APPLIED - : outputText(earlier ? fitToSize(part.result, EARLIER_TOOL_OUTPUT_CHARS) : part.result); history.push({ role: 'tool', toolCallId: toolCalls[index].id, name: part.toolName, - output, + output: replayedOutput(part, attachedTexts, told, earlier), ...(part.isError ? { isError: true } : {}), }); }); - if (readNote) history.push({ role: 'user', text: readNote }); }); return history; } diff --git a/server/src/nest/assistant/assistant-tools.constants.ts b/server/src/nest/assistant/assistant-tools.constants.ts index f1e182bec..bf0ede495 100644 --- a/server/src/nest/assistant/assistant-tools.constants.ts +++ b/server/src/nest/assistant/assistant-tools.constants.ts @@ -255,6 +255,18 @@ export const ASSISTANT_SERVER_TOOLS: ReadonlySet = new Set([ ...Object.keys(ASSISTANT_SERVER_WRITE_TOOLS), ]); +/** + * The server's tools the model calls itself. TREK answers them, but they are + * calls in the provider's own record of the step, so a replayed step has to + * answer them like any other; the rest of the server's tools are steps the + * server took, which the provider never saw. + */ +export const ASSISTANT_MODEL_CALLED_SERVER_TOOLS: ReadonlySet = new Set([ + REMEMBER_TOOL, + READ_ATTACHED_FILE_TOOL, + KEEP_ATTACHED_FILE_TOOL, +]); + /** The largest file the assistant reads. */ export const MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024; diff --git a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts index c76539fa1..b27e20b1f 100644 --- a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts +++ b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts @@ -28,6 +28,12 @@ import { toProviderSchema, type AiFetch } from '../../../../src/nest/ai-connecti import { describe, expect, it, vi } from 'vitest'; +const { logWarn } = vi.hoisted(() => ({ logWarn: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logWarn, +})); + /** An SSE body from whole events, cut every `every` characters so no event arrives in one piece. */ function sseBody(events: unknown[], every = 7): ReadableStream { const text = events.map((e) => `data: ${typeof e === 'string' ? e : JSON.stringify(e)}\n\n`).join(''); @@ -145,6 +151,40 @@ describe('provider errors', () => { expect(errorCodeOf(null)).toBeUndefined(); expect(errorCodeOf({})).toBeUndefined(); }); + + it('PERR-004: a refusal is logged with the path of the field it names, never with its message (#2845)', async () => { + const refused = (param: unknown) => + failure( + new OpenAiResponsesClient( + 'm', + async () => 'at', + fetchAnswering( + jsonResponse(400, { + error: { + type: 'invalid_request_error', + code: null, + message: 'No tool output found for function call call_note. Dinner in Rome at 20:00.', + param, + }, + }), + ).fetcher, + ).stream(REQUEST, signal()), + ); + + logWarn.mockClear(); + expect(await refused('input[3]')).toMatchObject({ code: 'generic', status: 400 }); + expect(logWarn).toHaveBeenCalledWith( + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error) at input[3]', + ); + + logWarn.mockClear(); + await refused('input[3] Dinner in Rome'); + await refused(null); + expect(logWarn.mock.calls.map(([line]) => line)).toEqual([ + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error)', + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error)', + ]); + }); }); describe('toProviderSchema', () => { diff --git a/server/tests/unit/nest/assistant/assistant-history.test.ts b/server/tests/unit/nest/assistant/assistant-history.test.ts index b52b412a7..fb478a68e 100644 --- a/server/tests/unit/nest/assistant/assistant-history.test.ts +++ b/server/tests/unit/nest/assistant/assistant-history.test.ts @@ -6,6 +6,9 @@ */ import type { AssistantMessageRow } from '../../../../src/db/repositories/AssistantMessages.repository'; import type { AssistantToolCallRow } from '../../../../src/db/repositories/AssistantToolCalls.repository'; +import type { AiChatMessage } from '../../../../src/nest/ai-connections/ai-chat-client.interface'; +import { toAnthropicMessages } from '../../../../src/nest/ai-connections/providers/anthropic-messages.client'; +import { toResponsesInput } from '../../../../src/nest/ai-connections/providers/openai-responses.client'; import { messageIdOf, outputText, @@ -249,7 +252,7 @@ describe('toChatHistory', () => { ]); }); - it('ASTHIS-005d: the text of a file the user let the model read comes after the step, once, cut in earlier turns', () => { + it('ASTHIS-005d: the text of a file the user let the model read answers its read, once, cut in earlier turns', () => { const id = '0b7c2f0e-6f6d-4a8e-9b1a-3d2c1e0f9a87'; type CallPart = Extract; const read = (toolCallId: string, extra: Partial): AssistantMessagePart => ({ @@ -270,20 +273,27 @@ describe('toChatHistory', () => { row('user', [{ type: 'text', text: 'Thanks' }]), ]; const history = toChatHistory(rows, new Map(), new Map([[id, long]])); - expect(history.map((message) => message.role)).toEqual(['user', 'assistant', 'user', 'user', 'user']); - expect(history[1]).toMatchObject({ role: 'assistant', text: 'I need to read it first.', toolCalls: [] }); - const told = (history[2] as { text: string }).text; + expect(history.map((message) => message.role)).toEqual(['user', 'assistant', 'tool', 'assistant', 'tool', 'user']); + expect(history[1]).toMatchObject({ + role: 'assistant', + text: 'I need to read it first.', + toolCalls: [{ id: 'r1', name: 'read_attached_file' }], + }); + expect(history[2]).toMatchObject({ role: 'tool', toolCallId: 'r1' }); + const told = (history[2] as { output: string }).output; expect(told).toContain('The user let you read the attached file "AirBnB Buchung.pdf".'); expect(told).toContain('Casa Lisboa'); // An earlier turn's text is cut so it does not crowd out the conversation. expect(told.length).toBeLessThan(long.length); expect(told.endsWith('[...]')).toBe(true); - expect((history[3] as { text: string }).text).toBe( - 'You read the attached file "AirBnB Buchung.pdf" again; its text is above.', - ); + expect(history[4]).toMatchObject({ + role: 'tool', + toolCallId: 'r2', + output: 'You read the attached file "AirBnB Buchung.pdf" again; its text is above.', + }); // The latest turn keeps the whole text. const latest = toChatHistory(rows.slice(0, 2), new Map(), new Map([[id, long]])); - expect((latest[2] as { text: string }).text).toContain(long); + expect((latest[2] as { output: string }).output).toContain(long); }); it('ASTHIS-005e: a read the user did not allow, one still waiting, a refused one and a swept file say so', () => { @@ -303,21 +313,21 @@ describe('toChatHistory', () => { ]), ], new Map(), - ).map((message) => (message as { text: string }).text); - expect(step({ approval: { id: 'r', approved: false } })[1]).toBe( + ).map((message) => (message.role === 'tool' ? message.output : null)); + expect(step({ approval: { id: 'r', approved: false } })[2]).toBe( 'The user did not let you read the attached file "a.pdf".', ); - expect(step({ approval: { id: 'r' } })[1]).toContain("waits for the user's agreement; you have not read it."); - expect(step({ approval: { id: 'r', resolution: 'expired' } })[1]).toContain('you have not read it'); - expect(step({ result: 'No such attachment in this conversation.', isError: true })[1]).toBe( + expect(step({ approval: { id: 'r' } })[2]).toContain("waits for the user's agreement; you have not read it."); + expect(step({ approval: { id: 'r', resolution: 'expired' } })[2]).toContain('you have not read it'); + expect(step({ result: 'No such attachment in this conversation.', isError: true })[2]).toBe( 'Reading the attached file "a.pdf" failed: No such attachment in this conversation.', ); - expect(step({ result: { read: true } })[1]).toBe( + expect(step({ result: { read: true } })[2]).toBe( 'The attached file "a.pdf" is no longer kept; you cannot read it again.', ); }); - it('ASTHIS-005f: keeping an attached file is told after its step, whatever became of it', () => { + it('ASTHIS-005f: keeping an attached file is answered with what became of it', () => { const keep = (extra: Partial>) => toChatHistory( [ @@ -336,11 +346,12 @@ describe('toChatHistory', () => { new Map(), ); const kept = keep({ approval: { id: 'k', approved: true }, result: { file: { id: 31 } } }); - expect(kept.map((message) => message.role)).toEqual(['user', 'assistant', 'user']); - expect((kept[2] as { text: string }).text).toBe( - 'The attached file "a.pdf" is in the trip\'s Files now (file id 31).', - ); - const lastOf = (extra: Parameters[0]) => (keep(extra).at(-1) as { text: string }).text; + expect(kept.map((message) => message.role)).toEqual(['user', 'assistant', 'tool']); + expect(kept[2]).toMatchObject({ + toolCallId: 'k', + output: 'The attached file "a.pdf" is in the trip\'s Files now (file id 31).', + }); + const lastOf = (extra: Parameters[0]) => (keep(extra).at(-1) as { output: string }).output; expect( lastOf({ approval: { id: 'k', approved: true }, result: 'The file could not be kept.', isError: true }), ).toBe('The attached file "a.pdf" could not be kept in Files: The file could not be kept.'); @@ -448,6 +459,151 @@ describe('toChatHistory', () => { }); }); +describe('toChatHistory: calls the model made of the server tools (#2845)', () => { + /** + * The thread of #2845: a step that kept a trip note on the way (it runs at + * once), then a step whose changes the user applied, both with the + * provider's own record of the step stored beside them. + */ + function appliedAfterANote( + noteState: Record, + changeState: Record, + ): AiChatMessage[] { + const rows = [ + row('user', [{ type: 'text', text: 'Dinner at 20:00 on day 2, and remember we like late dinners' }]), + row( + 'assistant', + [ + { + type: 'tool-call', + toolCallId: 'call_note', + toolName: 'remember_for_trip', + args: { note: 'Likes late dinners' }, + result: { notes: 'Likes late dinners' }, + }, + ], + { provider_state: JSON.stringify(noteState) }, + ), + row( + 'assistant', + [ + { type: 'text', text: 'Shall I move it?' }, + { + type: 'tool-call', + toolCallId: 'held-1', + toolName: 'update_assignment_time', + args: { assignmentId: 7, place_time: '20:00' }, + result: { assignment: { id: 7 } }, + approval: { id: 'held-1', approved: true }, + }, + ], + { provider_state: JSON.stringify(changeState) }, + ), + ]; + return toChatHistory(rows, new Map([['held-1', 'call_move']])); + } + + it('ASTHIS-015: the Responses API gets an answer for every call it made, a kept note included', () => { + const history = appliedAfterANote( + { + adapter: 'openai-responses', + items: [ + { type: 'reasoning', id: 'rs_1', encrypted_content: 'x' }, + { type: 'function_call', call_id: 'call_note', name: 'remember_for_trip', arguments: '{}' }, + ], + }, + { + adapter: 'openai-responses', + items: [ + { type: 'reasoning', id: 'rs_2', encrypted_content: 'y' }, + { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Shall I move it?' }] }, + { type: 'function_call', call_id: 'call_move', name: 'update_assignment_time', arguments: '{}' }, + ], + }, + ); + const input = toResponsesInput({ system: '', messages: history, tools: [] }); + const made = input.filter((item) => item.type === 'function_call').map((item) => item.call_id); + const answered = input.filter((item) => item.type === 'function_call_output').map((item) => item.call_id); + expect(made).toEqual(['call_note', 'call_move']); + expect(answered).toEqual(['call_note', 'call_move']); + for (const id of made) { + const call = input.findIndex((item) => item.type === 'function_call' && item.call_id === id); + const output = input.findIndex((item) => item.type === 'function_call_output' && item.call_id === id); + expect(output).toBeGreaterThan(call); + } + expect(input.find((item) => item.type === 'function_call_output' && item.call_id === 'call_note')).toMatchObject({ + output: '{"notes":"Likes late dinners"}', + }); + // Nothing empty goes out as the user's: a kept note is no step of the server to tell. + const userTexts = input + .filter((item) => item.type === 'message' && item.role === 'user') + .flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? '')); + expect(userTexts).toEqual(['Dinner at 20:00 on day 2, and remember we like late dinners']); + }); + + it('ASTHIS-016: Anthropic gets a tool_result for every tool_use it replays, a kept note included', () => { + const history = appliedAfterANote( + { + adapter: 'anthropic-messages', + blocks: [ + { type: 'thinking', thinking: 'note it', signature: 's1' }, + { type: 'tool_use', id: 'call_note', name: 'remember_for_trip', input: {} }, + ], + }, + { + adapter: 'anthropic-messages', + blocks: [ + { type: 'thinking', thinking: 'move it', signature: 's2' }, + { type: 'text', text: 'Shall I move it?' }, + { type: 'tool_use', id: 'call_move', name: 'update_assignment_time', input: {} }, + ], + }, + ); + const messages = toAnthropicMessages(history); + const uses = messages.flatMap((message) => message.content).filter((block) => block.type === 'tool_use'); + expect(uses.map((block) => block.id)).toEqual(['call_note', 'call_move']); + messages.forEach((message, index) => { + const ids = message.content.filter((block) => block.type === 'tool_use').map((block) => block.id); + if (ids.length === 0) return; + const next = messages[index + 1]; + expect(next?.role).toBe('user'); + const results = next.content.filter((block) => block.type === 'tool_result').map((block) => block.tool_use_id); + expect(results).toEqual(ids); + }); + expect(messages.flatMap((message) => message.content).some((block) => block.text === '')).toBe(false); + }); + + it('ASTHIS-017: a step that only kept a note is the model call it was, answered with what was kept', () => { + const history = toChatHistory( + [ + row('user', [{ type: 'text', text: 'Remember the ferry leaves at 7' }]), + row('assistant', [ + { + type: 'tool-call', + toolCallId: 'n1', + toolName: 'remember_for_trip', + args: { note: 'Ferry at 7' }, + result: { notes: 'Ferry at 7' }, + }, + ]), + row('assistant', [{ type: 'text', text: 'Noted.' }]), + ], + new Map(), + ); + expect(history).toEqual([ + { role: 'user', text: 'Remember the ferry leaves at 7' }, + { + role: 'assistant', + text: '', + toolCalls: [{ id: 'n1', name: 'remember_for_trip', args: { note: 'Ferry at 7' } }], + providerState: null, + }, + { role: 'tool', toolCallId: 'n1', name: 'remember_for_trip', output: '{"notes":"Ferry at 7"}' }, + { role: 'assistant', text: 'Noted.', toolCalls: [], providerState: null }, + ]); + }); +}); + describe('toThreadMessages', () => { it('ASTHIS-010: the steps of one answer are one message under the first row id', () => { nextId = 10;