diff --git a/server/src/nest/ai-connections/providers/provider-http.ts b/server/src/nest/ai-connections/providers/provider-http.ts index 13a657cb1..b1542772a 100644 --- a/server/src/nest/ai-connections/providers/provider-http.ts +++ b/server/src/nest/ai-connections/providers/provider-http.ts @@ -59,11 +59,21 @@ export async function refusalOf(kind: AiConnectionKind, res: Response): Promise< // refusal the provider's own code already explains (a plan that does not cover it) keeps that code. const image = namesImageInput(res.status, body) && providerFailure(kind, res.status, named).code === 'generic'; const code = image ? IMAGE_INPUT_REFUSED : named; - // Status and code only: an error body can quote the request, and the request holds the trip. - logWarn(`[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}`); + // Status, code and the refused field's path only: an error message can quote the request, and the + // request holds the trip. The path alone says which part of a replayed conversation the provider refused. + const param = refusedParamOf(body); + logWarn( + `[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}${param ? ` at ${param}` : ''}`, + ); return providerFailure(kind, res.status, code); } +/** The field an error body names as refused (OpenAI's `param`, such as `input[12]`), when it is a bare path. */ +function refusedParamOf(body: unknown): string | undefined { + const param = asRecord(asRecord(body).error).param; + return typeof param === 'string' && /^[\w.[\]-]{1,80}$/.test(param) ? param : undefined; +} + /** One SSE data payload as a JSON object; null for anything else (a `[DONE]`, a keepalive). */ export function parseEventData(data: string): Record | null { try { diff --git a/server/src/nest/assistant/assistant-history.ts b/server/src/nest/assistant/assistant-history.ts index bf3c619fe..ec3a47d19 100644 --- a/server/src/nest/assistant/assistant-history.ts +++ b/server/src/nest/assistant/assistant-history.ts @@ -9,6 +9,7 @@ import { fitToSize } from './assistant-output'; import { ADD_FOUND_PLACE_TOOL, APPROVAL_TTL_MS, + ASSISTANT_MODEL_CALLED_SERVER_TOOLS, ASSISTANT_SERVER_TOOLS, IMPORT_BOOKING_TOOL, IMPORT_TOUR_FILE_TOOL, @@ -190,13 +191,6 @@ function earlierCut(text: string, earlier: boolean): string { : text; } -/** - * The model's own calls on attached files: told after the step that made - * them, so the history ends on the user's side when the user's decision on - * them goes on. - */ -const ATTACHED_FILE_TOOLS: ReadonlySet = new Set([READ_ATTACHED_FILE_TOOL, KEEP_ATTACHED_FILE_TOOL]); - /** What became of keeping an attached file in Files, as its stored part says. */ function keptFileLine(part: ToolCallPart): string { const file = quoted(part.args.name); @@ -234,30 +228,31 @@ function attachedReadLine( } /** - * What the model is told of its reads of attached files in one step: the - * text of each file the user let it read, the first time the history holds - * it (`told` keeps track), and what became of a read the user did not allow. - * `texts` are the shared files' texts, by attachment id; one no longer there - * was swept with its file. + * The answer a replayed call gets: what became of an attached file (the text + * of a file the user let it read, the first time the history holds it, `told` + * keeping track; `texts` are the shared files' texts by attachment id), "not + * applied" for a held change that never ran, the stored result otherwise. */ -function attachedReadNote( - calls: readonly ToolCallPart[], +function replayedOutput( + part: ToolCallPart, texts: ReadonlyMap, told: Set, earlier: boolean, ): string { - return calls - .map((part) => - part.toolName === KEEP_ATTACHED_FILE_TOOL ? keptFileLine(part) : attachedReadLine(part, texts, told, earlier), - ) - .join('\n\n'); + if (part.toolName === KEEP_ATTACHED_FILE_TOOL) return keptFileLine(part); + if (part.toolName === READ_ATTACHED_FILE_TOOL) return attachedReadLine(part, texts, told, earlier); + if (part.result === undefined) return NOT_APPLIED; + return outputText(earlier ? fitToSize(part.result, EARLIER_TOOL_OUTPUT_CHARS) : part.result); } /** * The thread as the provider reads it, oldest first, from its first user * message on. A step's tool calls are followed by their answers; a held * change that was never applied answers that it was not, since every call - * must be answered. `providerIds` maps a held call's own id to the id the + * must be answered. That includes the server's tools the model called itself + * (a trip note, an attached file): the providers that replay their own record + * of a step refuse it with a call left unanswered, and every later message of + * the thread with it. `providerIds` maps a held call's own id to the id the * provider gave it. */ export function toChatHistory( @@ -282,42 +277,31 @@ export function toChatHistory( (part): part is Extract => part.type === 'tool-call' && part.toolName !== ASSISTANT_WEB_SEARCH_STEP, ); - const reads = allCalls.filter((part) => ATTACHED_FILE_TOOLS.has(part.toolName)); - const serverCalls = allCalls.filter( - (part) => ASSISTANT_SERVER_TOOLS.has(part.toolName) && !ATTACHED_FILE_TOOLS.has(part.toolName), - ); + const serverStep = (part: ToolCallPart) => + ASSISTANT_SERVER_TOOLS.has(part.toolName) && !ASSISTANT_MODEL_CALLED_SERVER_TOOLS.has(part.toolName); // Told as the user's side of the conversation: the history then ends on it // when a decision on these bookings starts the next answer. - if (serverCalls.length > 0) history.push({ role: 'user', text: serverStepsNote(serverCalls) }); - const calls = allCalls.filter((part) => !ASSISTANT_SERVER_TOOLS.has(part.toolName)); + const note = serverStepsNote(allCalls.filter(serverStep)); + if (note) history.push({ role: 'user', text: note }); + const calls = allCalls.filter((part) => !serverStep(part)); const toolCalls: AiToolCall[] = calls.map((part) => ({ id: providerIds.get(part.toolCallId) ?? part.toolCallId, name: part.toolName, args: part.args, })); const text = textOf(parts); - // Told after the step that read the files, so the history ends on the user's side when a consent goes on. - const readNote = reads.length > 0 ? attachedReadNote(reads, attachedTexts, told, earlier) : ''; - if (!text && toolCalls.length === 0) { - if (readNote) history.push({ role: 'user', text: readNote }); - return; - } + if (!text && toolCalls.length === 0) return; const providerState = decodeJson(ASSISTANT_PROVIDER_STATE, row.provider_state, `message ${row.id}`); history.push({ role: 'assistant', text, toolCalls, providerState }); calls.forEach((part, index) => { - const output = - part.result === undefined - ? NOT_APPLIED - : outputText(earlier ? fitToSize(part.result, EARLIER_TOOL_OUTPUT_CHARS) : part.result); history.push({ role: 'tool', toolCallId: toolCalls[index].id, name: part.toolName, - output, + output: replayedOutput(part, attachedTexts, told, earlier), ...(part.isError ? { isError: true } : {}), }); }); - if (readNote) history.push({ role: 'user', text: readNote }); }); return history; } diff --git a/server/src/nest/assistant/assistant-tools.constants.ts b/server/src/nest/assistant/assistant-tools.constants.ts index f1e182bec..bf0ede495 100644 --- a/server/src/nest/assistant/assistant-tools.constants.ts +++ b/server/src/nest/assistant/assistant-tools.constants.ts @@ -255,6 +255,18 @@ export const ASSISTANT_SERVER_TOOLS: ReadonlySet = new Set([ ...Object.keys(ASSISTANT_SERVER_WRITE_TOOLS), ]); +/** + * The server's tools the model calls itself. TREK answers them, but they are + * calls in the provider's own record of the step, so a replayed step has to + * answer them like any other; the rest of the server's tools are steps the + * server took, which the provider never saw. + */ +export const ASSISTANT_MODEL_CALLED_SERVER_TOOLS: ReadonlySet = new Set([ + REMEMBER_TOOL, + READ_ATTACHED_FILE_TOOL, + KEEP_ATTACHED_FILE_TOOL, +]); + /** The largest file the assistant reads. */ export const MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024; diff --git a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts index c76539fa1..b27e20b1f 100644 --- a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts +++ b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts @@ -28,6 +28,12 @@ import { toProviderSchema, type AiFetch } from '../../../../src/nest/ai-connecti import { describe, expect, it, vi } from 'vitest'; +const { logWarn } = vi.hoisted(() => ({ logWarn: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({ + ...(await importOriginal()), + logWarn, +})); + /** An SSE body from whole events, cut every `every` characters so no event arrives in one piece. */ function sseBody(events: unknown[], every = 7): ReadableStream { const text = events.map((e) => `data: ${typeof e === 'string' ? e : JSON.stringify(e)}\n\n`).join(''); @@ -145,6 +151,40 @@ describe('provider errors', () => { expect(errorCodeOf(null)).toBeUndefined(); expect(errorCodeOf({})).toBeUndefined(); }); + + it('PERR-004: a refusal is logged with the path of the field it names, never with its message (#2845)', async () => { + const refused = (param: unknown) => + failure( + new OpenAiResponsesClient( + 'm', + async () => 'at', + fetchAnswering( + jsonResponse(400, { + error: { + type: 'invalid_request_error', + code: null, + message: 'No tool output found for function call call_note. Dinner in Rome at 20:00.', + param, + }, + }), + ).fetcher, + ).stream(REQUEST, signal()), + ); + + logWarn.mockClear(); + expect(await refused('input[3]')).toMatchObject({ code: 'generic', status: 400 }); + expect(logWarn).toHaveBeenCalledWith( + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error) at input[3]', + ); + + logWarn.mockClear(); + await refused('input[3] Dinner in Rome'); + await refused(null); + expect(logWarn.mock.calls.map(([line]) => line)).toEqual([ + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error)', + '[ai] chatgpt refused the request: HTTP 400 (invalid_request_error)', + ]); + }); }); describe('toProviderSchema', () => { diff --git a/server/tests/unit/nest/assistant/assistant-history.test.ts b/server/tests/unit/nest/assistant/assistant-history.test.ts index b52b412a7..fb478a68e 100644 --- a/server/tests/unit/nest/assistant/assistant-history.test.ts +++ b/server/tests/unit/nest/assistant/assistant-history.test.ts @@ -6,6 +6,9 @@ */ import type { AssistantMessageRow } from '../../../../src/db/repositories/AssistantMessages.repository'; import type { AssistantToolCallRow } from '../../../../src/db/repositories/AssistantToolCalls.repository'; +import type { AiChatMessage } from '../../../../src/nest/ai-connections/ai-chat-client.interface'; +import { toAnthropicMessages } from '../../../../src/nest/ai-connections/providers/anthropic-messages.client'; +import { toResponsesInput } from '../../../../src/nest/ai-connections/providers/openai-responses.client'; import { messageIdOf, outputText, @@ -249,7 +252,7 @@ describe('toChatHistory', () => { ]); }); - it('ASTHIS-005d: the text of a file the user let the model read comes after the step, once, cut in earlier turns', () => { + it('ASTHIS-005d: the text of a file the user let the model read answers its read, once, cut in earlier turns', () => { const id = '0b7c2f0e-6f6d-4a8e-9b1a-3d2c1e0f9a87'; type CallPart = Extract; const read = (toolCallId: string, extra: Partial): AssistantMessagePart => ({ @@ -270,20 +273,27 @@ describe('toChatHistory', () => { row('user', [{ type: 'text', text: 'Thanks' }]), ]; const history = toChatHistory(rows, new Map(), new Map([[id, long]])); - expect(history.map((message) => message.role)).toEqual(['user', 'assistant', 'user', 'user', 'user']); - expect(history[1]).toMatchObject({ role: 'assistant', text: 'I need to read it first.', toolCalls: [] }); - const told = (history[2] as { text: string }).text; + expect(history.map((message) => message.role)).toEqual(['user', 'assistant', 'tool', 'assistant', 'tool', 'user']); + expect(history[1]).toMatchObject({ + role: 'assistant', + text: 'I need to read it first.', + toolCalls: [{ id: 'r1', name: 'read_attached_file' }], + }); + expect(history[2]).toMatchObject({ role: 'tool', toolCallId: 'r1' }); + const told = (history[2] as { output: string }).output; expect(told).toContain('The user let you read the attached file "AirBnB Buchung.pdf".'); expect(told).toContain('Casa Lisboa'); // An earlier turn's text is cut so it does not crowd out the conversation. expect(told.length).toBeLessThan(long.length); expect(told.endsWith('[...]')).toBe(true); - expect((history[3] as { text: string }).text).toBe( - 'You read the attached file "AirBnB Buchung.pdf" again; its text is above.', - ); + expect(history[4]).toMatchObject({ + role: 'tool', + toolCallId: 'r2', + output: 'You read the attached file "AirBnB Buchung.pdf" again; its text is above.', + }); // The latest turn keeps the whole text. const latest = toChatHistory(rows.slice(0, 2), new Map(), new Map([[id, long]])); - expect((latest[2] as { text: string }).text).toContain(long); + expect((latest[2] as { output: string }).output).toContain(long); }); it('ASTHIS-005e: a read the user did not allow, one still waiting, a refused one and a swept file say so', () => { @@ -303,21 +313,21 @@ describe('toChatHistory', () => { ]), ], new Map(), - ).map((message) => (message as { text: string }).text); - expect(step({ approval: { id: 'r', approved: false } })[1]).toBe( + ).map((message) => (message.role === 'tool' ? message.output : null)); + expect(step({ approval: { id: 'r', approved: false } })[2]).toBe( 'The user did not let you read the attached file "a.pdf".', ); - expect(step({ approval: { id: 'r' } })[1]).toContain("waits for the user's agreement; you have not read it."); - expect(step({ approval: { id: 'r', resolution: 'expired' } })[1]).toContain('you have not read it'); - expect(step({ result: 'No such attachment in this conversation.', isError: true })[1]).toBe( + expect(step({ approval: { id: 'r' } })[2]).toContain("waits for the user's agreement; you have not read it."); + expect(step({ approval: { id: 'r', resolution: 'expired' } })[2]).toContain('you have not read it'); + expect(step({ result: 'No such attachment in this conversation.', isError: true })[2]).toBe( 'Reading the attached file "a.pdf" failed: No such attachment in this conversation.', ); - expect(step({ result: { read: true } })[1]).toBe( + expect(step({ result: { read: true } })[2]).toBe( 'The attached file "a.pdf" is no longer kept; you cannot read it again.', ); }); - it('ASTHIS-005f: keeping an attached file is told after its step, whatever became of it', () => { + it('ASTHIS-005f: keeping an attached file is answered with what became of it', () => { const keep = (extra: Partial>) => toChatHistory( [ @@ -336,11 +346,12 @@ describe('toChatHistory', () => { new Map(), ); const kept = keep({ approval: { id: 'k', approved: true }, result: { file: { id: 31 } } }); - expect(kept.map((message) => message.role)).toEqual(['user', 'assistant', 'user']); - expect((kept[2] as { text: string }).text).toBe( - 'The attached file "a.pdf" is in the trip\'s Files now (file id 31).', - ); - const lastOf = (extra: Parameters[0]) => (keep(extra).at(-1) as { text: string }).text; + expect(kept.map((message) => message.role)).toEqual(['user', 'assistant', 'tool']); + expect(kept[2]).toMatchObject({ + toolCallId: 'k', + output: 'The attached file "a.pdf" is in the trip\'s Files now (file id 31).', + }); + const lastOf = (extra: Parameters[0]) => (keep(extra).at(-1) as { output: string }).output; expect( lastOf({ approval: { id: 'k', approved: true }, result: 'The file could not be kept.', isError: true }), ).toBe('The attached file "a.pdf" could not be kept in Files: The file could not be kept.'); @@ -448,6 +459,151 @@ describe('toChatHistory', () => { }); }); +describe('toChatHistory: calls the model made of the server tools (#2845)', () => { + /** + * The thread of #2845: a step that kept a trip note on the way (it runs at + * once), then a step whose changes the user applied, both with the + * provider's own record of the step stored beside them. + */ + function appliedAfterANote( + noteState: Record, + changeState: Record, + ): AiChatMessage[] { + const rows = [ + row('user', [{ type: 'text', text: 'Dinner at 20:00 on day 2, and remember we like late dinners' }]), + row( + 'assistant', + [ + { + type: 'tool-call', + toolCallId: 'call_note', + toolName: 'remember_for_trip', + args: { note: 'Likes late dinners' }, + result: { notes: 'Likes late dinners' }, + }, + ], + { provider_state: JSON.stringify(noteState) }, + ), + row( + 'assistant', + [ + { type: 'text', text: 'Shall I move it?' }, + { + type: 'tool-call', + toolCallId: 'held-1', + toolName: 'update_assignment_time', + args: { assignmentId: 7, place_time: '20:00' }, + result: { assignment: { id: 7 } }, + approval: { id: 'held-1', approved: true }, + }, + ], + { provider_state: JSON.stringify(changeState) }, + ), + ]; + return toChatHistory(rows, new Map([['held-1', 'call_move']])); + } + + it('ASTHIS-015: the Responses API gets an answer for every call it made, a kept note included', () => { + const history = appliedAfterANote( + { + adapter: 'openai-responses', + items: [ + { type: 'reasoning', id: 'rs_1', encrypted_content: 'x' }, + { type: 'function_call', call_id: 'call_note', name: 'remember_for_trip', arguments: '{}' }, + ], + }, + { + adapter: 'openai-responses', + items: [ + { type: 'reasoning', id: 'rs_2', encrypted_content: 'y' }, + { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'Shall I move it?' }] }, + { type: 'function_call', call_id: 'call_move', name: 'update_assignment_time', arguments: '{}' }, + ], + }, + ); + const input = toResponsesInput({ system: '', messages: history, tools: [] }); + const made = input.filter((item) => item.type === 'function_call').map((item) => item.call_id); + const answered = input.filter((item) => item.type === 'function_call_output').map((item) => item.call_id); + expect(made).toEqual(['call_note', 'call_move']); + expect(answered).toEqual(['call_note', 'call_move']); + for (const id of made) { + const call = input.findIndex((item) => item.type === 'function_call' && item.call_id === id); + const output = input.findIndex((item) => item.type === 'function_call_output' && item.call_id === id); + expect(output).toBeGreaterThan(call); + } + expect(input.find((item) => item.type === 'function_call_output' && item.call_id === 'call_note')).toMatchObject({ + output: '{"notes":"Likes late dinners"}', + }); + // Nothing empty goes out as the user's: a kept note is no step of the server to tell. + const userTexts = input + .filter((item) => item.type === 'message' && item.role === 'user') + .flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? '')); + expect(userTexts).toEqual(['Dinner at 20:00 on day 2, and remember we like late dinners']); + }); + + it('ASTHIS-016: Anthropic gets a tool_result for every tool_use it replays, a kept note included', () => { + const history = appliedAfterANote( + { + adapter: 'anthropic-messages', + blocks: [ + { type: 'thinking', thinking: 'note it', signature: 's1' }, + { type: 'tool_use', id: 'call_note', name: 'remember_for_trip', input: {} }, + ], + }, + { + adapter: 'anthropic-messages', + blocks: [ + { type: 'thinking', thinking: 'move it', signature: 's2' }, + { type: 'text', text: 'Shall I move it?' }, + { type: 'tool_use', id: 'call_move', name: 'update_assignment_time', input: {} }, + ], + }, + ); + const messages = toAnthropicMessages(history); + const uses = messages.flatMap((message) => message.content).filter((block) => block.type === 'tool_use'); + expect(uses.map((block) => block.id)).toEqual(['call_note', 'call_move']); + messages.forEach((message, index) => { + const ids = message.content.filter((block) => block.type === 'tool_use').map((block) => block.id); + if (ids.length === 0) return; + const next = messages[index + 1]; + expect(next?.role).toBe('user'); + const results = next.content.filter((block) => block.type === 'tool_result').map((block) => block.tool_use_id); + expect(results).toEqual(ids); + }); + expect(messages.flatMap((message) => message.content).some((block) => block.text === '')).toBe(false); + }); + + it('ASTHIS-017: a step that only kept a note is the model call it was, answered with what was kept', () => { + const history = toChatHistory( + [ + row('user', [{ type: 'text', text: 'Remember the ferry leaves at 7' }]), + row('assistant', [ + { + type: 'tool-call', + toolCallId: 'n1', + toolName: 'remember_for_trip', + args: { note: 'Ferry at 7' }, + result: { notes: 'Ferry at 7' }, + }, + ]), + row('assistant', [{ type: 'text', text: 'Noted.' }]), + ], + new Map(), + ); + expect(history).toEqual([ + { role: 'user', text: 'Remember the ferry leaves at 7' }, + { + role: 'assistant', + text: '', + toolCalls: [{ id: 'n1', name: 'remember_for_trip', args: { note: 'Ferry at 7' } }], + providerState: null, + }, + { role: 'tool', toolCallId: 'n1', name: 'remember_for_trip', output: '{"notes":"Ferry at 7"}' }, + { role: 'assistant', text: 'Noted.', toolCalls: [], providerState: null }, + ]); + }); +}); + describe('toThreadMessages', () => { it('ASTHIS-010: the steps of one answer are one message under the first row id', () => { nextId = 10;