diff --git a/server/src/nest/ai-connections/ai-chat-client.interface.ts b/server/src/nest/ai-connections/ai-chat-client.interface.ts index a9c80b91d..2fe17dfad 100644 --- a/server/src/nest/ai-connections/ai-chat-client.interface.ts +++ b/server/src/nest/ai-connections/ai-chat-client.interface.ts @@ -2,7 +2,7 @@ import type { AiProviderState } from '../../db/json-columns'; import type { AiConnectionKind } from '@trek/shared'; export type { AiProviderState }; -export { AiProviderError } from './providers/provider-errors'; +export { AiProviderError, isImageRefusal } from './providers/provider-errors'; /** A tool the model may call: its name, what it is for and its arguments as JSON Schema. */ export interface AiToolSpec { @@ -18,16 +18,30 @@ export interface AiToolCall { args: Record; } +/** + * An image a message carries to a model that reads images (the Studio page + * the assistant looks at), base64 without line breaks. It lives for one turn + * in memory; nothing stores it. + */ +export interface AiImage { + mimeType: 'image/jpeg' | 'image/png' | 'image/webp'; + data: string; + detail?: 'low' | 'high'; +} + /** * The provider-neutral history a turn sends. A tool result answers the call * with the same id; an assistant message may carry the items its provider * needs replayed verbatim (`providerState`), which an adapter for another - * provider ignores and rebuilds from the text and the calls instead. + * provider ignores and rebuilds from the text and the calls instead. A user + * message or a tool result may carry images; each adapter places them where + * its provider reads them, and the assistant sends none to a model that reads + * no images. */ export type AiChatMessage = - | { role: 'user'; text: string } + | { role: 'user'; text: string; images?: AiImage[] } | { role: 'assistant'; text: string; toolCalls: AiToolCall[]; providerState?: AiProviderState | null } - | { role: 'tool'; toolCallId: string; name: string; output: string; isError?: boolean }; + | { role: 'tool'; toolCallId: string; name: string; output: string; isError?: boolean; images?: AiImage[] }; export interface AiChatRequest { /** The standing instructions, the same on every turn of a conversation (cached where the provider can). */ diff --git a/server/src/nest/ai-connections/ai-connections.constants.ts b/server/src/nest/ai-connections/ai-connections.constants.ts index 63913baa5..120e6f69a 100644 --- a/server/src/nest/ai-connections/ai-connections.constants.ts +++ b/server/src/nest/ai-connections/ai-connections.constants.ts @@ -34,6 +34,13 @@ export const AI_DEFAULT_RETENTION: AiRetention = { mode: 'days', days: 90 }; /** The user setting that records which `kind@host` a user was told receives their messages. */ export const AI_PRIVACY_ACK_SETTING = 'assistant_privacy_ack'; +/** + * The user setting that records which `kind@host` a user let receive images of + * their Studio pages (journey-studio.md 3.4), asked for on its own: the + * planner's notice says no files leave, which a page image would contradict. + */ +export const AI_STUDIO_IMAGES_ACK_SETTING = 'assistant_privacy_ack_studio_images'; + /** * Why a connection request was refused, as the client reads it from the body's * `code`: the assistant's error codes, plus the ones only setting up a diff --git a/server/src/nest/ai-connections/ai-connections.service.ts b/server/src/nest/ai-connections/ai-connections.service.ts index b91413e22..7c45c7571 100644 --- a/server/src/nest/ai-connections/ai-connections.service.ts +++ b/server/src/nest/ai-connections/ai-connections.service.ts @@ -12,6 +12,7 @@ import { pickConnection } from './ai-connection.resolver'; import { AI_BASE_URLS, AI_PRIVACY_ACK_SETTING, + AI_STUDIO_IMAGES_ACK_SETTING, aiHostOf, aiRefusal, openrouterOAuthAvailable, @@ -128,6 +129,7 @@ export class AiConnectionsService implements SharedAiConnectionPort { ? { used: await this.usage.instanceTurnsToday(userId), limit: settings.dailyTurns } : null, privacyAcked: acked, + studioImagesAcked: !!effective && (await this.studioImagesAckedFor(userId, `${kindOf(effective)}@${host}`)), retention: settings.retention, openrouterOAuth: openrouterOAuthAvailable(), }; @@ -226,9 +228,23 @@ export class AiConnectionsService implements SharedAiConnectionPort { return viewOf(await this.ownRow(userId)); } - /** The user saw that `kind@host` receives their messages on the instance's connection. */ + /** + * The user saw that `kind@host` receives their messages on the instance's + * connection, or (`scope: 'studio_images'`) that it may also receive images + * of their Studio pages, on any connection. + */ async ackPrivacy(userId: number, req: AiPrivacyAckRequest): Promise { - await this.settings.upsertSetting(userId, AI_PRIVACY_ACK_SETTING, `${req.kind}@${req.host}`); + const key = req.scope === 'studio_images' ? AI_STUDIO_IMAGES_ACK_SETTING : AI_PRIVACY_ACK_SETTING; + await this.settings.upsertSetting(userId, key, `${req.kind}@${req.host}`); + } + + /** Whether the user let this connection receive images of their Studio pages; without it a turn stays text. */ + async studioImagesAcked(userId: number, connection: Pick): Promise { + return this.studioImagesAckedFor(userId, `${connection.kind}@${connection.host}`); + } + + private async studioImagesAckedFor(userId: number, target: string): Promise { + return (await this.settings.getDecryptedUserSetting(userId, AI_STUDIO_IMAGES_ACK_SETTING)) === target; } // ── The instance's connection (admin) ────────────────────────────────────── diff --git a/server/src/nest/ai-connections/model-catalog.service.ts b/server/src/nest/ai-connections/model-catalog.service.ts index 46505627d..ab723cdb8 100644 --- a/server/src/nest/ai-connections/model-catalog.service.ts +++ b/server/src/nest/ai-connections/model-catalog.service.ts @@ -35,6 +35,34 @@ const CHATGPT_FALLBACK_MODELS = [ /** OpenAI's list holds every model of the account; these are the ones a chat with tools cannot use. */ const OPENAI_NOT_FOR_CHAT = /(audio|realtime|transcribe|tts|image|search|instruct|embedding|moderation|codex)/; +/** + * OpenAI chat models that read no images: the first GPT-4 and its dated + * snapshots, GPT-3.5 and the small reasoning models of the o1 and o3 lines. + * Every other chat model on the list reads them. A model this misses is + * caught at run time: the provider refuses the image and the assistant goes + * on without it. + */ +const OPENAI_TEXT_ONLY = /^(?:gpt-3\.5|gpt-4(?:-0\d{3})?$|o1-mini|o1-preview|o3-mini)/; + +/** Whether an OpenAI chat model reads images. */ +function openAiReadsImages(id: string): boolean { + return !OPENAI_TEXT_ONLY.test(id); +} + +/** + * Whether the model a turn runs on reads images, without asking the provider: + * every current Anthropic and Gemini model and every model of a ChatGPT plan + * do, OpenAI's by name. OpenRouter's and a custom server's cannot be told + * from the id alone, so they stay on the page description (journey-studio.md + * 6, cut 5); the model list still says so for OpenRouter where it can. A + * model this takes for one that reads images and that refuses one is caught + * at run time and remembered as text-only. + */ +export function modelReadsImages(kind: AiConnectionKind, model: string): boolean { + if (kind === 'openai') return openAiReadsImages(model); + return kind === 'anthropic' || kind === 'gemini' || kind === 'chatgpt'; +} + const openAiListSchema = z.object({ data: z.array(z.object({ id: z.string() })) }); const anthropicListSchema = z.object({ data: z.array(z.object({ id: z.string(), display_name: z.string().optional() })), @@ -45,6 +73,7 @@ const openRouterListSchema = z.object({ id: z.string(), name: z.string().optional(), supported_parameters: z.array(z.string()).optional(), + architecture: z.object({ input_modalities: z.array(z.string()).optional() }).optional(), pricing: z.object({ prompt: z.string().optional(), completion: z.string().optional() }).optional(), }), ), @@ -150,6 +179,7 @@ export class ModelCatalogService { id: m.id, label: m.display_name ?? m.id, supportsTools: true, + supportsImages: true, })); } case 'openai': { @@ -158,14 +188,14 @@ export class ModelCatalogService { .data.map((m) => m.id) .filter((id) => (id.startsWith('gpt-') || /^o\d/.test(id)) && !OPENAI_NOT_FOR_CHAT.test(id)) .sort() - .map((id) => ({ id, label: id, supportsTools: true })); + .map((id) => ({ id, label: id, supportsTools: true, supportsImages: openAiReadsImages(id) })); } case 'gemini': { const body = await this.getJson(target.kind, `${AI_BASE_URLS.gemini}/models`, bearer(secret)); return parsed(openAiListSchema, body) .data.map((m) => m.id.replace(/^models\//, '')) .filter((id) => id.startsWith('gemini') && !/(embedding|image|tts|audio)/.test(id)) - .map((id) => ({ id, label: id, supportsTools: true })); + .map((id) => ({ id, label: id, supportsTools: true, supportsImages: true })); } case 'openrouter': { // The model list is public; the key is checked on its own first. @@ -178,6 +208,8 @@ export class ModelCatalogService { label: m.name ?? m.id, supportsTools: true, free: m.pricing?.prompt === '0' && m.pricing?.completion === '0', + // A model OpenRouter lists without its input kinds is taken as one that reads no images. + supportsImages: m.architecture?.input_modalities?.includes('image') ?? false, })); } case 'chatgpt': @@ -193,12 +225,12 @@ export class ModelCatalogService { const body = await this.getJson('chatgpt', `${AI_BASE_URLS.chatgpt}/models`, bearer(accessToken)); listed = parsed(chatgptListSchema, body) .models.filter((m) => m.visibility === 'list' && m.supported_in_api !== false) - .map((m) => ({ id: m.slug, label: m.display_name ?? m.slug, supportsTools: true })); + .map((m) => chatgptModel(m.slug, m.display_name)); } catch (err) { // A refused sign-in is the answer; anything else falls back to the known list. if (err instanceof AiProviderError && !err.retryable && err.code !== 'generic') throw err; } - return listed.length > 0 ? listed : CHATGPT_FALLBACK_MODELS.map((id) => ({ id, label: id, supportsTools: true })); + return listed.length > 0 ? listed : CHATGPT_FALLBACK_MODELS.map((id) => chatgptModel(id)); } private async customModels(baseUrl: string | null, apiKey: string | null): Promise { @@ -231,6 +263,11 @@ export class ModelCatalogService { } } +/** A model of a ChatGPT plan: every one reads images through the Responses API. */ +function chatgptModel(id: string, label?: string): AiModelInfo { + return { id, label: label ?? id, supportsTools: true, supportsImages: true }; +} + function bearer(secret: string): Record { return { authorization: `Bearer ${secret}` }; } diff --git a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts index d81f7ca11..7b4e77730 100644 --- a/server/src/nest/ai-connections/providers/anthropic-messages.client.ts +++ b/server/src/nest/ai-connections/providers/anthropic-messages.client.ts @@ -4,6 +4,7 @@ import type { AiChatMessage, AiChatRequest, AiFinishReason, + AiImage, AiToolCall, } from '../ai-chat-client.interface'; import { AI_BASE_URLS } from '../ai-connections.constants'; @@ -32,10 +33,18 @@ const MAX_TOKENS = 32_000; type ContentBlock = Record; type AnthropicMessage = { role: 'user' | 'assistant'; content: ContentBlock[] }; +/** An image as a Messages API block: base64 in the request, never a URL. */ +function imageBlock(image: AiImage): ContentBlock { + return { type: 'image', source: { type: 'base64', media_type: image.mimeType, data: image.data } }; +} + /** * The provider-neutral history as Messages API turns: a tool result is a * `tool_result` block of a user turn, and turns of the same role that follow - * each other merge into one, since the API wants them alternating. + * each other merge into one, since the API wants them alternating. An image + * of a tool result goes inside that `tool_result`, after its text, where the + * model reads it as what the tool produced; a user's images go before their + * words. */ export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage[] { const out: AnthropicMessage[] = []; @@ -47,18 +56,20 @@ export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage }; for (const message of messages) { if (message.role === 'user') { - push('user', [{ type: 'text', text: message.text }]); + push('user', [...(message.images ?? []).map(imageBlock), { type: 'text', text: message.text }]); } else if (message.role === 'assistant') { push('assistant', [ ...(message.text ? [{ type: 'text', text: message.text }] : []), ...message.toolCalls.map((call) => ({ type: 'tool_use', id: call.id, name: call.name, input: call.args })), ]); } else { + const images = (message.images ?? []).map(imageBlock); + const output = images.length > 0 ? [{ type: 'text', text: message.output }, ...images] : message.output; push('user', [ { type: 'tool_result', tool_use_id: message.toolCallId, - content: message.output, + content: output, ...(message.isError ? { is_error: true } : {}), }, ]); diff --git a/server/src/nest/ai-connections/providers/openai-chat.client.ts b/server/src/nest/ai-connections/providers/openai-chat.client.ts index a0b7526e6..1f2724b24 100644 --- a/server/src/nest/ai-connections/providers/openai-chat.client.ts +++ b/server/src/nest/ai-connections/providers/openai-chat.client.ts @@ -4,6 +4,7 @@ import type { AiChatMessage, AiChatRequest, AiFinishReason, + AiImage, AiToolCall, } from '../ai-chat-client.interface'; import { AiProviderError, errorCodeOf, providerFailure } from './provider-errors'; @@ -17,20 +18,43 @@ import { toProviderSchema, type AiFetch, } from './provider-http'; +import { imageDataUrl, toolImageCaption, toolImagesEndingAt } from './provider-images'; import { readSseEvents } from './sse-lines'; import type { AiConnectionKind } from '@trek/shared'; /** The end of an OpenAI-compatible stream. */ const DONE = '[DONE]'; -/** The provider-neutral history as Chat Completions messages, the instructions and the trip card first. */ +/** + * The provider-neutral history as Chat Completions messages, the instructions + * and the trip card first. A tool message takes no image, so the images of a + * step's tool results follow its last result as one user message. + */ export function toChatMessages(request: AiChatRequest): Record[] { const system = [request.system, request.context].filter(Boolean).join('\n\n'); - return [{ role: 'system', content: system }, ...request.messages.map(toChatMessage)]; + const out: Record[] = [{ role: 'system', content: system }]; + request.messages.forEach((message, index) => { + out.push(toChatMessage(message)); + const withImages = toolImagesEndingAt(request.messages, index); + if (withImages.length === 0) return; + const content = withImages.flatMap((tool) => [ + { type: 'text', text: toolImageCaption(tool) }, + ...(tool.images ?? []).map(imagePart), + ]); + out.push({ role: 'user', content }); + }); + return out; +} + +function imagePart(image: AiImage): Record { + return { type: 'image_url', image_url: { url: imageDataUrl(image), detail: image.detail ?? 'high' } }; } function toChatMessage(message: AiChatMessage): Record { - if (message.role === 'user') return { role: 'user', content: message.text }; + if (message.role === 'user') { + if (!message.images?.length) return { role: 'user', content: message.text }; + return { role: 'user', content: [{ type: 'text', text: message.text }, ...message.images.map(imagePart)] }; + } if (message.role === 'tool') return { role: 'tool', tool_call_id: message.toolCallId, content: message.output }; return { role: 'assistant', diff --git a/server/src/nest/ai-connections/providers/openai-responses.client.ts b/server/src/nest/ai-connections/providers/openai-responses.client.ts index 3790b9db6..59ffc58d9 100644 --- a/server/src/nest/ai-connections/providers/openai-responses.client.ts +++ b/server/src/nest/ai-connections/providers/openai-responses.client.ts @@ -4,6 +4,7 @@ import type { AiChatEvent, AiChatMessage, AiChatRequest, + AiImage, AiProviderState, } from '../ai-chat-client.interface'; import { AI_BASE_URLS } from '../ai-connections.constants'; @@ -18,6 +19,7 @@ import { toProviderSchema, type AiFetch, } from './provider-http'; +import { imageDataUrl, toolImageCaption, toolImagesEndingAt } from './provider-images'; import { readSseEvents } from './sse-lines'; /** The namespace the tools are grouped in, as Sign in with ChatGPT asks for. */ @@ -40,13 +42,28 @@ type ChatgptTokenSource = (rejected?: string) => Promise; export function toResponsesInput(request: AiChatRequest): Record[] { const input: Record[] = []; if (request.context) input.push({ type: 'message', role: 'developer', content: request.context }); - for (const message of request.messages) input.push(...toItems(message)); + request.messages.forEach((message, index) => { + input.push(...toItems(message)); + // A call's output is text; the images of a step's results follow its last one as a user message. + const withImages = toolImagesEndingAt(request.messages, index); + if (withImages.length === 0) return; + const content = withImages.flatMap((tool) => [ + { type: 'input_text', text: toolImageCaption(tool) }, + ...(tool.images ?? []).map(inputImage), + ]); + input.push({ type: 'message', role: 'user', content }); + }); return input; } +function inputImage(image: AiImage): Record { + return { type: 'input_image', image_url: imageDataUrl(image), detail: image.detail ?? 'high' }; +} + function toItems(message: AiChatMessage): Record[] { if (message.role === 'user') { - return [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: message.text }] }]; + const images = (message.images ?? []).map(inputImage); + return [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: message.text }, ...images] }]; } if (message.role === 'tool') { return [{ type: 'function_call_output', call_id: message.toolCallId, output: message.output }]; diff --git a/server/src/nest/ai-connections/providers/provider-errors.ts b/server/src/nest/ai-connections/providers/provider-errors.ts index 701c53441..12e89f630 100644 --- a/server/src/nest/ai-connections/providers/provider-errors.ts +++ b/server/src/nest/ai-connections/providers/provider-errors.ts @@ -83,3 +83,40 @@ export function errorCodeOf(body: unknown): string | undefined { } return undefined; } + +/** + * The code a refusal carries when the provider said the request's image was + * the problem: a model that reads no images, a format it does not take. The + * assistant then sends the step again without images and remembers the model + * as one that reads none. + */ +export const IMAGE_INPUT_REFUSED = 'image_input_refused'; + +/** How far into an error body's message the image check reads. */ +const MAX_MESSAGE_CHARS = 2000; + +/** The providers' wording for an image they refuse ("image_url is only supported by certain models"). */ +const NAMES_IMAGE = /\b(?:image|images|image_url|input_image|vision)\b/i; + +/** The message out of a provider's JSON error body, wherever it puts it; '' when it has none. */ +function errorMessageOf(body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const record = body as Record; + const inner = record.error; + if (inner && typeof inner === 'object' && typeof (inner as Record).message === 'string') { + return (inner as Record).message; + } + if (typeof record.message === 'string') return record.message; + return typeof record.detail === 'string' ? record.detail : ''; +} + +/** Whether a refused request was refused over its image, read from the status and the error body. */ +export function namesImageInput(status: number, body: unknown): boolean { + if (status !== 400 && status !== 415 && status !== 422) return false; + return NAMES_IMAGE.test(errorMessageOf(body).slice(0, MAX_MESSAGE_CHARS)); +} + +/** Whether a provider error is the refusal of an image (see IMAGE_INPUT_REFUSED). */ +export function isImageRefusal(err: unknown): boolean { + return err instanceof AiProviderError && err.providerCode === IMAGE_INPUT_REFUSED; +} diff --git a/server/src/nest/ai-connections/providers/provider-http.ts b/server/src/nest/ai-connections/providers/provider-http.ts index 1dfc25587..13a657cb1 100644 --- a/server/src/nest/ai-connections/providers/provider-http.ts +++ b/server/src/nest/ai-connections/providers/provider-http.ts @@ -1,6 +1,6 @@ import { readCappedJson } from '../../../utils/cappedFetch'; import { logWarn } from '../../audit/audit-log.logger'; -import { AiProviderError, errorCodeOf, providerFailure } from './provider-errors'; +import { AiProviderError, errorCodeOf, IMAGE_INPUT_REFUSED, namesImageInput, providerFailure } from './provider-errors'; import type { AiConnectionKind } from '@trek/shared'; /** @@ -54,7 +54,11 @@ export async function openProviderStream( /** The AiProviderError a non-success response stands for, read from its status and its error body. */ export async function refusalOf(kind: AiConnectionKind, res: Response): Promise { const body = await readCappedJson(res, MAX_ERROR_BYTES).catch(() => undefined); - const code = errorCodeOf(body); + const named = errorCodeOf(body); + // A refused image is named by its own code, so the turn can send the step again without it; a + // refusal the provider's own code already explains (a plan that does not cover it) keeps that code. + const image = namesImageInput(res.status, body) && providerFailure(kind, res.status, named).code === 'generic'; + const code = image ? IMAGE_INPUT_REFUSED : named; // Status and code only: an error body can quote the request, and the request holds the trip. logWarn(`[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}`); return providerFailure(kind, res.status, code); diff --git a/server/src/nest/ai-connections/providers/provider-images.ts b/server/src/nest/ai-connections/providers/provider-images.ts new file mode 100644 index 000000000..7693d38ec --- /dev/null +++ b/server/src/nest/ai-connections/providers/provider-images.ts @@ -0,0 +1,29 @@ +import type { AiChatMessage, AiImage } from '../ai-chat-client.interface'; + +type ToolMessage = Extract; + +/** An image as the data URL the OpenAI-style APIs read: base64 in the request, never a link to TREK. */ +export function imageDataUrl(image: AiImage): string { + return `data:${image.mimeType};base64,${image.data}`; +} + +/** The line an image of a tool result is introduced with, where it has to travel in a user message. */ +export function toolImageCaption(message: ToolMessage): string { + return `The image the call ${message.name} (${message.toolCallId}) produced:`; +} + +/** + * The tool results with images whose run ends at `index`: an API that takes + * no image inside a tool result (Chat Completions, Responses) gets them in + * one user message after the last result of the step, since the results must + * follow the assistant's calls directly. Empty while the run goes on, or when + * nothing in it carried an image. + */ +export function toolImagesEndingAt(messages: readonly AiChatMessage[], index: number): ToolMessage[] { + if (messages[index]?.role !== 'tool' || messages[index + 1]?.role === 'tool') return []; + let start = index; + while (start > 0 && messages[start - 1]?.role === 'tool') start--; + return messages + .slice(start, index + 1) + .filter((message): message is ToolMessage => message.role === 'tool' && (message.images?.length ?? 0) > 0); +} diff --git a/server/src/nest/assistant/assistant-agent.ts b/server/src/nest/assistant/assistant-agent.ts index 7258a5084..662edc508 100644 --- a/server/src/nest/assistant/assistant-agent.ts +++ b/server/src/nest/assistant/assistant-agent.ts @@ -1,8 +1,10 @@ import type { AiProviderState } from '../../db/json-columns'; import { AiProviderError, + isImageRefusal, type AiChatClient, type AiChatMessage, + type AiImage, type AiToolCall, type AiToolSpec, } from '../ai-connections/ai-chat-client.interface'; @@ -15,22 +17,42 @@ import type { AssistantToolKind, ToolRun } from './assistant-tools.service'; import type { AnswerInput } from './cards/card-types'; import { answerCardOf } from './cards/registry'; import type { UiStreamWriter } from './ui-stream.writer'; -import type { - AssistantChangePreview, - AssistantErrorCode, - AssistantMessagePart, - AssistantWeatherDay, - TrekCard, +import { + ASSISTANT_PAGE_IMAGE, + type AssistantChangePreview, + type AssistantErrorCode, + type AssistantMessagePart, + type AssistantPageView, + type AssistantWeatherDay, + type TrekCard, } from '@trek/shared'; type ToolArgs = Record; -/** The tools of the turn, as the loop uses them (an AssistantToolSession). */ -interface AgentToolPort { +/** + * What a tool call came back with; a look at a Studio page also brings the + * page image for the model (in memory for this turn only) and what the + * message keeps of the look. + */ +export interface AgentToolRun extends ToolRun { + images?: AiImage[]; + view?: AssistantPageView; +} + +/** The tools of the turn, as the loop uses them (an AssistantToolSession, in the Studio with its page preview). */ +export interface AgentToolPort { specs(): AiToolSpec[]; kindOf(name: string): AssistantToolKind | null; pinArgs(name: string, args: ToolArgs): { args: ToolArgs } | { error: string }; - call(name: string, args: ToolArgs): Promise; + call(name: string, args: ToolArgs): Promise; +} + +/** A Studio turn's page images, as the loop uses them (assistant-vision.ts). */ +export interface AgentVision { + /** The provider refused an image: the rest of the turn, and later turns on this model, go without. */ + refused(): void; + /** The history a turn starts from, with views of the pages the changes just applied left. */ + afterApply(history: readonly AiChatMessage[]): Promise; } /** What one step leaves in its message row. */ @@ -71,6 +93,13 @@ export interface AgentDeps { today: string; /** The card a read's answer shows; the registry's (`answerCardOf`) unless a test hands in its own. */ answerCard?: (tool: string, input: AnswerInput) => TrekCard | null; + /** + * Why a proposed change goes back to the model as its call's error instead + * of to the user (a Studio turn's book check); null holds it as usual. + */ + refusalOf?: (tool: string, args: ToolArgs) => Promise; + /** A Studio turn's page images; without it no image ever goes to the model. */ + vision?: AgentVision | null; } export interface AgentInput { @@ -103,6 +132,8 @@ interface StepParts { knownPlaceIds: Set; /** The answer cards the step has shown; past MAX_CARDS_PER_STEP a read shows none. */ cards: number; + /** The page images of the step's reads, by call id, for the model's next request only. */ + images: Map; } /** The longest call id a card is kept under; a read under a longer one shows no card. */ @@ -169,6 +200,12 @@ async function runRead( ...(run.ok ? {} : { isError: true }), }); if (!run.ok) return; + // A look at a Studio page: the message keeps what was looked at, the model gets one image of it. + if (run.view) { + w.pageView(run.view); + step.parts.push({ type: 'data-trek-page-view', data: run.view }); + } + if (run.images?.length) step.images.set(id, run.images.slice(0, 1)); const input = { toolCallId: id, args, value: run.value, card: deps.card, today: deps.today }; const card = id.length <= MAX_CARD_CALL_ID ? (deps.answerCard ?? answerCardOf)(name, input) : null; if (card && step.cards++ < MAX_CARDS_PER_STEP) { @@ -221,6 +258,39 @@ interface StepAnswer { cutOff: boolean; } +/** + * The history as one model request carries it: the page images of the + * newest `max` messages that have any, one image each; older messages go as + * their text alone, which holds the page's description. + */ +function limitImages(history: readonly AiChatMessage[], max: number): AiChatMessage[] { + let kept = 0; + return history + .slice() + .reverse() + .map((message) => { + if (message.role === 'assistant' || !message.images?.length) return message; + return { ...message, images: kept++ < max ? message.images.slice(0, 1) : undefined }; + }) + .reverse(); +} + +/** + * A provider that refused the request over its images (a model that reads + * none after all): the turn's history loses its images, the vision remembers + * the model as text only, and the step is asked again, once. False for any + * other failure, or when there was no image to drop. + */ +function dropImagesAfter(turn: Turn, err: unknown): boolean { + const withImages = turn.history.some((message) => message.role !== 'assistant' && !!message.images?.length); + if (!turn.deps.vision || !withImages || !isImageRefusal(err)) return false; + turn.deps.vision.refused(); + turn.history.forEach((message, index) => { + if (message.role !== 'assistant' && message.images) turn.history[index] = { ...message, images: undefined }; + }); + return true; +} + /** Streams one answer of the model; a failure or a stop ends the turn here, saved as far as it got. */ async function streamAnswer(turn: Turn, rowId: number): Promise { const { deps, input, w, signal } = turn; @@ -230,7 +300,7 @@ async function streamAnswer(turn: Turn, rowId: number): Promise MAX_WRITES_PER_STEP : reads > MAX_READS_PER_STEP; if ('error' in prepared || overCap) { @@ -334,19 +411,30 @@ async function holdChanges(turn: Turn, rowId: number, step: StepRecord, held: He return outcomeOf(turn, 'awaiting'); } -/** Records a step that ran its reads, and hands their answers back for the next one. */ -async function carryOn(turn: Turn, rowId: number, step: StepRecord, text: string, calls: AiToolCall[]): Promise { +/** + * Records a step that ran its reads, and hands their answers back for the + * next one, a page image beside the answer of the look that brought it. + */ +async function carryOn( + turn: Turn, + rowId: number, + step: StepRecord, + answer: { text: string; calls: AiToolCall[]; images: ReadonlyMap }, +): Promise { await turn.deps.store.saveStep(rowId, step, 'complete'); turn.w.finishStep(); + const { text, calls, images } = answer; turn.history.push({ role: 'assistant', text, toolCalls: calls, providerState: step.providerState }); for (const part of step.parts) { if (part.type !== 'tool-call') continue; + const seen = images.get(part.toolCallId); turn.history.push({ role: 'tool', toolCallId: part.toolCallId, name: part.toolName, output: outputText(part.result), ...(part.isError ? { isError: true } : {}), + ...(seen ? { images: seen } : {}), }); } } @@ -387,7 +475,7 @@ async function runStep(turn: Turn, step: number): Promise { } const calls = received.map((call) => ({ ...call, id: uniqueCallId(call, input.usedCallIds, deps.newId) })); - const collected: StepParts = { parts, weather: [], knownPlaceIds: turn.knownPlaceIds, cards: 0 }; + const collected: StepParts = { parts, weather: [], knownPlaceIds: turn.knownPlaceIds, cards: 0, images: new Map() }; const held = await runCalls(turn, calls, collected); // One weather strip per step, whatever the step held: it is told before the changes. const weather = weatherOf(collected.weather); @@ -403,7 +491,7 @@ async function runStep(turn: Turn, step: number): Promise { return outcomeOf(turn, 'aborted'); } if (held.length > 0) return holdChanges(turn, rowId, { parts, providerState }, held); - await carryOn(turn, rowId, { parts, providerState }, text, calls); + await carryOn(turn, rowId, { parts, providerState }, { text, calls, images: collected.images }); return null; } @@ -428,12 +516,13 @@ async function stepsFrom(turn: Turn, step: number): Promise { * before its held changes are announced, so a card the client shows always * exists on the server. */ -export function runAgentLoop( +export async function runAgentLoop( deps: AgentDeps, input: AgentInput, w: UiStreamWriter, signal: AbortSignal, ): Promise { + const history = deps.vision ? await deps.vision.afterApply(input.history) : [...input.history]; const knownPlaceIds = new Set( [...deps.card.places.values()].flatMap((place) => (place.googlePlaceId ? [place.googlePlaceId] : [])), ); @@ -442,7 +531,7 @@ export function runAgentLoop( input, w, signal, - history: [...input.history], + history, usage: { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 }, knownPlaceIds, nudged: false, diff --git a/server/src/nest/assistant/assistant-book-preview.ts b/server/src/nest/assistant/assistant-book-preview.ts new file mode 100644 index 000000000..acf40f9a3 --- /dev/null +++ b/server/src/nest/assistant/assistant-book-preview.ts @@ -0,0 +1,101 @@ +import type { AiToolSpec } from '../ai-connections/ai-chat-client.interface'; +import type { AgentToolPort, AgentToolRun } from './assistant-agent'; +import { PREVIEW_BOOK_PAGE_TOOL } from './assistant-tools.constants'; +import type { TurnVision } from './assistant-vision'; +import { ASSISTANT_PAGE_IMAGE, previewBookPageRequestSchema } from '@trek/shared'; + +/** + * `preview_book_page`, offered in every Studio turn beside the session's own + * tools (journey-studio.md 4.3, step 2): a read that answers one page in words + * with its findings, as it stands or as an `edit_book_page` with the given + * ops would leave it, and for a model that reads images the page as the + * Studio draws it (AssistantVision). Nothing is saved; the change itself is + * proposed afterwards like any other. + * + * Its `ops` take exactly the schema `edit_book_page` takes in this session, + * so a preview and the edit it checks are one shape. A session without that + * tool (a viewer's) previews pages as they stand. Previews are capped per + * step and per turn, since each can wait seconds for the Studio and costs a + * model an image. + */ + +const EDIT_TOOL = 'edit_book_page'; + +const DESCRIPTION = + 'Look at one page of the book before you change it or after: its elements, its findings (overlaps, words past ' + + 'their frame, empty frames, low resolution, content in the fold) and, when your model reads images, the page as ' + + 'the Studio draws it, with photos of other contributors left as empty frames. With ops (the same ops ' + + 'edit_book_page takes) you see the page as that edit would leave it, without saving anything. Preview every ' + + 'page you are about to change once, fix what the findings or the image show, then propose the edit. ' + + 'With designId (from design_book) you see the page of that design draft instead of the book. ' + + `At most ${ASSISTANT_PAGE_IMAGE.previewsPerStep} previews per step and ${ASSISTANT_PAGE_IMAGE.previewsPerTurn} ` + + 'per turn.'; + +/** The draft a preview may look at instead of the book, by design_book's design_id. */ +const DESIGN_ID = { type: 'string', pattern: '^[\\w-]{1,40}$', description: 'A design draft, by its design_id.' }; + +const TOO_MANY = + `Too many previews: at most ${ASSISTANT_PAGE_IMAGE.previewsPerStep} in one step and ` + + `${ASSISTANT_PAGE_IMAGE.previewsPerTurn} in one turn. Propose the edit and let its card show the findings.`; + +/** The preview's spec, its ops taken from the session's `edit_book_page` when it has that tool. */ +function previewSpec(specs: readonly AiToolSpec[]): AiToolSpec { + const edit = specs.find((spec) => spec.name === EDIT_TOOL); + const properties = (edit?.parameters.properties ?? {}) as Record; + const ops = properties.ops; + // Shared parts the ops point at, where the schema keeps them apart. + const shared: Record = {}; + for (const key of ['$defs', 'definitions']) if (edit?.parameters[key]) shared[key] = edit.parameters[key]; + return { + name: PREVIEW_BOOK_PAGE_TOOL, + description: DESCRIPTION, + parameters: { + type: 'object', + properties: { + spreadId: { type: 'string', pattern: '^[\\w-]{1,40}$', description: 'The page, by its id from get_book.' }, + // A design draft is an editor's: a session without the edit tool previews the book only. + ...(ops ? { ops, designId: DESIGN_ID } : {}), + }, + required: ['spreadId'], + additionalProperties: false, + ...(ops ? shared : {}), + }, + }; +} + +/** The session's tools with the preview beside them, answered through the turn's vision. */ +export function withPagePreview(tools: AgentToolPort, vision: TurnVision): AgentToolPort { + let inStep = 0; + let inTurn = 0; + const preview = async (args: Record): Promise => { + if (inStep >= ASSISTANT_PAGE_IMAGE.previewsPerStep || inTurn >= ASSISTANT_PAGE_IMAGE.previewsPerTurn) { + return { ok: false, value: TOO_MANY }; + } + const parsed = previewBookPageRequestSchema.safeParse(args); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + const detail = issue ? [issue.path.join('.'), issue.message].join(' ') : ''; + return { ok: false, value: `Invalid arguments: ${detail}` }; + } + inStep += 1; + inTurn += 1; + const { spreadId, ops, designId } = parsed.data; + return designId ? vision.look(spreadId, ops, designId) : vision.look(spreadId, ops); + }; + return { + // The loop asks for the specs once at the start of every step: a new step may preview again. + specs: () => { + inStep = 0; + const listed = tools.specs(); + return [...listed, previewSpec(listed)]; + }, + kindOf: (name) => (name === PREVIEW_BOOK_PAGE_TOOL ? 'read' : tools.kindOf(name)), + pinArgs: (name, args) => { + if (name !== PREVIEW_BOOK_PAGE_TOOL) return tools.pinArgs(name, args); + // The journey is the conversation's; the preview takes no other argument for it. + const { journeyId: _journeyId, ...rest } = args; + return { args: rest }; + }, + call: (name, args) => (name === PREVIEW_BOOK_PAGE_TOOL ? preview(args) : tools.call(name, args)), + }; +} diff --git a/server/src/nest/assistant/assistant-render.registry.ts b/server/src/nest/assistant/assistant-render.registry.ts new file mode 100644 index 000000000..dbe79218b --- /dev/null +++ b/server/src/nest/assistant/assistant-render.registry.ts @@ -0,0 +1,84 @@ +/** + * The Studio page images a running turn waits for (journey-studio.md 3.1), as + * an injectable port. The turn streams a `data-trek-render-request` and waits + * here; the Studio tab that holds the stream draws the page and uploads it to + * the renders route, which hands it over. A turn waits a bounded time and + * stops waiting with its own abort; an upload nobody waits for (too late, + * another user's, never asked) is refused. The image lives in memory for that + * one hand-over: nothing here keeps it. In memory today, like the running + * turns themselves (AssistantStreamRegistry). + */ + +/** A page image as the Studio uploaded it, checked by the renders route. */ +export interface PageImage { + mimeType: 'image/jpeg' | 'image/png' | 'image/webp'; + data: Buffer; + width: number; + height: number; +} + +/** Whose request an upload answers: the user, their journey and thread, and the request's own id. */ +interface RenderKey { + userId: number; + journeyId: number; + threadId: number; + requestId: string; +} + +/** A request a turn streamed: whose it is, and the page and book version it asked for. */ +export interface RenderWait extends RenderKey { + spreadId: string; + bookVersion: number; +} + +/** + * What became of an upload: `taken` by the turn waiting for it, `unknown` + * when no turn of this user waits for that request, `mismatch` when it shows + * another page or version than the one asked for (the turn goes on waiting). + */ +type RenderDelivery = 'taken' | 'unknown' | 'mismatch'; + +export abstract class AssistantRenderRegistry { + /** The image of a streamed request, or null once `timeoutMs` passed or `signal` aborted. */ + abstract wait(request: RenderWait, signal: AbortSignal, timeoutMs: number): Promise; + /** Hands an upload to the turn waiting for it. */ + abstract deliver(key: RenderKey, page: { spreadId: string; bookVersion: number }, image: PageImage): RenderDelivery; +} + +interface Waiting { + request: RenderWait; + settle: (image: PageImage | null) => void; +} + +export class InMemoryAssistantRenderRegistry extends AssistantRenderRegistry { + private readonly waiting = new Map(); + + wait(request: RenderWait, signal: AbortSignal, timeoutMs: number): Promise { + if (signal.aborted) return Promise.resolve(null); + return new Promise((resolve) => { + let timer: ReturnType | undefined; + const settle = (image: PageImage | null) => { + clearTimeout(timer); + signal.removeEventListener('abort', onAbort); + if (this.waiting.get(request.requestId)?.settle === settle) this.waiting.delete(request.requestId); + resolve(image); + }; + const onAbort = () => settle(null); + timer = setTimeout(() => settle(null), timeoutMs); + signal.addEventListener('abort', onAbort, { once: true }); + this.waiting.set(request.requestId, { request, settle }); + }); + } + + deliver(key: RenderKey, page: { spreadId: string; bookVersion: number }, image: PageImage): RenderDelivery { + const entry = this.waiting.get(key.requestId); + const asked = entry?.request; + if (!entry || !asked) return 'unknown'; + if (asked.userId !== key.userId || asked.journeyId !== key.journeyId || asked.threadId !== key.threadId) { + return 'unknown'; + } + if (asked.spreadId !== page.spreadId || asked.bookVersion !== page.bookVersion) return 'mismatch'; + entry.settle(image); + return 'taken'; + } +} diff --git a/server/src/nest/assistant/assistant-renders.controller.ts b/server/src/nest/assistant/assistant-renders.controller.ts new file mode 100644 index 000000000..9c4d85a01 --- /dev/null +++ b/server/src/nest/assistant/assistant-renders.controller.ts @@ -0,0 +1,123 @@ +import { ADDON_IDS } from '../../addons'; +import type { User } from '../../types'; +import { AddonsService } from '../addons/addons.service'; +import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { CurrentUser } from '../auth-core/current-user.decorator'; +import { JwtAuthGuard } from '../auth-core/jwt-auth.guard'; +import { readImageHeader } from '../common/image-header'; +import { isManagedBlocked, MANAGED_FORBIDDEN_ERROR, ManagedForbidden } from '../common/managed'; +import { ResponseContract } from '../common/response-contract'; +import { JourneyAiPhotosService } from '../journey/journey-ai-photos.service'; +import { AssistantBookContext } from './assistant-book-context'; +import { AssistantRenderRegistry } from './assistant-render.registry'; +import { AssistantRenderUploadDto } from './assistant.dto'; +import { + Body, + Controller, + HttpCode, + HttpException, + Param, + Post, + UploadedFile, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { + ASSISTANT_PAGE_IMAGE, + assistantRenderUploadResponseSchema, + drawnPhotoIdsOf, + type AssistantRenderUploadResponse, +} from '@trek/shared'; + +import { memoryStorage } from 'multer'; + +const UPLOAD = { + storage: memoryStorage(), + limits: { fileSize: ASSISTANT_PAGE_IMAGE.maxBytes, files: 1 }, +}; + +/** A positive integer route id, or null. */ +function routeId(value: string): number | null { + const id = Number(value); + return Number.isInteger(id) && id > 0 ? id : null; +} + +/** + * `POST /api/journeys/:journeyId/assistant/threads/:threadId/renders/:requestId`: + * the Studio page a running turn asked for (journey-studio.md 3.1), drawn by + * the Studio tab that holds the turn's stream and uploaded as one image in + * the field `image`. + * + * It takes a multipart upload, so the addons, the managed install and the + * journey are checked in the handler, as for a file in the trip chat + * (AssistantAttachmentsController): a guard would answer before multer read + * the body. Then the image itself (at most 1 MB by multer, a JPEG, PNG or + * WebP by its first bytes, at most 2048 px on its long edge and 4 MP), the + * photos it says it shows (only the user's own, JourneyAiPhotosService), and + * that a turn of this user on this journey and thread still waits for exactly + * this page and version. The bytes go on as received, decoded by nobody here, + * and are kept nowhere. + */ +@Controller('api/journeys/:journeyId/assistant') +@UseGuards(JwtAuthGuard) +export class AssistantRendersController { + constructor( + private readonly registry: AssistantRenderRegistry, + private readonly books: AssistantBookContext, + private readonly photos: JourneyAiPhotosService, + private readonly addons: AddonsService, + private readonly env: RuntimeEnvService, + ) {} + + @ManagedForbidden('the operator decides which model a managed install talks to, and pays for it', { + enforcedInHandler: true, + }) + @Post('threads/:threadId/renders/:requestId') + @HttpCode(200) + @ResponseContract(assistantRenderUploadResponseSchema) + @UseInterceptors(FileInterceptor('image', UPLOAD)) + async upload( + @CurrentUser() user: User, + @Param('journeyId') journeyParam: string, + @Param('threadId') threadParam: string, + @Param('requestId') requestId: string, + @UploadedFile() file: Express.Multer.File | undefined, + @Body() body: AssistantRenderUploadDto, + ): Promise { + if (!(await this.addons.isAddonEnabled(ADDON_IDS.AI_ASSISTANT))) { + throw new HttpException({ error: 'AI assistant addon is not enabled' }, 404); + } + if (isManagedBlocked(this.env)) throw new HttpException(MANAGED_FORBIDDEN_ERROR, 403); + if (!(await this.addons.isAddonEnabled(ADDON_IDS.JOURNEY))) { + throw new HttpException({ error: 'Journey addon is not enabled' }, 404); + } + const journeyId = routeId(journeyParam); + if (journeyId === null || !(await this.books.canOpen(user.id, journeyId))) { + throw new HttpException({ error: 'Journey not found' }, 404); + } + const threadId = routeId(threadParam); + if (threadId === null) throw new HttpException({ error: 'No page was asked for' }, 404); + if (!file) throw new HttpException({ error: 'No image uploaded' }, 400); + const header = readImageHeader(file.buffer); + if (!header) throw new HttpException({ error: 'The upload is not a JPEG, PNG or WebP image' }, 415); + const { maxEdge, maxPixels } = ASSISTANT_PAGE_IMAGE; + if (Math.max(header.width, header.height) > maxEdge || header.width * header.height > maxPixels) { + throw new HttpException({ error: `The image is larger than ${maxEdge} px or ${maxPixels / 1_000_000} MP` }, 413); + } + const own = await this.photos.ownPhotoIds(journeyId, user.id); + if (drawnPhotoIdsOf(body.drawnPhotoIds).some((photoId) => !own.has(photoId))) { + throw new HttpException({ error: 'The image shows a photo another member contributed' }, 403); + } + const delivered = this.registry.deliver( + { userId: user.id, journeyId, threadId, requestId }, + { spreadId: body.spreadId, bookVersion: body.bookVersion }, + { mimeType: header.mimeType, data: file.buffer, width: header.width, height: header.height }, + ); + if (delivered === 'unknown') throw new HttpException({ error: 'No page was asked for' }, 404); + if (delivered === 'mismatch') { + throw new HttpException({ error: 'The image shows another page or version than the one asked for' }, 409); + } + return { accepted: true }; + } +} diff --git a/server/src/nest/assistant/assistant-vision.ts b/server/src/nest/assistant/assistant-vision.ts new file mode 100644 index 000000000..824f30dd8 --- /dev/null +++ b/server/src/nest/assistant/assistant-vision.ts @@ -0,0 +1,291 @@ +import type { AiChatMessage, AiImage } from '../ai-connections/ai-chat-client.interface'; +import { AiConnectionsService } from '../ai-connections/ai-connections.service'; +import type { ResolvedAiConnection } from '../ai-connections/ai-connections.types'; +import { modelReadsImages } from '../ai-connections/model-catalog.service'; +import { logDebug } from '../audit/audit-log.logger'; +import { DomainError } from '../common/domain-error'; +import { withRequestContext } from '../database/request-context'; +import { JourneyAiPhotosService } from '../journey/journey-ai-photos.service'; +import { JourneyBookEditService, type BookTrialBase } from '../journey/journey-book-edit.service'; +import type { AgentToolPort, AgentToolRun, AgentVision } from './assistant-agent'; +import { withPagePreview } from './assistant-book-preview'; +import { AssistantRenderRegistry, type PageImage } from './assistant-render.registry'; +import type { UiStreamWriter } from './ui-stream.writer'; +import { MikroORM } from '@mikro-orm/core'; +import { Injectable } from '@nestjs/common'; +import { + applyBookOps, + ASSISTANT_PAGE_IMAGE, + bookWriteResultSchema, + describeSpread, + type AssistantPageView, + type AssistantScope, + type BookDocument, + type BookElement, + type BookPageOp, +} from '@trek/shared'; + +import { randomUUID } from 'node:crypto'; + +/** + * The Studio assistant looking at a page (journey-studio.md 3): the page in + * words for every model, and for a model that reads images the page as the + * Studio draws it. The server cannot draw a page, so a look streams a render + * request to the Studio tab that holds the turn and waits a bounded time for + * its upload (AssistantRenderRegistry); no image in time is the description + * alone, never an error. + * + * Images go only when all of this holds for the turn: the client draws pages + * (the Studio's chat says so with `renders`), the model reads images (by its + * kind and id, and not refused one before in this process), and the user let + * this connection receive page images (their Studio acknowledgement). Only the + * user's own photos are drawn; the others go as empty frames, before anything + * is rasterised, and the description says so. Nothing of an image is stored: + * the message keeps what was looked at (`data-trek-page-view`), the bytes live + * for the turn's next model request. + */ + +/** One Studio turn's view of its pages. */ +export interface TurnVision extends AgentVision { + /** Whether this turn sends page images to the model, as of now. */ + sendsImages(): boolean; + /** + * One page as it stands or, with `ops`, as an `edit_book_page` with them + * would leave it, nothing saved; with `designId` the page of that design + * draft (design_book) instead of the book's. + */ + look(spreadId: string, ops?: BookPageOp[], designId?: string): Promise; +} + +/** What a Studio turn brings to its pages. */ +export interface VisionTurnInput { + userId: number; + journeyId: number; + threadId: number; + connection: Pick; + /** The client draws pages for this turn (the Studio's chat). */ + renders: boolean; + w: UiStreamWriter; + signal: AbortSignal; +} + +const NO_BOOK = 'This journey has no book yet: create_book makes one.'; +const LATE = 'The Studio did not send the page image in time; check the page on its description.'; + +/** A page image as a model request carries it. */ +function asAiImage(image: PageImage): AiImage { + return { mimeType: image.mimeType, data: image.data.toString('base64') }; +} + +/** The photos an element shows: a photo frame's own, and the pictures a map draws its stops with. */ +function photosOf(el: BookElement): number[] { + if (el.kind === 'photo') return el.photoId === null ? [] : [el.photoId]; + if (el.kind === 'map') return el.points.flatMap((point) => (point.photoId === null ? [] : [point.photoId])); + return []; +} + +/** The photos on a page that the user did not contribute: drawn as empty frames (or plain map stops). */ +function hiddenOn(doc: BookDocument, spreadId: string, own: ReadonlySet): number[] { + const spread = doc.spreads.find((s) => s.id === spreadId); + const ids = new Set(); + for (const el of spread?.elements ?? []) { + for (const photoId of photosOf(el)) if (!own.has(photoId)) ids.add(photoId); + } + return [...ids]; +} + +/** The page a successful book write left, from its answer as the model read it; null for anything else. */ +function writtenPage(message: AiChatMessage): string | null { + if (message.role !== 'tool' || message.isError) return null; + let parsed: unknown; + try { + parsed = JSON.parse(message.output); + } catch { + return null; + } + const result = bookWriteResultSchema.safeParse(parsed); + return result.success ? (result.data.spread_ids[0] ?? null) : null; +} + +/** The page as a proposed edit would leave it (the stored one without ops), or why the book refuses the edit. */ +function pageAfter( + doc: BookDocument, + spreadId: string, + ops: BookPageOp[] | undefined, + base: BookTrialBase, +): BookDocument | string { + if (!ops) return doc; + const made = applyBookOps(doc, { spreadId, ops }, base.ctx); + if (made.ok) return made.doc; + const at = made.error.opIndex === null ? '' : ` (operation ${made.error.opIndex})`; + return `Not a change the book takes${at}: ${made.error.code}: ${made.error.message}`; +} + +/** What the message keeps of a look: the page, the version, the image's size when one came, the description. */ +function viewOf(spreadId: string, bookVersion: number, text: string, image: PageImage | null): AssistantPageView { + return { + spreadId, + bookVersion, + width: image?.width ?? null, + height: image?.height ?? null, + bytes: image ? image.data.length : null, + description: text.slice(0, 20_000), + }; +} + +/** Why a look found nothing, in the words a tool would answer. */ +function reason(err: unknown): string { + if (err instanceof DomainError) return err.mcpMessage ?? err.publicMessage; + return err instanceof Error ? err.message : String(err); +} + +/** Builds each Studio turn's vision, and remembers the models a provider refused an image for. */ +@Injectable() +export class AssistantVision { + constructor( + private readonly connections: AiConnectionsService, + private readonly books: JourneyBookEditService, + private readonly photos: JourneyAiPhotosService, + private readonly registry: AssistantRenderRegistry, + private readonly orm: MikroORM, + ) {} + + /** `kind:model` of the models a provider refused an image for, while this process runs. */ + private readonly textOnly = new Set(); + + /** A turn's tools and its vision: a Studio turn's with the page preview beside them, a trip turn's as they are. */ + async turnTools( + scope: AssistantScope, + session: AgentToolPort, + input: Omit, + ): Promise<{ tools: AgentToolPort; vision: TurnVision | null }> { + if (scope.kind !== 'journey') return { tools: session, vision: null }; + const vision = await this.forTurn({ ...input, journeyId: scope.journeyId }); + return { tools: withPagePreview(session, vision), vision }; + } + + async forTurn(input: VisionTurnInput): Promise { + const model = `${input.connection.kind}:${input.connection.model}`; + let images = + input.renders && + modelReadsImages(input.connection.kind, input.connection.model) && + !this.textOnly.has(model) && + (await withRequestContext(this.orm, () => this.connections.studioImagesAcked(input.userId, input.connection))); + const vision: TurnVision = { + sendsImages: () => images, + look: (spreadId, ops, designId) => this.look(input, images, spreadId, { ops, designId }), + refused: () => { + images = false; + this.textOnly.add(model); + }, + afterApply: (history) => (images ? this.afterApply(vision, history) : Promise.resolve([...history])), + }; + return vision; + } + + /** + * The history a decision's turn starts from, with a view of each page the + * applied book writes left attached to the write's own answer (4.3, step + * 3), so the model checks what the user took before it goes on. At most as + * many as one request carries, one image each, the same page once. + */ + private async afterApply(vision: TurnVision, history: readonly AiChatMessage[]): Promise { + const out = [...history]; + let start = out.length; + while (start > 0 && out[start - 1]?.role === 'tool') start--; + const seen = new Set(); + const targets: Array<{ index: number; spreadId: string }> = []; + for (let index = out.length - 1; index >= start && targets.length < ASSISTANT_PAGE_IMAGE.perRequest; index--) { + const message = out[index]; + const spreadId = message ? writtenPage(message) : null; + if (spreadId && !seen.has(spreadId)) { + seen.add(spreadId); + targets.push({ index, spreadId }); + } + } + // One after another: the Studio draws one page at a time. + await targets.reduce>(async (previous, { index, spreadId }) => { + await previous; + const seenPage = await vision.look(spreadId); + const message = out[index]; + if (seenPage.images?.length && message?.role === 'tool') out[index] = { ...message, images: seenPage.images }; + }, Promise.resolve()); + return out; + } + + private async look( + input: VisionTurnInput, + withImage: boolean, + spreadId: string, + { ops, designId }: { ops?: BookPageOp[]; designId?: string }, + ): Promise { + let base: BookTrialBase; + let own: ReadonlySet | null; + const { journeyId, userId } = input; + try { + // A draft's page is read from the draft at the version it was made against; the book's as it stands. + base = await withRequestContext(this.orm, () => + designId ? this.books.trialBase(journeyId, userId, designId) : this.books.trialBase(journeyId, userId), + ); + // Whose photos the image may show; the journey can have gone away since the turn began. + own = withImage + ? await withRequestContext(this.orm, () => this.photos.ownPhotoIds(input.journeyId, input.userId)) + : null; + } catch (err) { + return { ok: false, value: reason(err) }; + } + if (!base.doc || base.version === null) return { ok: false, value: NO_BOOK }; + const doc = pageAfter(base.doc, spreadId, ops, base); + if (typeof doc === 'string') return { ok: false, value: doc }; + const hidden = own ? hiddenOn(doc, spreadId, own) : []; + // A page naming more photos than a render request may list is checked on its description alone. + const draws = withImage && hidden.length <= ASSISTANT_PAGE_IMAGE.maxPagePhotos; + const photos = (base.ctx.photos ?? []).map((p) => (hidden.includes(p.photoId) ? { ...p, hidden: true } : p)); + const described = describeSpread(doc, spreadId, photos); + if (!described) return { ok: false, value: `This book has no page ${spreadId}: get_book lists its pages.` }; + const image = draws + ? await this.render(input, { spreadId, bookVersion: base.version, ops, designId, hidden }) + : null; + const value = { + spreadId, + index: described.index, + role: described.role, + proposed: !!ops, + description: described.text, + findings: described.findings, + ...(withImage ? { image: image ? 'The page as the Studio draws it is attached.' : LATE } : {}), + }; + const view = viewOf(spreadId, base.version, described.text, image); + return { ok: true, value, view, ...(image ? { images: [asAiImage(image)] } : {}) }; + } + + /** Asks the Studio tab for the page and waits for it; null when none came in time or the turn stopped. */ + private async render( + input: VisionTurnInput, + page: { spreadId: string; bookVersion: number; ops?: BookPageOp[]; designId?: string; hidden: number[] }, + ): Promise { + const requestId = randomUUID(); + const { userId, journeyId, threadId, signal } = input; + const { spreadId, bookVersion, ops, designId, hidden } = page; + // Waiting before asking: an upload that comes back at once finds its turn. + const waiting = this.registry.wait( + { userId, journeyId, threadId, requestId, spreadId, bookVersion }, + signal, + ASSISTANT_PAGE_IMAGE.timeoutMs, + ); + // A draft's page: the Studio draws it from the draft (GET .../book-designs/:designId). + const asked = { + requestId, + threadId, + spreadId, + bookVersion, + hiddenPhotoIds: hidden, + ...(designId ? { designId } : {}), + }; + input.w.renderRequest(ops ? { ...asked, ops } : asked); + const image = await waiting; + // Size and dimensions only: the page shows the user's photos. + if (image) logDebug(`[assistant] page image ${image.width}x${image.height}, ${image.data.length} bytes`); + return image; + } +} diff --git a/server/src/nest/assistant/ui-stream.writer.ts b/server/src/nest/assistant/ui-stream.writer.ts index 6d146cc2d..209ad7afe 100644 --- a/server/src/nest/assistant/ui-stream.writer.ts +++ b/server/src/nest/assistant/ui-stream.writer.ts @@ -6,6 +6,8 @@ import { type AssistantCard, type AssistantChangePreview, type AssistantErrorCode, + type AssistantPageRender, + type AssistantPageView, type AssistantPlaces, type AssistantWeather, type UiStreamChunk, @@ -108,6 +110,17 @@ export class UiStreamWriter { this.send({ type: 'data-trek-activity', data: activity, transient: true }); } + /** A Studio page the turn asks the Studio tab to draw and upload; only streamed, the message never keeps it. */ + renderRequest(request: AssistantPageRender): void { + this.endText(); + this.send({ type: 'data-trek-render-request', data: request, transient: true }); + } + + /** A Studio page the turn looked at: what it was and its description, never the image. */ + pageView(view: AssistantPageView): void { + this.send({ type: 'data-trek-page-view', data: view }); + } + quota(used: number, limit: number): void { this.send({ type: 'data-trek-quota', data: { used, limit } }); } diff --git a/server/src/nest/common/image-header.ts b/server/src/nest/common/image-header.ts new file mode 100644 index 000000000..a82f95f27 --- /dev/null +++ b/server/src/nest/common/image-header.ts @@ -0,0 +1,86 @@ +/** + * What an image is and how large, read from its first bytes without decoding + * it. Shared by the domains that take an image from outside and must check it + * before anything decodes or sends it: the photo imports of AI parsing and + * the Studio page images the assistant hands a model. + */ + +export interface ImageHeader { + mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; + width: number; + height: number; +} + +/** JPEG start-of-frame markers: every SOFn except DHT (C4), JPG (C8) and DAC (CC). */ +function isStartOfFrame(marker: number): boolean { + return marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; +} + +function jpegSize(data: Buffer): { width: number; height: number } | null { + let i = 2; + while (i + 4 <= data.length) { + // Stray bytes between segments, which some cameras write, are stepped over. + if (data[i] !== 0xff) { + i += 1; + continue; + } + const marker = data[i + 1]; + // Fill bytes, and the markers that carry no length. + if (marker === 0xff) { + i += 1; + continue; + } + if (marker === 0x01 || marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7)) { + i += 2; + continue; + } + // The scan or the end came before any frame header. + if (marker === 0xd9 || marker === 0xda) return null; + const length = data.readUInt16BE(i + 2); + if (length < 2) return null; + if (isStartOfFrame(marker)) { + if (i + 9 > data.length) return null; + return { height: data.readUInt16BE(i + 5), width: data.readUInt16BE(i + 7) }; + } + i += 2 + length; + } + return null; +} + +function webpSize(data: Buffer): { width: number; height: number } | null { + const chunk = data.toString('latin1', 12, 16); + if (chunk === 'VP8 ' && data.length >= 30) { + return { width: data.readUInt16LE(26) & 0x3fff, height: data.readUInt16LE(28) & 0x3fff }; + } + if (chunk === 'VP8L' && data.length >= 25) { + const bits = data.readUInt32LE(21); + return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 }; + } + if (chunk === 'VP8X' && data.length >= 30) { + return { width: data.readUIntLE(24, 3) + 1, height: data.readUIntLE(27, 3) + 1 }; + } + return null; +} + +/** + * PNG from IHDR, JPEG from its frame header, WebP from its VP8, VP8L or VP8X + * chunk. Null when the bytes are none of the three, or do not say. + */ +export function readImageHeader(data: Buffer): ImageHeader | null { + let found: ImageHeader | null = null; + if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47 && data.toString('latin1', 12, 16) === 'IHDR') { + found = { mimeType: 'image/png', width: data.readUInt32BE(16), height: data.readUInt32BE(20) }; + } else if (data.length >= 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) { + const size = jpegSize(data); + found = size && { mimeType: 'image/jpeg', ...size }; + } else if ( + data.length >= 16 && + data.toString('latin1', 0, 4) === 'RIFF' && + data.toString('latin1', 8, 12) === 'WEBP' + ) { + const size = webpSize(data); + found = size && { mimeType: 'image/webp', ...size }; + } + // A zero edge is a header that defers its size (a JPEG's DNL), which says nothing. + return found && found.width > 0 && found.height > 0 ? found : null; +} diff --git a/server/src/nest/common/validate-managed-routes.ts b/server/src/nest/common/validate-managed-routes.ts index 351f4ff34..7e3307f66 100644 --- a/server/src/nest/common/validate-managed-routes.ts +++ b/server/src/nest/common/validate-managed-routes.ts @@ -97,11 +97,19 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [ 'AssistantController.thread', 'AssistantController.threads', 'AssistantController.undo', + 'AssistantRendersController.upload', 'AuthController.createMcpToken', 'AuthController.validateKeys', 'BackupController.restore', 'BackupController.updateAutoSettings', 'BackupController.uploadRestore', + 'JourneyAssistantController.chat', + 'JourneyAssistantController.decide', + 'JourneyAssistantController.deleteThread', + 'JourneyAssistantController.stop', + 'JourneyAssistantController.thread', + 'JourneyAssistantController.threads', + 'JourneyAssistantController.undo', 'LlmLocalController.models', 'LlmLocalController.pull', 'NotificationsController.testSmtp', diff --git a/server/src/nest/journey/journey-ai-photos.service.ts b/server/src/nest/journey/journey-ai-photos.service.ts new file mode 100644 index 000000000..9a17fdbc8 --- /dev/null +++ b/server/src/nest/journey/journey-ai-photos.service.ts @@ -0,0 +1,35 @@ +import { JourneyPhotos } from '../../db/entities/JourneyPhotos.entity'; +import type { JourneyPhotosRepository } from '../../db/repositories/JourneyPhotos.repository'; +import { DomainError } from '../common/domain-error'; +import { JourneyDomainService } from './journey-domain.service'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; + +/** + * Which photos of a journey's gallery the Studio assistant may show a model + * that reads images, for the user asking (journey-studio.md 3.4). + * + * TREK cannot tell who is in a photo; it knows who contributed it. So the + * rule is strict: only the photos the asking user contributed, as + * `trek_photos.owner_id` says, with the journey's owner standing in for a + * photo that names nobody (the fallback the photo proxy uses). Every other + * photo is drawn as an empty frame before the page is rasterised, and an + * uploaded page image that says it shows one is refused. + */ +@Injectable() +export class JourneyAiPhotosService { + constructor( + private readonly journey: JourneyDomainService, + @InjectRepository(JourneyPhotos) private readonly photosRepo: JourneyPhotosRepository, + ) {} + + /** The gallery photos the user contributed; refused like the journey routes for a journey they cannot open. */ + async ownPhotoIds(journeyId: number, userId: number): Promise> { + const journey = await this.journey.canAccessJourney(journeyId, userId); + if (!journey) { + throw new DomainError(404, 'Journey not found', { mcpMessage: 'Journey not found or access denied.' }); + } + const gallery = await this.photosRepo.galleryRead(journeyId); + return new Set(gallery.filter((p) => (p.owner_id || journey.user_id) === userId).map((p) => p.photo_id)); + } +} diff --git a/server/src/nest/llm-parse/image-input.ts b/server/src/nest/llm-parse/image-input.ts index 4a1455891..0e5b5452f 100644 --- a/server/src/nest/llm-parse/image-input.ts +++ b/server/src/nest/llm-parse/image-input.ts @@ -1,3 +1,4 @@ +import { readImageHeader } from '../common/image-header'; import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'; import { Jimp } from 'jimp'; @@ -46,87 +47,6 @@ export const IMAGE_MAX_BYTES = Math.floor(3.5 * 1024 * 1024); /** A photo TREK will not decode or send: the message says why, for the import's warning. */ export class ImageTooLargeError extends Error {} -export interface ImageHeader { - mimeType: 'image/png' | 'image/jpeg' | 'image/webp'; - width: number; - height: number; -} - -/** JPEG start-of-frame markers: every SOFn except DHT (C4), JPG (C8) and DAC (CC). */ -function isStartOfFrame(marker: number): boolean { - return marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; -} - -function jpegSize(data: Buffer): { width: number; height: number } | null { - let i = 2; - while (i + 4 <= data.length) { - // Stray bytes between segments, which some cameras write, are stepped over. - if (data[i] !== 0xff) { - i += 1; - continue; - } - const marker = data[i + 1]; - // Fill bytes, and the markers that carry no length. - if (marker === 0xff) { - i += 1; - continue; - } - if (marker === 0x01 || marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7)) { - i += 2; - continue; - } - // The scan or the end came before any frame header. - if (marker === 0xd9 || marker === 0xda) return null; - const length = data.readUInt16BE(i + 2); - if (length < 2) return null; - if (isStartOfFrame(marker)) { - if (i + 9 > data.length) return null; - return { height: data.readUInt16BE(i + 5), width: data.readUInt16BE(i + 7) }; - } - i += 2 + length; - } - return null; -} - -function webpSize(data: Buffer): { width: number; height: number } | null { - const chunk = data.toString('latin1', 12, 16); - if (chunk === 'VP8 ' && data.length >= 30) { - return { width: data.readUInt16LE(26) & 0x3fff, height: data.readUInt16LE(28) & 0x3fff }; - } - if (chunk === 'VP8L' && data.length >= 25) { - const bits = data.readUInt32LE(21); - return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 }; - } - if (chunk === 'VP8X' && data.length >= 30) { - return { width: data.readUIntLE(24, 3) + 1, height: data.readUIntLE(27, 3) + 1 }; - } - return null; -} - -/** - * What a photo is and how large, read from its first bytes without decoding it: - * PNG from IHDR, JPEG from its frame header, WebP from its VP8, VP8L or VP8X - * chunk. Null when the bytes are none of the three, or do not say. - */ -export function readImageHeader(data: Buffer): ImageHeader | null { - let found: ImageHeader | null = null; - if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47 && data.toString('latin1', 12, 16) === 'IHDR') { - found = { mimeType: 'image/png', width: data.readUInt32BE(16), height: data.readUInt32BE(20) }; - } else if (data.length >= 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) { - const size = jpegSize(data); - found = size && { mimeType: 'image/jpeg', ...size }; - } else if ( - data.length >= 16 && - data.toString('latin1', 0, 4) === 'RIFF' && - data.toString('latin1', 8, 12) === 'WEBP' - ) { - const size = webpSize(data); - found = size && { mimeType: 'image/webp', ...size }; - } - // A zero edge is a header that defers its size (a JPEG's DNL), which says nothing. - return found && found.width > 0 && found.height > 0 ? found : null; -} - /** Bytes sent as they came, as long as a provider takes them at that size. */ function asTheyCame(data: Buffer, mimeType: string): { data: Buffer; mimeType: string } { if (data.length > IMAGE_MAX_BYTES) { diff --git a/server/tests/e2e/assistant-page-images.e2e.test.ts b/server/tests/e2e/assistant-page-images.e2e.test.ts new file mode 100644 index 000000000..c5d9b245a --- /dev/null +++ b/server/tests/e2e/assistant-page-images.e2e.test.ts @@ -0,0 +1,343 @@ +/** + * The Studio page image reaching the model, per provider kind, end to end: + * the whole app (buildApp, every global guard, the Zod pipe, the real MCP + * registry, the real agent and vision) with the provider's own adapter + * (Anthropic Messages, Chat Completions, the Responses API of a ChatGPT plan) + * talking to a scripted fetch instead of the network. + * + * What is stood in for, and why: the connection the turn resolves names the + * kind under test (an instance can only point a `custom` connection at + * loopback), the user's Studio acknowledgement is given, and the Studio tab's + * upload is the render registry answering at once with a page image (the + * upload route itself is ASTRNE2E's). Everything between, from the look to + * the request on the wire, is the real code. + * + * Pinned: a model that reads images gets the page in the place its provider + * reads it (inside the tool_result for Anthropic, a user message after the + * step's tool answers for Chat Completions and the Responses API), the turn + * streams one render request and keeps the image's size, never its bytes; a + * kind that cannot be told to read images (OpenRouter) asks for no image and + * checks the page on its description. + */ +import { buildApp } from '../../src/bootstrap'; +import { AiChatClientService } from '../../src/nest/ai-connections/ai-chat-client.service'; +import { AiConnectionsService } from '../../src/nest/ai-connections/ai-connections.service'; +import type { ResolvedAiConnection } from '../../src/nest/ai-connections/ai-connections.types'; +import { AnthropicMessagesClient } from '../../src/nest/ai-connections/providers/anthropic-messages.client'; +import { OpenAiChatClient } from '../../src/nest/ai-connections/providers/openai-chat.client'; +import { OpenAiResponsesClient } from '../../src/nest/ai-connections/providers/openai-responses.client'; +import type { AiFetch } from '../../src/nest/ai-connections/providers/provider-http'; +import { AssistantRenderRegistry, type PageImage } from '../../src/nest/assistant/assistant-render.registry'; +import { JourneyBookEditService } from '../../src/nest/journey/journey-book-edit.service'; +import { makeJourney } from '../helpers/factories/journeys'; +import { setAddonEnabled } from '../helpers/factories/settings'; +import { makeUser } from '../helpers/factories/users'; +import { resetRateLimits } from '../helpers/test-db'; +import { sessionCookie } from './harness'; +import { MikroORM } from '@mikro-orm/core'; +import type { INestApplication } from '@nestjs/common'; +import type { AiConnectionKind, UiStreamChunk } from '@trek/shared'; + +import { createServer, type IncomingMessage, type Server as HttpServer, type ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +// ── The page the Studio "uploads" ──────────────────────────────────────────── + +const JPEG = Buffer.from('ffd8ffe000104a46494600010100ffd9', 'hex'); +const PAGE: PageImage = { mimeType: 'image/jpeg', data: JPEG, width: 1280, height: 640 }; +const DATA_URL = `data:image/jpeg;base64,${JPEG.toString('base64')}`; + +// ── The provider, per kind: what each adapter streams back ─────────────────── + +/** An SSE answer from whole events, as the adapters read it. */ +function sse(events: unknown[]): Response { + const text = events.map((e) => `data: ${typeof e === 'string' ? e : JSON.stringify(e)}\n\n`).join(''); + return new Response(text, { status: 200, headers: { 'content-type': 'text/event-stream' } }); +} + +type Wire = 'anthropic' | 'chat' | 'responses'; + +const PREVIEW = 'preview_book_page'; + +/** The model looks at a page with one call, as each provider streams a tool call. */ +function lookAt(wire: Wire, args: Record): Response { + const json = JSON.stringify(args); + if (wire === 'anthropic') { + return sse([ + { type: 'message_start', message: { usage: { input_tokens: 10, output_tokens: 1 } } }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'call_look', name: PREVIEW }, + }, + { type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: json } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 20 } }, + { type: 'message_stop' }, + ]); + } + if (wire === 'chat') { + const call = { index: 0, id: 'call_look', type: 'function', function: { name: PREVIEW, arguments: json } }; + return sse([ + { choices: [{ index: 0, delta: { tool_calls: [call] } }] }, + { choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] }, + { choices: [], usage: { prompt_tokens: 12, completion_tokens: 4 } }, + '[DONE]', + ]); + } + const item = { type: 'function_call', id: 'fc_1', call_id: 'call_look', name: PREVIEW, arguments: json }; + return sse([ + { type: 'response.created', response: {} }, + { type: 'response.output_item.done', item: { ...item, namespace: 'trek' } }, + { type: 'response.completed', response: { usage: { input_tokens: 50, output_tokens: 6 } } }, + ]); +} + +/** The model's last words, as each provider streams text. */ +function says(wire: Wire, text: string): Response { + if (wire === 'anthropic') { + return sse([ + { type: 'message_start', message: { usage: { input_tokens: 10, output_tokens: 1 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 4 } }, + { type: 'message_stop' }, + ]); + } + if (wire === 'chat') { + return sse([ + { choices: [{ index: 0, delta: { content: text } }] }, + { choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, + { choices: [], usage: { prompt_tokens: 12, completion_tokens: 4 } }, + '[DONE]', + ]); + } + return sse([ + { type: 'response.created', response: {} }, + { type: 'response.output_text.delta', delta: text }, + { type: 'response.completed', response: { usage: { input_tokens: 50, output_tokens: 6 } } }, + ]); +} + +/** What the adapters sent, request by request, and what they are answered next. */ +const wire = { bodies: [] as Array>, answers: [] as Response[] }; + +const scripted: AiFetch = async (_url, init) => { + wire.bodies.push(JSON.parse(String(init.body)) as Record); + const next = wire.answers.shift(); + if (!next) throw new Error('the scripted provider has no answer left'); + return next; +}; + +/** The real adapter of a kind, on the scripted fetch. */ +function adapterFor(kind: AiConnectionKind, model: string) { + if (kind === 'anthropic') return new AnthropicMessagesClient(model, 'sk-ant-test', scripted); + if (kind === 'chatgpt') return new OpenAiResponsesClient(model, async () => 'at-test', scripted); + return new OpenAiChatClient(kind, model, 'https://provider.invalid/v1', 'sk-test', scripted); +} + +// ── A loopback server, only so the instance connection can be saved ───────── + +async function answerModels(req: IncomingMessage, res: ServerResponse): Promise { + if (req.method === 'GET' && req.url?.startsWith('/api/tags')) { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ models: [{ name: 'fake-model', size: 1 }] })); + return; + } + res.writeHead(404).end(); +} + +// ── The app ────────────────────────────────────────────────────────────────── + +let nestApp: INestApplication; +let orm: MikroORM; +let fake: HttpServer; +let books: JourneyBookEditService; +let connections: AiConnectionsService; +let chatClients: AiChatClientService; +let registry: AssistantRenderRegistry; + +beforeAll(async () => { + fake = createServer((req, res) => { + answerModels(req, res).catch(() => res.destroy()); + }); + await new Promise((resolve) => fake.listen(0, '127.0.0.1', resolve)); + nestApp = await buildApp(); + orm = nestApp.get(MikroORM); + books = nestApp.get(JourneyBookEditService); + connections = nestApp.get(AiConnectionsService); + chatClients = nestApp.get(AiChatClientService); + registry = nestApp.get(AssistantRenderRegistry); + await setAddonEnabled(orm, 'ai_assistant', true); + await setAddonEnabled(orm, 'journey', true); + const adminId = (await makeUser(orm, { role: 'admin' })).user.id; + const { port } = fake.address() as AddressInfo; + const res = await request(nestApp.getHttpServer()) + .put('/api/admin/ai/connection') + .set('Cookie', sessionCookie(adminId)) + .send({ kind: 'custom', baseUrl: `http://127.0.0.1:${port}/v1`, model: 'fake-model' }); + expect(res.status).toBe(200); +}); + +beforeEach(async () => { + await resetRateLimits(nestApp); + vi.restoreAllMocks(); + wire.bodies = []; + wire.answers = []; +}); + +afterAll(async () => { + vi.restoreAllMocks(); + await nestApp.close(); + await new Promise((resolve) => fake.close(() => resolve())); +}); + +/** The turn runs on `kind`'s adapter as if that connection were saved, the Studio images acknowledged. */ +function runOn(kind: AiConnectionKind, model: string) { + const resolve = connections.resolveForUser.bind(connections); + vi.spyOn(connections, 'resolveForUser').mockImplementation(async (userId: number): Promise => ({ + ...(await resolve(userId)), + kind, + model, + })); + vi.spyOn(connections, 'studioImagesAcked').mockResolvedValue(true); + vi.spyOn(chatClients, 'create').mockImplementation(() => adapterFor(kind, model)); + return vi.spyOn(registry, 'wait').mockResolvedValue(PAGE); +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const CTX = { locale: 'en', timeZone: 'Europe/Lisbon' }; + +async function stream(path: string, userId: number, body: object): Promise { + const res = await request(nestApp.getHttpServer()) + .post(path) + .set('Cookie', sessionCookie(userId)) + .send(body) + .buffer(true) + .parse((s, done) => { + let text = ''; + s.setEncoding('utf8'); + s.on('data', (chunk: string) => (text += chunk)); + s.on('end', () => done(null, text)); + }); + expect(res.status).toBe(200); + return (res.body as string) + .split('\n\n') + .filter((event) => event.startsWith('data: ') && event !== 'data: [DONE]') + .map((event) => JSON.parse(event.slice('data: '.length)) as UiStreamChunk); +} + +const ofType = (chunks: UiStreamChunk[], type: T) => + chunks.filter((chunk): chunk is Extract => chunk.type === type); + +/** A user with a journey of their own and a blank book in it: its double page is `inner`. */ +async function ownerWithBook() { + const { user } = await makeUser(orm); + const journey = await makeJourney(orm, user.id, { title: 'Portugal by train' }); + const created = await books.createBook(journey.id, user.id, {}); + const inner = created.pages.find((page) => page.role === 'inner')?.id ?? 'no inner page'; + return { userId: user.id, journeyId: journey.id, inner, base: `/api/journeys/${journey.id}/assistant` }; +} + +/** One Studio turn on `kind`: the model previews warmer paper on the inner page, then answers. */ +async function lookTurn(kind: AiConnectionKind, model: string, via: Wire) { + const j = await ownerWithBook(); + const waited = runOn(kind, model); + const warmer = { spreadId: j.inner, ops: [{ op: 'background', color: '#f4efe6' }] }; + wire.answers = [lookAt(via, warmer), says(via, 'Fine.')]; + const turn = await stream(`${j.base}/chat`, j.userId, { ...CTX, threadId: null, text: 'Warmer?', renders: true }); + expect(wire.bodies).toHaveLength(2); + return { j, turn, waited, sent: wire.bodies[1] ?? {} }; +} + +/** A list in a request body, empty when that key holds none. */ +const listIn = (body: Record, key: string): Array> => { + const value = body[key]; + return Array.isArray(value) ? (value as Array>) : []; +}; + +/** The turn asked the Studio once for the page and kept the image's size, never its bytes. */ +function expectOneImage(turn: UiStreamChunk[], spreadId: string) { + const [asked, ...more] = ofType(turn, 'data-trek-render-request'); + expect(more).toEqual([]); + expect(asked?.data).toMatchObject({ spreadId, bookVersion: 1, hiddenPhotoIds: [] }); + const [view] = ofType(turn, 'data-trek-page-view'); + expect(view?.data).toMatchObject({ spreadId, bookVersion: 1, width: 1280, height: 640, bytes: JPEG.length }); + expect(JSON.stringify(ofType(turn, 'tool-output-available'))).toContain('is attached'); + expect(JSON.stringify(turn)).not.toContain(JPEG.toString('base64')); +} + +// ── Cases ──────────────────────────────────────────────────────────────────── + +describe('the Studio page image per provider kind', () => { + it('ASTIMGE2E-001: Anthropic gets the page as an image block inside the tool_result of the look', async () => { + const { j, turn, waited, sent } = await lookTurn('anthropic', 'claude-sonnet-4-5', 'anthropic'); + expect(waited).toHaveBeenCalledTimes(1); + expectOneImage(turn, j.inner); + const results = listIn(sent, 'messages').flatMap((message) => + Array.isArray(message.content) ? (message.content as Array>) : [], + ); + const result = results.find((block) => block.type === 'tool_result' && block.tool_use_id === 'call_look'); + expect(result?.content).toEqual([ + { type: 'text', text: expect.stringContaining('is attached') }, + { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: JPEG.toString('base64') } }, + ]); + }); + + /** Chat Completions: the tool answer stays text, the page follows in a user message of its own. */ + async function expectChatImage(kind: AiConnectionKind, model: string) { + const { j, turn, sent } = await lookTurn(kind, model, 'chat'); + expectOneImage(turn, j.inner); + const messages = listIn(sent, 'messages'); + const answer = messages.findIndex((m) => m.role === 'tool' && m.tool_call_id === 'call_look'); + expect(answer).toBeGreaterThan(0); + expect(typeof messages[answer]?.content).toBe('string'); + expect(messages[answer + 1]).toEqual({ + role: 'user', + content: [ + { type: 'text', text: 'The image the call preview_book_page (call_look) produced:' }, + { type: 'image_url', image_url: { url: DATA_URL, detail: 'high' } }, + ], + }); + } + + it('ASTIMGE2E-002: OpenAI keeps the tool answer text and sends the page in a user message after it', async () => { + await expectChatImage('openai', 'gpt-4o'); + }); + + it('ASTIMGE2E-005: Gemini, on the same Chat Completions shape, gets the page the same way', async () => { + await expectChatImage('gemini', 'gemini-2.5-flash'); + }); + + it('ASTIMGE2E-003: a ChatGPT plan (Responses API) gets the output as text, then the page as an image', async () => { + const { j, turn, sent } = await lookTurn('chatgpt', 'gpt-5', 'responses'); + expectOneImage(turn, j.inner); + const input = listIn(sent, 'input'); + const output = input.findIndex((item) => item.type === 'function_call_output' && item.call_id === 'call_look'); + expect(output).toBeGreaterThan(0); + expect(typeof input[output]?.output).toBe('string'); + expect(input[output + 1]).toEqual({ + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'The image the call preview_book_page (call_look) produced:' }, + { type: 'input_image', image_url: DATA_URL, detail: 'high' }, + ], + }); + }); + + it('ASTIMGE2E-004: OpenRouter (vision unknown by id) asks for no image and reads the page in words', async () => { + const { j, turn, waited, sent } = await lookTurn('openrouter', 'some-lab/some-model', 'chat'); + expect(waited).not.toHaveBeenCalled(); + expect(ofType(turn, 'data-trek-render-request')).toEqual([]); + const [view] = ofType(turn, 'data-trek-page-view'); + expect(view?.data).toMatchObject({ spreadId: j.inner, width: null, height: null, bytes: null }); + expect(JSON.stringify(sent)).not.toContain('image_url'); + const answer = listIn(sent, 'messages').find((m) => m.role === 'tool' && m.tool_call_id === 'call_look'); + expect(String(answer?.content)).toContain('Findings'); + }); +}); diff --git a/server/tests/e2e/assistant-renders.e2e.test.ts b/server/tests/e2e/assistant-renders.e2e.test.ts new file mode 100644 index 000000000..ec794d114 --- /dev/null +++ b/server/tests/e2e/assistant-renders.e2e.test.ts @@ -0,0 +1,193 @@ +/** + * The Studio page images, end to end: the whole app (buildApp, every global + * guard, the Zod pipe, the response contracts) with a turn's wait primed on + * the container's own AssistantRenderRegistry, and the page uploaded the way + * the Studio tab uploads it (multipart, the image in `image`, the page and the + * photos it shows as text fields). + * + * Covered: the gates in the handler (the AI addon, the managed install, the + * journey addon, the journey itself), the image checks (none, not an image, + * over the edge or byte cap), a photo of another member refused, and the + * hand-over: only to the waiting turn of the same user and thread, for the + * page and version asked for, once. + */ +import { buildApp } from '../../src/bootstrap'; +import { JourneyPhotos } from '../../src/db/entities/JourneyPhotos.entity'; +import { TrekPhotos } from '../../src/db/entities/TrekPhotos.entity'; +import { AssistantRenderRegistry, type PageImage } from '../../src/nest/assistant/assistant-render.registry'; +import { addJourneyContributor, makeJourney } from '../helpers/factories/journeys'; +import { insertRow } from '../helpers/factories/rows'; +import { setAddonEnabled } from '../helpers/factories/settings'; +import { makeUser } from '../helpers/factories/users'; +import { resetRateLimits } from '../helpers/test-db'; +import { sessionCookie } from './harness'; +import { MikroORM } from '@mikro-orm/core'; +import type { INestApplication } from '@nestjs/common'; + +import { randomUUID } from 'node:crypto'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +let nestApp: INestApplication; +let orm: MikroORM; +let registry: AssistantRenderRegistry; + +beforeAll(async () => { + nestApp = await buildApp(); + orm = nestApp.get(MikroORM); + registry = nestApp.get(AssistantRenderRegistry); + await setAddonEnabled(orm, 'ai_assistant', true); + await setAddonEnabled(orm, 'journey', true); +}); + +beforeEach(async () => { + await resetRateLimits(nestApp); +}); + +afterAll(async () => { + await nestApp.close(); +}); + +/** The first bytes of a baseline JPEG of w x h: all the route reads. */ +function jpeg(w: number, h: number, size = 64): Buffer { + const out = Buffer.alloc(size); + Buffer.from([0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08]).copy(out, 0); + out.writeUInt16BE(h, 7); + out.writeUInt16BE(w, 9); + return out; +} + +async function photoOf(journeyId: number, ownerId: number): Promise { + const photoId = await insertRow(orm, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg' }); + await insertRow(orm, JourneyPhotos, { journey: journeyId, photo: photoId, sort_order: 0, created_at: Date.now() }); + return photoId; +} + +async function studio() { + const { user } = await makeUser(orm); + const { user: friend } = await makeUser(orm); + const journey = await makeJourney(orm, user.id, { title: 'Portugal by train' }); + await addJourneyContributor(orm, journey.id, friend.id, 'editor'); + const own = await photoOf(journey.id, user.id); + const friends = await photoOf(journey.id, friend.id); + return { userId: user.id, friendId: friend.id, journeyId: journey.id, own, friends }; +} + +interface Upload { + userId: number; + journeyId: number | string; + threadId?: number; + requestId: string; + image?: Buffer; + fields?: Record; +} + +function upload(u: Upload) { + const path = `/api/journeys/${u.journeyId}/assistant/threads/${u.threadId ?? 4}/renders/${u.requestId}`; + let req = request(nestApp.getHttpServer()).post(path).set('Cookie', sessionCookie(u.userId)); + for (const [key, value] of Object.entries(u.fields ?? { bookVersion: '3', spreadId: 'sp-2' })) { + req = req.field(key, value); + } + if (u.image === undefined) return req; + return req.attach('image', u.image, { filename: 'page.jpg', contentType: 'image/jpeg' }); +} + +/** A turn of the user waiting for page sp-2 at version 3 of their journey, in thread 4. */ +function waitFor(userId: number, journeyId: number) { + const requestId = randomUUID(); + const stop = new AbortController(); + const image = registry.wait( + { userId, journeyId, threadId: 4, requestId, spreadId: 'sp-2', bookVersion: 3 }, + stop.signal, + 15_000, + ); + return { requestId, image, stop }; +} + +describe('Studio page images e2e', () => { + it('ASTRNE2E-001: the addons, the managed install and the journey are checked before the image', async () => { + const s = await studio(); + const { user: stranger } = await makeUser(orm); + const turn = waitFor(s.userId, s.journeyId); + const page = { requestId: turn.requestId, image: jpeg(1280, 640) }; + await setAddonEnabled(orm, 'ai_assistant', false); + try { + expect((await upload({ ...page, userId: s.userId, journeyId: s.journeyId })).status).toBe(404); + } finally { + await setAddonEnabled(orm, 'ai_assistant', true); + } + vi.stubEnv('TREK_MANAGED', 'true'); + try { + const managed = await upload({ ...page, userId: s.userId, journeyId: s.journeyId }); + expect(managed.status).toBe(403); + expect(managed.body).toMatchObject({ code: 'MANAGED_FORBIDDEN' }); + } finally { + vi.unstubAllEnvs(); + } + await setAddonEnabled(orm, 'journey', false); + try { + const off = await upload({ ...page, userId: s.userId, journeyId: s.journeyId }); + expect([off.status, off.body]).toEqual([404, { error: 'Journey addon is not enabled' }]); + } finally { + await setAddonEnabled(orm, 'journey', true); + } + const outsider = await upload({ ...page, userId: stranger.id, journeyId: s.journeyId }); + expect([outsider.status, outsider.body]).toEqual([404, { error: 'Journey not found' }]); + expect((await upload({ ...page, userId: s.userId, journeyId: 'abc' })).status).toBe(404); + const path = `/api/journeys/${s.journeyId}/assistant/threads/4/renders/x`; + expect((await request(nestApp.getHttpServer()).post(path)).status).toBe(401); + turn.stop.abort(); + await expect(turn.image).resolves.toBeNull(); + }); + + it('ASTRNE2E-002: an upload that is no image, too large or without its fields is refused', async () => { + const s = await studio(); + const turn = waitFor(s.userId, s.journeyId); + const base = { requestId: turn.requestId, userId: s.userId, journeyId: s.journeyId }; + expect((await upload({ ...base })).status).toBe(400); + const text = await upload({ ...base, image: Buffer.from('not an image at all') }); + expect([text.status, text.body]).toEqual([415, { error: 'The upload is not a JPEG, PNG or WebP image' }]); + expect((await upload({ ...base, image: jpeg(4000, 1000) })).status).toBe(413); + expect((await upload({ ...base, image: jpeg(2048, 2048) })).status).toBe(413); + expect((await upload({ ...base, image: jpeg(1280, 640, 1024 * 1024 + 1) })).status).toBe(413); + const noPage = await upload({ ...base, image: jpeg(1280, 640), fields: { bookVersion: '3' } }); + expect(noPage.status).toBe(400); + const extra = { bookVersion: '3', spreadId: 'sp-2', data: 'x' }; + expect((await upload({ ...base, image: jpeg(1280, 640), fields: extra })).status).toBe(400); + turn.stop.abort(); + await expect(turn.image).resolves.toBeNull(); + }); + + it('ASTRNE2E-003: an image that shows a photo of another member is refused; the turn goes on waiting', async () => { + const s = await studio(); + const turn = waitFor(s.userId, s.journeyId); + const base = { requestId: turn.requestId, userId: s.userId, journeyId: s.journeyId, image: jpeg(1280, 640) }; + const fields = { bookVersion: '3', spreadId: 'sp-2', drawnPhotoIds: `${s.own},${s.friends}` }; + const foreign = await upload({ ...base, fields }); + expect([foreign.status, foreign.body]).toEqual([ + 403, + { error: 'The image shows a photo another member contributed' }, + ]); + const mine = await upload({ ...base, fields: { ...fields, drawnPhotoIds: String(s.own) } }); + expect([mine.status, mine.body]).toEqual([200, { accepted: true }]); + const image = (await turn.image) as PageImage; + expect(image).toMatchObject({ mimeType: 'image/jpeg', width: 1280, height: 640 }); + expect(image.data.equals(jpeg(1280, 640))).toBe(true); + }); + + it('ASTRNE2E-004: only the waiting turn of this user and thread takes the page it asked for, once', async () => { + const s = await studio(); + const turn = waitFor(s.userId, s.journeyId); + const base = { requestId: turn.requestId, journeyId: s.journeyId, image: jpeg(1280, 640) }; + // The friend may open the journey, but the turn is not theirs. + expect((await upload({ ...base, userId: s.friendId })).status).toBe(404); + expect((await upload({ ...base, userId: s.userId, threadId: 5 })).status).toBe(404); + expect((await upload({ ...base, userId: s.userId, requestId: randomUUID() })).status).toBe(404); + const stale = await upload({ ...base, userId: s.userId, fields: { bookVersion: '2', spreadId: 'sp-2' } }); + expect(stale.status).toBe(409); + expect((await upload({ ...base, userId: s.userId })).status).toBe(200); + await expect(turn.image).resolves.toMatchObject({ width: 1280, height: 640 }); + const again = await upload({ ...base, userId: s.userId }); + expect([again.status, again.body]).toEqual([404, { error: 'No page was asked for' }]); + }); +}); diff --git a/server/tests/unit/nest/ai-connections/ai-connections.service.test.ts b/server/tests/unit/nest/ai-connections/ai-connections.service.test.ts index 2b37ab54b..d977ac236 100644 --- a/server/tests/unit/nest/ai-connections/ai-connections.service.test.ts +++ b/server/tests/unit/nest/ai-connections/ai-connections.service.test.ts @@ -267,6 +267,27 @@ describe('the instance connection', () => { expect((await service().status(id)).privacyAcked).toBe(false); }); + it('AICS-016: images of Studio pages are acknowledged on their own, per kind@host, on any connection', async () => { + const admin = await user(); + const id = await user(); + await service().saveInstance(admin, { kind: 'anthropic', apiKey: 'sk-ant-instance' }); + const target = { kind: 'anthropic' as const, host: 'api.anthropic.com' }; + expect((await service().status(id)).studioImagesAcked).toBe(false); + // The planner's acknowledgement says nothing about images. + await service().ackPrivacy(id, target); + expect(await service().studioImagesAcked(id, target)).toBe(false); + await service().ackPrivacy(id, { ...target, scope: 'studio_images' }); + expect(await readUserSetting(orm(), id, 'assistant_privacy_ack_studio_images')).toContain( + 'anthropic@api.anthropic.com', + ); + expect(await service().studioImagesAcked(id, target)).toBe(true); + expect(await service().status(id)).toMatchObject({ privacyAcked: true, studioImagesAcked: true }); + // Another provider asks again. + expect(await service().studioImagesAcked(id, { kind: 'openai', host: 'api.openai.com' })).toBe(false); + await service().saveInstance(admin, { kind: 'openai', apiKey: 'sk-proj-other' }); + expect((await service().status(id)).studioImagesAcked).toBe(false); + }); + it('AICS-014: settings are read with their defaults and written back', async () => { const admin = await user(); expect(await service().readSettings()).toEqual({ diff --git a/server/tests/unit/nest/ai-connections/model-catalog.service.test.ts b/server/tests/unit/nest/ai-connections/model-catalog.service.test.ts index b44339cf9..d4a6b33fc 100644 --- a/server/tests/unit/nest/ai-connections/model-catalog.service.test.ts +++ b/server/tests/unit/nest/ai-connections/model-catalog.service.test.ts @@ -6,6 +6,7 @@ */ import { AiConnectionsModule } from '../../../../src/nest/ai-connections/ai-connections.module'; import { + modelReadsImages, ModelCatalogService, pickDefaultModel, withRecommendation, @@ -66,7 +67,7 @@ describe('ModelCatalogService', () => { safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'claude-sonnet-5', display_name: 'Claude Sonnet 5' }] })); const key = secret(); expect(await catalog().list({ kind: 'anthropic', baseUrl: null, secret: key })).toEqual([ - { id: 'claude-sonnet-5', label: 'Claude Sonnet 5', supportsTools: true, recommended: true }, + { id: 'claude-sonnet-5', label: 'Claude Sonnet 5', supportsTools: true, supportsImages: true, recommended: true }, ]); const [url, init] = safeFetch.mock.calls[0] as [string, RequestInit, { maxBytes: number }]; expect(url).toBe('https://api.anthropic.com/v1/models?limit=1000'); @@ -109,6 +110,7 @@ describe('ModelCatalogService', () => { id: 'anthropic/claude-sonnet-5', name: 'Sonnet', supported_parameters: ['tools'], + architecture: { input_modalities: ['text', 'image'] }, pricing: { prompt: '0.000003', completion: '0.000015' }, }, { id: 'meta/llama:free', supported_parameters: ['tools'], pricing: { prompt: '0', completion: '0' } }, @@ -117,8 +119,15 @@ describe('ModelCatalogService', () => { }), ); expect(await catalog().list({ kind: 'openrouter', baseUrl: null, secret: secret() })).toEqual([ - { id: 'anthropic/claude-sonnet-5', label: 'Sonnet', supportsTools: true, free: false, recommended: true }, - { id: 'meta/llama:free', label: 'meta/llama:free', supportsTools: true, free: true }, + { + id: 'anthropic/claude-sonnet-5', + label: 'Sonnet', + supportsTools: true, + free: false, + supportsImages: true, + recommended: true, + }, + { id: 'meta/llama:free', label: 'meta/llama:free', supportsTools: true, free: true, supportsImages: false }, ]); expect(safeFetch.mock.calls.map((call) => call[0])).toEqual([ 'https://openrouter.ai/api/v1/key', @@ -167,7 +176,7 @@ describe('ModelCatalogService', () => { }), ); expect(await catalog().list({ kind: 'chatgpt', baseUrl: null, secret: secret() })).toEqual([ - { id: 'gpt-6.1-sol', label: 'GPT-6.1 Sol', supportsTools: true, recommended: true }, + { id: 'gpt-6.1-sol', label: 'GPT-6.1 Sol', supportsTools: true, supportsImages: true, recommended: true }, ]); safeFetch.mockResolvedValueOnce(json(500, {})); expect((await catalog().list({ kind: 'chatgpt', baseUrl: null, secret: secret() }))[0]?.id).toBe('gpt-6.1-sol'); @@ -218,6 +227,30 @@ describe('ModelCatalogService', () => { expect(pickDefaultModel('openai', list('gpt-4.1-mini', 'gpt-5.5-mini', 'gpt-5-mini'))).toBe('gpt-5.5-mini'); }); + it('MCAT-011: says which models read images: OpenAI by name, Gemini all, OpenRouter by its input kinds', async () => { + safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'gpt-5-mini' }, { id: 'gpt-4' }, { id: 'o3-mini' }] })); + const openai = await catalog().list({ kind: 'openai', baseUrl: null, secret: secret() }); + expect(Object.fromEntries(openai.map((m) => [m.id, m.supportsImages]))).toEqual({ + 'gpt-4': false, + 'gpt-5-mini': true, + 'o3-mini': false, + }); + safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'models/gemini-3-flash' }] })); + expect((await catalog().list({ kind: 'gemini', baseUrl: null, secret: secret() }))[0]?.supportsImages).toBe(true); + }); + + it('MCAT-012: a turn tells from the kind and the id alone whether its model reads images', () => { + expect(modelReadsImages('anthropic', 'claude-sonnet-5')).toBe(true); + expect(modelReadsImages('gemini', 'gemini-3-flash')).toBe(true); + expect(modelReadsImages('chatgpt', 'gpt-6.1-sol')).toBe(true); + expect(modelReadsImages('openai', 'gpt-4o')).toBe(true); + expect(modelReadsImages('openai', 'gpt-4-0613')).toBe(false); + expect(modelReadsImages('openai', 'gpt-3.5-turbo')).toBe(false); + // OpenRouter and a custom server cannot be told from the id: the turn stays on the description. + expect(modelReadsImages('openrouter', 'anthropic/claude-sonnet-5')).toBe(false); + expect(modelReadsImages('custom', 'llava')).toBe(false); + }); + it('MCAT-010: a picker lists the recommended model first and marked, the rest newest first', () => { const list = (...ids: string[]) => ids.map((id) => ({ id, label: id, supportsTools: true })); expect( diff --git a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts index 1a4ce7b23..37fa6484f 100644 --- a/server/tests/unit/nest/ai-connections/provider-adapters.test.ts +++ b/server/tests/unit/nest/ai-connections/provider-adapters.test.ts @@ -19,6 +19,9 @@ import { AiProviderError, CHATGPT_USAGE_URL, errorCodeOf, + IMAGE_INPUT_REFUSED, + isImageRefusal, + namesImageInput, providerFailure, } from '../../../../src/nest/ai-connections/providers/provider-errors'; import { toProviderSchema, type AiFetch } from '../../../../src/nest/ai-connections/providers/provider-http'; @@ -542,3 +545,91 @@ describe('OpenAiResponsesClient', () => { }); }); }); + +describe('images for a model that reads them', () => { + const PAGE = { mimeType: 'image/jpeg' as const, data: 'aGVsbG8=' }; + const WITH_IMAGES: AiChatRequest = { + ...REQUEST, + messages: [ + { role: 'user', text: 'Tidy page 2', images: [PAGE] }, + { + role: 'assistant', + text: '', + toolCalls: [ + { id: 'call_1', name: 'preview_book_page', args: { spreadId: 'sp-2' } }, + { id: 'call_2', name: 'get_book', args: {} }, + ], + }, + { role: 'tool', toolCallId: 'call_1', name: 'preview_book_page', output: 'Page 2', images: [PAGE] }, + { role: 'tool', toolCallId: 'call_2', name: 'get_book', output: '{"pages":4}' }, + ], + }; + + it('PIMG-001: Anthropic puts a tool image in its tool_result after the text, a user image before the words', () => { + const turns = toAnthropicMessages(WITH_IMAGES.messages); + const image = { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: 'aGVsbG8=' } }; + expect(turns[0]?.content).toEqual([image, { type: 'text', text: 'Tidy page 2' }]); + expect(turns[2]?.content).toEqual([ + { type: 'tool_result', tool_use_id: 'call_1', content: [{ type: 'text', text: 'Page 2' }, image] }, + { type: 'tool_result', tool_use_id: 'call_2', content: '{"pages":4}' }, + ]); + }); + + it('PIMG-002: Chat Completions keeps the tool message text and sends the step images after its last result', () => { + const messages = toChatMessages(WITH_IMAGES); + const part = { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,aGVsbG8=', detail: 'high' } }; + expect(messages[1]).toEqual({ role: 'user', content: [{ type: 'text', text: 'Tidy page 2' }, part] }); + expect(messages.slice(3)).toEqual([ + { role: 'tool', tool_call_id: 'call_1', content: 'Page 2' }, + { role: 'tool', tool_call_id: 'call_2', content: '{"pages":4}' }, + { + role: 'user', + content: [{ type: 'text', text: 'The image the call preview_book_page (call_1) produced:' }, part], + }, + ]); + }); + + it('PIMG-003: the Responses API sends the output as text, then the image in a user message', () => { + const input = toResponsesInput(WITH_IMAGES); + const image = { type: 'input_image', image_url: 'data:image/jpeg;base64,aGVsbG8=', detail: 'high' }; + expect(dig(input, 1, 'content', 1)).toEqual(image); + expect(input.slice(-3)).toEqual([ + { type: 'function_call_output', call_id: 'call_1', output: 'Page 2' }, + { type: 'function_call_output', call_id: 'call_2', output: '{"pages":4}' }, + { + type: 'message', + role: 'user', + content: [{ type: 'input_text', text: 'The image the call preview_book_page (call_1) produced:' }, image], + }, + ]); + }); + + it('PIMG-004: a history without images reads as it always did, with no extra user message', () => { + expect(toChatMessages(REQUEST).filter((message) => Array.isArray(message.content))).toEqual([]); + expect(toAnthropicMessages(REQUEST.messages)[2]?.content).toEqual([ + { type: 'tool_result', tool_use_id: 'call_1', content: '{"id":7}' }, + { type: 'text', text: 'Thanks' }, + ]); + }); + + it('PIMG-005: a refusal over the image carries its own code; other refusals and plan limits do not', async () => { + const refused = (status: number, body: unknown) => { + const http = fetchAnswering(jsonResponse(status, body)); + const client = new OpenAiChatClient('openai', 'gpt-4', 'https://o', 'k', http.fetcher); + return failure(client.stream(WITH_IMAGES, signal())); + }; + const imageRefusal = await refused(400, { error: { message: 'image_url is only supported by certain models.' } }); + expect(imageRefusal).toMatchObject({ code: 'generic', providerCode: IMAGE_INPUT_REFUSED }); + expect(isImageRefusal(imageRefusal)).toBe(true); + const other = await refused(400, { error: { code: 'model_not_found', message: 'No such model' } }); + expect(isImageRefusal(other)).toBe(false); + expect(isImageRefusal(await refused(500, { error: { message: 'image service down' } }))).toBe(false); + const code = 'subscription_sharing_unsupported_capability'; + const plan = await refused(400, { error: { code, message: 'Image input is not part of this plan' } }); + expect(plan).toMatchObject({ code: 'not_eligible', providerCode: code }); + expect(namesImageInput(422, { message: 'This model does not support vision' })).toBe(true); + expect(namesImageInput(400, { detail: 'Invalid input_image' })).toBe(true); + expect(namesImageInput(400, 'image')).toBe(false); + expect(isImageRefusal(new Error('image'))).toBe(false); + }); +}); diff --git a/server/tests/unit/nest/assistant/assistant-agent.test.ts b/server/tests/unit/nest/assistant/assistant-agent.test.ts index 36c9a67f5..40e854e92 100644 --- a/server/tests/unit/nest/assistant/assistant-agent.test.ts +++ b/server/tests/unit/nest/assistant/assistant-agent.test.ts @@ -7,6 +7,7 @@ import type { AiChatClient, AiChatEvent, + AiChatMessage, AiChatRequest, } from '../../../../src/nest/ai-connections/ai-chat-client.interface'; import { AiProviderError } from '../../../../src/nest/ai-connections/ai-chat-client.interface'; @@ -14,6 +15,7 @@ import { runAgentLoop, type AgentDeps, type AgentInput, + type AgentToolRun, type HeldCall, type StepRecord, } from '../../../../src/nest/assistant/assistant-agent'; @@ -559,6 +561,34 @@ describe('runAgentLoop', () => { ]); }); + it("ASTAGT-023: a change its gate sends back is that call's error for the model, never held; the next one is", async () => { + const refusalOf = vi.fn(async (_tool: string, args: Record) => + args.name === 'Broken' ? 'Not proposed yet: two photos overlap' : null, + ); + const t = setup( + [ + [ + { type: 'tool-call', call: { id: 'w1', name: 'create_place', args: { name: 'Broken' } } }, + { type: 'tool-call', call: { id: 'r1', name: 'list_places', args: {} } }, + ], + [{ type: 'tool-call', call: { id: 'w2', name: 'create_place', args: { name: 'Fixed' } } }], + ], + { refusalOf }, + ); + const outcome = await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal); + expect(outcome.end).toBe('awaiting'); + // Only changes are asked about, each with the arguments it would be held with. + expect(refusalOf.mock.calls).toEqual([ + ['create_place', { name: 'Broken' }], + ['create_place', { name: 'Fixed' }], + ]); + expect(t.deps.describe).toHaveBeenCalledTimes(1); + const sentBack = t.requests[1].messages.find((m) => m.role === 'tool' && m.toolCallId === 'w1'); + expect(sentBack).toMatchObject({ isError: true, output: expect.stringContaining('two photos overlap') }); + expect(t.chunks).toContainEqual({ type: 'tool-output-error', toolCallId: 'w1', errorText: expect.any(String) }); + expect(t.memory.held[0]?.held.map((call) => call.args)).toEqual([{ name: 'Fixed' }]); + }); + describe('answer cards', () => { /** A card as a package would build it; its fields do not matter to the loop. */ const CARD = { kind: 'tourList' } as TrekCard; @@ -623,4 +653,83 @@ describe('runAgentLoop', () => { expect(t.memory.saved[0].step.parts.map((part) => part.type)).toEqual(['tool-call']); }); }); + + describe('page images', () => { + const PAGE = { mimeType: 'image/jpeg' as const, data: 'aGVsbG8=' }; + const VIEW = { spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 5, description: 'Page 2' }; + const LOOKED: AgentToolRun = { ok: true, value: { description: 'Page 2' }, images: [PAGE, PAGE], view: VIEW }; + const look = (id: string) => ({ type: 'tool-call' as const, call: { id, name: 'list_places', args: {} } }); + const done = (): Step => [{ type: 'text-delta', delta: 'Looks good.' }]; + const vision = () => ({ + refused: vi.fn(), + afterApply: vi.fn(async (history: readonly AiChatMessage[]) => [...history]), + }); + const imagesOf = (messages: AiChatMessage[]) => + messages.filter((m) => m.role === 'tool').map((m) => (m.role === 'tool' ? (m.images?.length ?? 0) : 0)); + + it('ASTAGT-030: a look keeps what it looked at in its step and streams it; one image goes to the next request', async () => { + const t = setup([[look('c1')], done()], { vision: vision() }); + t.tools.call.mockImplementation(async () => LOOKED); + await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal); + const parts = t.memory.saved[0]?.step.parts ?? []; + expect(parts.map((part) => part.type)).toEqual(['tool-call', 'data-trek-page-view']); + expect(parts[1]).toEqual({ type: 'data-trek-page-view', data: VIEW }); + expect(t.chunks).toContainEqual({ type: 'data-trek-page-view', data: VIEW }); + // Never stored, never streamed: the bytes live for the next request. + expect(JSON.stringify([t.memory.saved, t.chunks])).not.toContain('aGVsbG8='); + expect(t.requests[1]?.messages.find((m) => m.role === 'tool')).toMatchObject({ + toolCallId: 'c1', + images: [PAGE], + }); + }); + + it('ASTAGT-031: a request carries the images of the newest three looks, one each; older looks go as their text', async () => { + const t = setup([[look('c1')], [look('c2')], [look('c3')], [look('c4')], done()], { vision: vision() }); + t.tools.call.mockImplementation(async () => LOOKED); + await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal); + expect(imagesOf(t.requests[1]?.messages ?? [])).toEqual([1]); + expect(imagesOf(t.requests[4]?.messages ?? [])).toEqual([0, 1, 1, 1]); + }); + + it('ASTAGT-032: a refused image drops the images, tells the vision and asks the step again, once', async () => { + const refusedImage = new AiProviderError('generic', 400, false, undefined, 'image_input_refused'); + const seeing = vision(); + const t = setup([[look('c1')], refusedImage, done()], { vision: seeing }); + t.tools.call.mockImplementation(async () => LOOKED); + const outcome = await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal); + expect(outcome.end).toBe('stop'); + expect(seeing.refused).toHaveBeenCalledOnce(); + expect(t.requests).toHaveLength(3); + expect(imagesOf(t.requests[1]?.messages ?? [])).toEqual([1]); + expect(imagesOf(t.requests[2]?.messages ?? [])).toEqual([0]); + expect(t.memory.saved.map((row) => row.status)).toEqual(['complete', 'complete']); + }); + + it('ASTAGT-033: another refusal, or one without a vision, ends the turn as before, asked once', async () => { + const otherRefusal = new AiProviderError('generic', 400, false, undefined, 'model_not_found'); + const seeing = vision(); + const t = setup([[look('c1')], otherRefusal], { vision: seeing }); + t.tools.call.mockImplementation(async () => LOOKED); + expect((await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal)).end).toBe('failed'); + expect(seeing.refused).not.toHaveBeenCalled(); + expect(t.requests).toHaveLength(2); + + const refusedImage = new AiProviderError('generic', 400, false, undefined, 'image_input_refused'); + const plain = setup([[look('c1')], refusedImage]); + plain.tools.call.mockImplementation(async () => LOOKED); + const plainOutcome = await runAgentLoop(plain.deps, plain.input, plain.writer, new AbortController().signal); + expect(plainOutcome.end).toBe('failed'); + expect(plain.requests).toHaveLength(2); + }); + + it('ASTAGT-034: a turn starts from the history the vision gives back, with views of applied pages', async () => { + const seeing = vision(); + const seen = { role: 'user' as const, text: 'seen', images: [PAGE] }; + seeing.afterApply.mockImplementation(async (history) => [...history, seen]); + const t = setup([done()], { vision: seeing }); + await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal); + expect(seeing.afterApply).toHaveBeenCalledWith(t.input.history); + expect(t.requests[0]?.messages.at(-1)).toEqual(seen); + }); + }); }); diff --git a/server/tests/unit/nest/assistant/assistant-book-preview.test.ts b/server/tests/unit/nest/assistant/assistant-book-preview.test.ts new file mode 100644 index 000000000..56031b615 --- /dev/null +++ b/server/tests/unit/nest/assistant/assistant-book-preview.test.ts @@ -0,0 +1,110 @@ +/** + * withPagePreview (ASTPVW-*): a Studio turn's tools with `preview_book_page` + * beside them. The preview takes the session's own `edit_book_page` ops + * schema, runs as a read through the turn's vision, refuses arguments it does + * not take, and is capped per step and per turn. + */ +import type { AgentToolPort, AgentToolRun } from '../../../../src/nest/assistant/assistant-agent'; +import { withPagePreview } from '../../../../src/nest/assistant/assistant-book-preview'; +import type { TurnVision } from '../../../../src/nest/assistant/assistant-vision'; +import { ASSISTANT_PAGE_IMAGE } from '@trek/shared'; + +import { describe, expect, it, vi } from 'vitest'; + +const OPS_SCHEMA = { type: 'array', items: { $ref: '#/$defs/op' } }; +const EDIT = { + name: 'edit_book_page', + description: 'Edit a page', + parameters: { type: 'object', properties: { spreadId: { type: 'string' }, ops: OPS_SCHEMA }, $defs: { op: {} } }, +}; + +function setup(specs = [EDIT]) { + const session: AgentToolPort = { + specs: () => specs, + kindOf: (name) => (name === 'edit_book_page' ? 'write' : 'read'), + pinArgs: vi.fn((_name: string, args: Record) => ({ args })), + call: vi.fn(async (): Promise => ({ ok: true, value: 'session' })), + }; + const looked: AgentToolRun = { ok: true, value: { description: 'Page 2' } }; + const vision = { + sendsImages: () => true, + look: vi.fn(async () => looked), + refused: vi.fn(), + afterApply: vi.fn(async () => []), + }; + return { session, vision, looked, tools: withPagePreview(session, vision as TurnVision) }; +} + +describe('withPagePreview', () => { + it('ASTPVW-001: offers the preview after the session tools, its ops as the session edit_book_page takes them', () => { + const { tools } = setup(); + const specs = tools.specs(); + expect(specs.map((spec) => spec.name)).toEqual(['edit_book_page', 'preview_book_page']); + expect(specs[1]?.parameters).toMatchObject({ + properties: { spreadId: { type: 'string' }, ops: OPS_SCHEMA }, + required: ['spreadId'], + additionalProperties: false, + $defs: { op: {} }, + }); + // A viewer has no edit tool: the preview shows pages as they stand. + const viewer = setup([]).tools.specs()[0]; + expect(viewer?.parameters.properties).toEqual({ spreadId: expect.objectContaining({ type: 'string' }) }); + expect(viewer?.parameters).not.toHaveProperty('$defs'); + }); + + it('ASTPVW-002: the preview is a read through the vision; every other tool stays the session one', async () => { + const { tools, session, vision, looked } = setup(); + expect(tools.kindOf('preview_book_page')).toBe('read'); + expect(tools.kindOf('edit_book_page')).toBe('write'); + const pinned = tools.pinArgs('preview_book_page', { spreadId: 'sp-2', journeyId: 12 }); + expect(pinned).toEqual({ args: { spreadId: 'sp-2' } }); + expect(session.pinArgs).not.toHaveBeenCalled(); + tools.pinArgs('edit_book_page', { spreadId: 'sp-2' }); + expect(session.pinArgs).toHaveBeenCalledWith('edit_book_page', { spreadId: 'sp-2' }); + + const ops = [{ op: 'background', color: '#112233' }]; + expect(await tools.call('preview_book_page', { spreadId: 'sp-2', ops })).toBe(looked); + expect(vision.look).toHaveBeenCalledWith('sp-2', ops); + expect(await tools.call('get_book', {})).toEqual({ ok: true, value: 'session' }); + expect(session.call).toHaveBeenCalledWith('get_book', {}); + }); + + it('ASTPVW-003: arguments the preview does not take are refused without a look', async () => { + const { tools, vision } = setup(); + const refused = await tools.call('preview_book_page', { spreadId: '../x' }); + expect(refused).toMatchObject({ ok: false, value: expect.stringContaining('Invalid arguments: spreadId') }); + expect((await tools.call('preview_book_page', { spreadId: 'sp-2', ops: [] })).ok).toBe(false); + expect(vision.look).not.toHaveBeenCalled(); + }); + + it('ASTPVW-004: previews are capped per step, which the next step resets, and per turn', async () => { + const { tools, vision } = setup(); + expect(ASSISTANT_PAGE_IMAGE).toMatchObject({ previewsPerStep: 3, previewsPerTurn: 8 }); + const preview = () => tools.call('preview_book_page', { spreadId: 'sp-2' }); + /** `count` previews one after another, as the loop runs a step's calls; whether each was answered. */ + const previews = (count: number) => + Array.from({ length: count }).reduce>( + async (done) => [...(await done), (await preview()).ok], + Promise.resolve([]), + ); + tools.specs(); + expect(await previews(4)).toEqual([true, true, true, false]); + expect(await preview()).toMatchObject({ ok: false, value: expect.stringContaining('Too many previews') }); + tools.specs(); + expect(await previews(3)).toEqual([true, true, true]); + tools.specs(); + expect(await previews(3)).toEqual([true, true, false]); + expect(vision.look).toHaveBeenCalledTimes(8); + }); + + it('ASTPVW-005: a session that edits may preview a page of a design draft by its designId; a viewer is not offered it', async () => { + const { tools, vision, looked } = setup(); + expect(tools.specs()[1]?.parameters.properties).toHaveProperty('designId'); + expect(setup([]).tools.specs()[0]?.parameters.properties).not.toHaveProperty('designId'); + + expect(await tools.call('preview_book_page', { spreadId: 'sp-2', designId: 'd-1' })).toBe(looked); + expect(vision.look).toHaveBeenCalledWith('sp-2', undefined, 'd-1'); + const refused = await tools.call('preview_book_page', { spreadId: 'sp-2', designId: '../d' }); + expect(refused).toMatchObject({ ok: false, value: expect.stringContaining('designId') }); + }); +}); diff --git a/server/tests/unit/nest/assistant/assistant-history.test.ts b/server/tests/unit/nest/assistant/assistant-history.test.ts index c63de15c9..d9911e1fc 100644 --- a/server/tests/unit/nest/assistant/assistant-history.test.ts +++ b/server/tests/unit/nest/assistant/assistant-history.test.ts @@ -275,6 +275,31 @@ describe('toChatHistory', () => { 'The hike "Herzogstand" was not added; the user moved on without deciding.', ); }); + + it('ASTHIS-014: a page the Studio drew is replayed as its description; no image is stored or sent again', () => { + const view = { spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 182_000, description: 'Page 2' }; + const looked = { description: 'Page 2, a double spread', findings: [] }; + const args = { spreadId: 'sp-2' }; + const rows = [ + row('user', [{ type: 'text', text: 'Check page 2' }]), + row('assistant', [ + { type: 'tool-call', toolCallId: 'p1', toolName: 'preview_book_page', args, result: looked }, + // Whatever reaches the store is what the shared part keeps: no bytes. + { type: 'data-trek-page-view', data: { ...view, data: 'aGVsbG8=' } as typeof view }, + ]), + row('assistant', [{ type: 'text', text: 'It looks balanced.' }]), + ]; + const history = toChatHistory(rows, new Map()); + expect(history).toEqual([ + { role: 'user', text: 'Check page 2' }, + { role: 'assistant', text: '', toolCalls: [{ id: 'p1', name: 'preview_book_page', args }], providerState: null }, + { role: 'tool', toolCallId: 'p1', name: 'preview_book_page', output: JSON.stringify(looked) }, + { role: 'assistant', text: 'It looks balanced.', toolCalls: [], providerState: null }, + ]); + expect(JSON.stringify(history)).not.toContain('aGVsbG8='); + const [, answer] = toThreadMessages(rows, new Map(), NOW); + expect(answer?.parts[1]).toEqual({ type: 'data-trek-page-view', data: view }); + }); }); describe('toThreadMessages', () => { diff --git a/server/tests/unit/nest/assistant/assistant-render.registry.test.ts b/server/tests/unit/nest/assistant/assistant-render.registry.test.ts new file mode 100644 index 000000000..05d03b439 --- /dev/null +++ b/server/tests/unit/nest/assistant/assistant-render.registry.test.ts @@ -0,0 +1,81 @@ +/** + * InMemoryAssistantRenderRegistry (ASTRND-*): a turn waits for the page image + * it asked the Studio for; the upload is handed over only to the turn of the + * same user, journey and thread, for the page and version asked for, and only + * while the turn still waits. + */ +import { + InMemoryAssistantRenderRegistry, + type PageImage, + type RenderWait, +} from '../../../../src/nest/assistant/assistant-render.registry'; + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const REQUEST: RenderWait = { + userId: 7, + journeyId: 12, + threadId: 4, + requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76', + spreadId: 'sp-2', + bookVersion: 3, +}; +const PAGE = { spreadId: 'sp-2', bookVersion: 3 }; +const IMAGE: PageImage = { mimeType: 'image/jpeg', data: Buffer.from('jpeg'), width: 1280, height: 640 }; + +beforeEach(() => { + vi.useFakeTimers(); +}); +afterEach(() => { + vi.useRealTimers(); +}); + +describe('InMemoryAssistantRenderRegistry', () => { + it('ASTRND-001: the upload of the page asked for reaches the waiting turn once', async () => { + const registry = new InMemoryAssistantRenderRegistry(); + const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken'); + await expect(waiting).resolves.toBe(IMAGE); + // Taken: a second upload for the same request finds nobody waiting. + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown'); + }); + + it('ASTRND-002: a turn stops waiting after the timeout, and an upload after it is too late', async () => { + const registry = new InMemoryAssistantRenderRegistry(); + const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000); + await vi.advanceTimersByTimeAsync(15_000); + await expect(waiting).resolves.toBeNull(); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown'); + }); + + it('ASTRND-003: a stopped turn stops waiting, also when it was stopped before it asked', async () => { + const registry = new InMemoryAssistantRenderRegistry(); + const stop = new AbortController(); + const waiting = registry.wait(REQUEST, stop.signal, 15_000); + stop.abort(); + await expect(waiting).resolves.toBeNull(); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown'); + await expect(registry.wait(REQUEST, stop.signal, 15_000)).resolves.toBeNull(); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown'); + }); + + it('ASTRND-004: another user, journey or thread cannot answer the request; the turn goes on waiting', async () => { + const registry = new InMemoryAssistantRenderRegistry(); + const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000); + expect(registry.deliver({ ...REQUEST, userId: 8 }, PAGE, IMAGE)).toBe('unknown'); + expect(registry.deliver({ ...REQUEST, journeyId: 13 }, PAGE, IMAGE)).toBe('unknown'); + expect(registry.deliver({ ...REQUEST, threadId: 5 }, PAGE, IMAGE)).toBe('unknown'); + expect(registry.deliver({ ...REQUEST, requestId: 'other' }, PAGE, IMAGE)).toBe('unknown'); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken'); + await expect(waiting).resolves.toBe(IMAGE); + }); + + it('ASTRND-005: an image of another page or version is refused, and the right one still comes through', async () => { + const registry = new InMemoryAssistantRenderRegistry(); + const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000); + expect(registry.deliver(REQUEST, { ...PAGE, spreadId: 'sp-3' }, IMAGE)).toBe('mismatch'); + expect(registry.deliver(REQUEST, { ...PAGE, bookVersion: 2 }, IMAGE)).toBe('mismatch'); + expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken'); + await expect(waiting).resolves.toBe(IMAGE); + }); +}); diff --git a/server/tests/unit/nest/assistant/assistant-vision.test.ts b/server/tests/unit/nest/assistant/assistant-vision.test.ts new file mode 100644 index 000000000..040b3d0bb --- /dev/null +++ b/server/tests/unit/nest/assistant/assistant-vision.test.ts @@ -0,0 +1,258 @@ +/** + * AssistantVision (ASTVSN-*): a Studio turn looking at a page. Images go only + * when the client draws pages, the model reads images and the user let the + * connection receive them; only the user's own photos are drawn; no image in + * time is the description alone; a refused image makes the model text-only + * for the process; a decision's turn starts with views of the pages the + * applied book writes left. The book, the photos and the acknowledgement are + * fakes; the registry is the real in-memory one, the request context a real + * ORM over a snapshot test db. + */ +import type { AiChatMessage } from '../../../../src/nest/ai-connections/ai-chat-client.interface'; +import type { AiConnectionsService } from '../../../../src/nest/ai-connections/ai-connections.service'; +import { InMemoryAssistantRenderRegistry } from '../../../../src/nest/assistant/assistant-render.registry'; +import type { AssistantToolSession } from '../../../../src/nest/assistant/assistant-tools.service'; +import { AssistantVision, type VisionTurnInput } from '../../../../src/nest/assistant/assistant-vision'; +import type { UiStreamWriter } from '../../../../src/nest/assistant/ui-stream.writer'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import type { JourneyAiPhotosService } from '../../../../src/nest/journey/journey-ai-photos.service'; +import type { JourneyBookEditService } from '../../../../src/nest/journey/journey-book-edit.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; +import { + blankBook, + bookMapElementSchema, + bookPhotoElementSchema, + type AssistantPageRender, + type BookDocument, +} from '@trek/shared'; + +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +const testDb = createSnapshotTestDb(); +let orm: TestOrm; + +beforeAll(async () => { + orm = await createTestOrm(testDb); +}); +afterAll(async () => { + await orm.close(); + testDb.close(); +}); + +const JPEG = Buffer.from('jpeg-bytes'); + +/** A book whose inner spread holds photo 1 (the user's) and photo 2 (another member's). */ +function book(): { doc: BookDocument; inner: string } { + let n = 0; + const doc = blankBook({ title: 'Lisbon', newId: (prefix) => `${prefix}-${++n}` }); + const inner = doc.spreads[2]; + if (!inner) throw new Error('no inner spread'); + const photo = (id: string, photoId: number, x: number) => + bookPhotoElementSchema.parse({ id, kind: 'photo', frame: { x, y: 20, w: 120, h: 90 }, photoId }); + doc.spreads[2] = { ...inner, elements: [photo('p-own', 1, 20), photo('p-other', 2, 260)] }; + return { doc, inner: inner.id }; +} + +interface Fakes { + acked?: boolean; + doc?: BookDocument | null; + /** What the Studio does with a render request: upload the page, or nothing (the turn is stopped instead). */ + upload?: boolean; +} + +function setup(fakes: Fakes = {}) { + const registry = new InMemoryAssistantRenderRegistry(); + const connections = { studioImagesAcked: vi.fn(async () => fakes.acked ?? true) }; + const fixture = book(); + const books = { + trialBase: vi.fn(async () => ({ + doc: fakes.doc === undefined ? fixture.doc : fakes.doc, + version: fakes.doc === null ? null : 4, + ctx: { photos: [{ photoId: 1 }, { photoId: 2 }] }, + })), + }; + const photos = { ownPhotoIds: vi.fn(async () => new Set([1])) }; + const vision = new AssistantVision( + connections as unknown as AiConnectionsService, + books as unknown as JourneyBookEditService, + photos as unknown as JourneyAiPhotosService, + registry, + orm, + ); + const stop = new AbortController(); + const asked: AssistantPageRender[] = []; + const w = { + renderRequest: vi.fn((request: AssistantPageRender) => { + asked.push(request); + if (fakes.upload === false) { + stop.abort(); + return; + } + const key = { userId: 7, journeyId: 12, threadId: 4, requestId: request.requestId }; + registry.deliver(key, request, { mimeType: 'image/jpeg', data: JPEG, width: 1280, height: 640 }); + }), + }; + const input: VisionTurnInput = { + userId: 7, + journeyId: 12, + threadId: 4, + connection: { kind: 'anthropic', model: 'claude-sonnet-5', host: 'api.anthropic.com' }, + renders: true, + w: w as unknown as UiStreamWriter, + signal: stop.signal, + }; + return { vision, connections, books, photos, asked, input, inner: fixture.inner }; +} + +describe('AssistantVision', () => { + it('ASTVSN-001: images go only when the Studio draws, the model reads images and the user said yes', async () => { + const t = setup(); + expect((await t.vision.forTurn(t.input)).sendsImages()).toBe(true); + expect(t.connections.studioImagesAcked).toHaveBeenCalledWith(7, t.input.connection); + expect((await t.vision.forTurn({ ...t.input, renders: false })).sendsImages()).toBe(false); + const textModel = { kind: 'custom' as const, model: 'llama3', host: 'ollama:11434' }; + expect((await t.vision.forTurn({ ...t.input, connection: textModel })).sendsImages()).toBe(false); + const notAcked = setup({ acked: false }); + expect((await notAcked.vision.forTurn(notAcked.input)).sendsImages()).toBe(false); + }); + + it('ASTVSN-002: a look draws only the user own photos; the image goes to the model, never stored', async () => { + const t = setup(); + const run = await (await t.vision.forTurn(t.input)).look(t.inner); + expect(t.asked).toEqual([ + { requestId: expect.any(String), threadId: 4, spreadId: t.inner, bookVersion: 4, hiddenPhotoIds: [2] }, + ]); + expect(run.ok).toBe(true); + expect(run.images).toEqual([{ mimeType: 'image/jpeg', data: JPEG.toString('base64') }]); + expect(run.view).toMatchObject({ spreadId: t.inner, bookVersion: 4, width: 1280, height: 640, bytes: JPEG.length }); + expect(run.view?.description).toContain('photo 2, by another contributor, not shown'); + expect(run.value).toMatchObject({ spreadId: t.inner, proposed: false, findings: expect.any(Array) }); + expect(JSON.stringify([run.value, run.view])).not.toContain(JPEG.toString('base64')); + }); + + it('ASTVSN-003: a text-only turn describes the page without asking the Studio or hiding anything', async () => { + const t = setup(); + const run = await (await t.vision.forTurn({ ...t.input, renders: false })).look(t.inner); + expect(t.asked).toEqual([]); + expect(t.photos.ownPhotoIds).not.toHaveBeenCalled(); + expect(run.images).toBeUndefined(); + expect(run.view).toMatchObject({ width: null, height: null, bytes: null }); + expect(run.view?.description).not.toContain('not shown'); + expect(run.value).not.toHaveProperty('image'); + }); + + it('ASTVSN-004: a proposal is drawn on a copy with its ops; no image in time leaves the description', async () => { + const t = setup({ upload: false }); + const ops = [{ op: 'background' as const, color: '#112233' }]; + const run = await (await t.vision.forTurn(t.input)).look(t.inner, ops); + expect(t.asked[0]).toMatchObject({ spreadId: t.inner, bookVersion: 4, ops }); + expect(run.ok).toBe(true); + expect(run.images).toBeUndefined(); + expect(run.value).toMatchObject({ proposed: true, image: expect.stringContaining('did not send') }); + expect(t.books.trialBase).toHaveBeenCalledWith(12, 7); + }); + + it('ASTVSN-005: a look at nothing says why: no book, no such page, an op the book refuses, no access', async () => { + const t = setup(); + const vision = await t.vision.forTurn(t.input); + expect(await vision.look('sp-missing')).toMatchObject({ ok: false, value: expect.stringContaining('no page') }); + const locked = await vision.look(t.inner, [{ op: 'remove', elementId: 'nope' } as never]); + expect(locked).toMatchObject({ ok: false, value: expect.stringContaining('Not a change the book takes') }); + const empty = setup({ doc: null }); + const noBook = await (await empty.vision.forTurn(empty.input)).look('sp-1'); + expect(noBook).toEqual({ ok: false, value: 'This journey has no book yet: create_book makes one.' }); + t.books.trialBase.mockRejectedValueOnce(new DomainError(404, 'Journey not found')); + expect(await vision.look(t.inner)).toEqual({ ok: false, value: 'Journey not found' }); + expect(t.asked).toEqual([]); + }); + + it('ASTVSN-006: a refused image makes the model text-only, for this turn and the next', async () => { + const t = setup(); + const vision = await t.vision.forTurn(t.input); + vision.refused(); + expect(vision.sendsImages()).toBe(false); + expect((await vision.look(t.inner)).images).toBeUndefined(); + expect((await t.vision.forTurn(t.input)).sendsImages()).toBe(false); + const other = { ...t.input.connection, model: 'claude-opus-5' }; + expect((await t.vision.forTurn({ ...t.input, connection: other })).sendsImages()).toBe(true); + }); + + it('ASTVSN-007: a decision turn starts with a view of each page the applied book writes left', async () => { + const t = setup(); + const written = JSON.stringify({ version: 5, spread_ids: [t.inner], pages: [], findings: [], dropped: [] }); + const history: AiChatMessage[] = [ + { role: 'user', text: 'Tidy it' }, + { role: 'assistant', text: '', toolCalls: [] }, + { role: 'tool', toolCallId: 'a', name: 'edit_book_page', output: written }, + { role: 'tool', toolCallId: 'b', name: 'edit_book_page', output: written }, + { role: 'tool', toolCallId: 'c', name: 'get_book', output: '{"exists":true}' }, + { role: 'tool', toolCallId: 'd', name: 'edit_book_page', output: written, isError: true }, + ]; + const seen = await (await t.vision.forTurn(t.input)).afterApply(history); + expect(t.asked).toHaveLength(1); + expect(seen.map((m) => (m.role === 'tool' ? (m.images?.length ?? 0) : null))).toEqual([null, null, 0, 1, 0, 0]); + expect(history[3]).not.toHaveProperty('images'); + + const textOnly = setup(); + const unchanged = await (await textOnly.vision.forTurn({ ...textOnly.input, renders: false })).afterApply(history); + expect(unchanged).toEqual(history); + expect(textOnly.books.trialBase).not.toHaveBeenCalled(); + }); + + it('ASTVSN-008: a Studio turn gets the page preview beside its tools; a trip turn its session as it is', async () => { + const t = setup(); + const session = { + specs: () => [], + kindOf: () => null, + pinArgs: (_name: string, args: Record) => ({ args }), + call: async () => ({ ok: true, value: null }), + } as unknown as AssistantToolSession; + const trip = await t.vision.turnTools({ kind: 'trip', tripId: 3 }, session, t.input); + expect(trip).toEqual({ tools: session, vision: null }); + const studio = await t.vision.turnTools({ kind: 'journey', journeyId: 12 }, session, t.input); + expect(studio.vision?.sendsImages()).toBe(true); + expect(studio.tools.specs().map((spec) => spec.name)).toEqual(['preview_book_page']); + }); + + it('ASTVSN-009: a map that marks its stops with photos draws only the user own ones', async () => { + const t = setup(); + const { doc, inner } = book(); + const spread = doc.spreads[2]; + if (!spread) throw new Error('no inner spread'); + const point = (photoId: number | null) => ({ lat: 38.7, lng: -9.1, label: '', photoId }); + const map = bookMapElementSchema.parse({ + id: 'm-1', + kind: 'map', + frame: { x: 20, y: 140, w: 200, h: 120 }, + pinStyle: 'photo', + points: [point(1), point(3), point(null), point(3)], + }); + doc.spreads[2] = { ...spread, elements: [...spread.elements, map] }; + t.books.trialBase.mockResolvedValueOnce({ doc, version: 4, ctx: { photos: [{ photoId: 1 }, { photoId: 2 }] } }); + await (await t.vision.forTurn(t.input)).look(inner); + expect(t.asked[0]?.hiddenPhotoIds).toEqual([2, 3]); + }); + + it('ASTVSN-010: a journey that went away while the turn ran is a look that says so, with no page drawn', async () => { + const t = setup(); + const vision = await t.vision.forTurn(t.input); + t.photos.ownPhotoIds.mockRejectedValueOnce(new DomainError(404, 'Journey not found')); + expect(await vision.look(t.inner)).toEqual({ ok: false, value: 'Journey not found' }); + expect(t.asked).toEqual([]); + }); + + it('ASTVSN-011: a page of a design draft is read from the draft and the Studio is told to draw it from there', async () => { + const t = setup(); + const vision = await t.vision.forTurn(t.input); + + const run = await vision.look(t.inner, undefined, 'd-1'); + + expect(t.books.trialBase).toHaveBeenCalledWith(12, 7, 'd-1'); + expect(t.asked[0]).toMatchObject({ spreadId: t.inner, bookVersion: 4, designId: 'd-1', hiddenPhotoIds: [2] }); + expect(run.ok).toBe(true); + await vision.look(t.inner); + expect(t.books.trialBase).toHaveBeenLastCalledWith(12, 7); + expect(t.asked[1]).not.toHaveProperty('designId'); + }); +}); diff --git a/server/tests/unit/nest/assistant/ui-stream.writer.test.ts b/server/tests/unit/nest/assistant/ui-stream.writer.test.ts index 37b0b02de..f2326fbf3 100644 --- a/server/tests/unit/nest/assistant/ui-stream.writer.test.ts +++ b/server/tests/unit/nest/assistant/ui-stream.writer.test.ts @@ -217,4 +217,32 @@ describe('UiStreamWriter', () => { w.close(); expect(stream.text()).toBe(CARDS_FIXTURE); }); + + it('ASTWRT-007: asks the Studio for a page in a transient chunk and tells the page it looked at', () => { + const stream = memoryStream(); + const w = new UiStreamWriter(stream); + w.text('Let me look'); + w.renderRequest({ + requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76', + threadId: 4, + spreadId: 'sp-2', + bookVersion: 3, + hiddenPhotoIds: [12], + }); + w.pageView({ spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 1000, description: 'Page 2' }); + const chunks = stream + .text() + .split('\n\n') + .filter((event) => event.startsWith('data: ')) + .map((event) => uiStreamChunkSchema.parse(JSON.parse(event.slice('data: '.length)))); + expect(chunks.map((chunk) => chunk.type)).toEqual([ + 'text-start', + 'text-delta', + 'text-end', + 'data-trek-render-request', + 'data-trek-page-view', + ]); + expect(chunks[3]).toMatchObject({ transient: true, data: { spreadId: 'sp-2', hiddenPhotoIds: [12] } }); + expect(chunks[4]).not.toHaveProperty('transient'); + }); }); diff --git a/server/tests/unit/nest/common/image-header.test.ts b/server/tests/unit/nest/common/image-header.test.ts new file mode 100644 index 000000000..9722e2912 --- /dev/null +++ b/server/tests/unit/nest/common/image-header.test.ts @@ -0,0 +1,57 @@ +import { readImageHeader } from '../../../../src/nest/common/image-header'; + +import { describe, expect, it } from 'vitest'; + +/** SOI, then the given segments, then a baseline frame header of w x h. */ +function jpeg(w: number, h: number, ...before: number[][]): Buffer { + const sof = Buffer.alloc(19); + sof.writeUInt16BE(0xffc0, 0); + sof.writeUInt16BE(17, 2); + sof.writeUInt8(8, 4); + sof.writeUInt16BE(h, 5); + sof.writeUInt16BE(w, 7); + return Buffer.concat([Buffer.from([0xff, 0xd8]), ...before.map((bytes) => Buffer.from(bytes)), sof]); +} + +function webp(chunk: string, length: number): Buffer { + const out = Buffer.alloc(length); + out.write('RIFF', 0, 'latin1'); + out.write('WEBP', 8, 'latin1'); + out.write(chunk, 12, 'latin1'); + return out; +} + +describe('readImageHeader', () => { + it('IMGHDR-001: steps over stray bytes, fill bytes and the markers that carry no length', () => { + const segments = [ + [0x00, 0x12], + [0xff, 0xff], + [0xff, 0x01], + [0xff, 0xd0], + [0xff, 0xe1, 0x00, 0x04, 0, 0], + ]; + expect(readImageHeader(jpeg(800, 600, ...segments))).toEqual({ mimeType: 'image/jpeg', width: 800, height: 600 }); + }); + + it('IMGHDR-002: a JPEG whose segment length is broken, or whose frame header is cut, says no size', () => { + expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x01, 0, 0]))).toBeNull(); + expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 8, 0]))).toBeNull(); + expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xda, 0x00, 0x08]))).toBeNull(); + expect(readImageHeader(jpeg(0, 600))).toBeNull(); + }); + + it('IMGHDR-003: a WebP too short for its chunk, or with a chunk it does not know, says no size', () => { + expect(readImageHeader(webp('VP8 ', 20))).toBeNull(); + expect(readImageHeader(webp('VP8L', 22))).toBeNull(); + expect(readImageHeader(webp('VP8X', 26))).toBeNull(); + expect(readImageHeader(webp('ALPH', 30))).toBeNull(); + }); + + it('IMGHDR-004: a PNG signature without its IHDR chunk is no PNG', () => { + const data = Buffer.alloc(24); + data.writeUInt32BE(0x89504e47, 0); + data.write('IDAT', 12, 'latin1'); + expect(readImageHeader(data)).toBeNull(); + expect(readImageHeader(Buffer.from([0xff, 0xd8]))).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/journey-ai-photos.service.test.ts b/server/tests/unit/nest/journey-ai-photos.service.test.ts new file mode 100644 index 000000000..e4705c84d --- /dev/null +++ b/server/tests/unit/nest/journey-ai-photos.service.test.ts @@ -0,0 +1,72 @@ +/** + * JourneyAiPhotosService (JAIPH-*): which gallery photos the Studio assistant + * may show a model, taken from JourneyBookModule over a snapshot test db. + * Only the asking user's own photos, the journey's owner standing in for a + * photo that names nobody; a stranger is told the journey is not there. + */ +import { JourneyPhotos } from '../../../src/db/entities/JourneyPhotos.entity'; +import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity'; +import { JourneyAiPhotosService } from '../../../src/nest/journey/journey-ai-photos.service'; +import { JourneyBookModule } from '../../../src/nest/journey/journey-book.module'; +import { createSnapshotTestDb } from '../../helpers/db-mock'; +import { addJourneyContributor, makeJourney } from '../../helpers/factories/journeys'; +import { insertRow } from '../../helpers/factories/rows'; +import { makeUser } from '../../helpers/factories/users'; +import { resetTestDb } from '../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../helpers/test-module'; +import { createTestOrm, type TestOrm } from '../../helpers/test-orm'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const db = createSnapshotTestDb(); +let t: TestModule; +let orm: TestOrm; + +beforeAll(async () => { + t = await createTestModule({ db, imports: [JourneyBookModule] }); + orm = await createTestOrm(db); +}); +beforeEach(() => { + resetTestDb(db); + orm.clear(); +}); +afterAll(async () => { + await orm.close(); + await t.close(); + db.close(); +}); + +/** A photo in the journey's gallery, contributed by `ownerId` (null: nobody named); answers its photo id. */ +async function galleryPhoto(journeyId: number, ownerId: number | null): Promise { + const photoId = await insertRow(orm, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg' }); + await insertRow(orm, JourneyPhotos, { journey: journeyId, photo: photoId, sort_order: 0, created_at: Date.now() }); + return photoId; +} + +describe('JourneyAiPhotosService', () => { + it('JAIPH-001: each member may show their own photos only; a photo naming nobody counts as the owner', async () => { + const { user: owner } = await makeUser(orm); + const { user: editor } = await makeUser(orm); + const journey = await makeJourney(orm, owner.id); + await addJourneyContributor(orm, journey.id, editor.id, 'editor'); + const ownersPhoto = await galleryPhoto(journey.id, owner.id); + const unnamed = await galleryPhoto(journey.id, null); + const editorsPhoto = await galleryPhoto(journey.id, editor.id); + + const service = t.get(JourneyAiPhotosService); + expect([...(await service.ownPhotoIds(journey.id, owner.id))].sort()).toEqual([ownersPhoto, unnamed].sort()); + expect([...(await service.ownPhotoIds(journey.id, editor.id))]).toEqual([editorsPhoto]); + }); + + it('JAIPH-002: a photo of another journey never counts; a stranger is told the journey is not there', async () => { + const { user: owner } = await makeUser(orm); + const { user: stranger } = await makeUser(orm); + const journey = await makeJourney(orm, owner.id); + const other = await makeJourney(orm, owner.id); + await galleryPhoto(other.id, owner.id); + + const service = t.get(JourneyAiPhotosService); + expect((await service.ownPhotoIds(journey.id, owner.id)).size).toBe(0); + await expect(service.ownPhotoIds(journey.id, stranger.id)).rejects.toThrow('Journey not found'); + }); +}); diff --git a/server/tests/unit/nest/llm-parse/image-input.test.ts b/server/tests/unit/nest/llm-parse/image-input.test.ts index bad1fbe9c..7b87266bf 100644 --- a/server/tests/unit/nest/llm-parse/image-input.test.ts +++ b/server/tests/unit/nest/llm-parse/image-input.test.ts @@ -1,10 +1,10 @@ +import { readImageHeader } from '../../../../src/nest/common/image-header'; import { capImage, IMAGE_MAX_BYTES, IMAGE_MAX_EDGE, imageMimeType, ImageTooLargeError, - readImageHeader, renderPdfPages, } from '../../../../src/nest/llm-parse/image-input'; diff --git a/shared/src/ai/ai-connections.schema.spec.ts b/shared/src/ai/ai-connections.schema.spec.ts index 8c5498f8c..4f735634a 100644 --- a/shared/src/ai/ai-connections.schema.spec.ts +++ b/shared/src/ai/ai-connections.schema.spec.ts @@ -3,7 +3,9 @@ import { aiApiKeyConnectRequestSchema, aiChatgptFinishRequestSchema, aiInstanceConnectionRequestSchema, + aiModelInfoSchema, aiOpenrouterStartRequestSchema, + aiPrivacyAckRequestSchema, aiRetentionSchema, aiStatusResponseSchema, } from './ai-connections.schema'; @@ -145,3 +147,22 @@ describe('aiStatusResponseSchema', () => { ).toBe(true); }); }); + +describe('the Studio page image', () => { + it('AIVIS-001: a model may say whether it reads images, and a list that cannot say leaves it out', () => { + expect(aiModelInfoSchema.parse({ id: 'm', label: 'm', supportsTools: true, supportsImages: false })).toEqual({ + id: 'm', + label: 'm', + supportsTools: true, + supportsImages: false, + }); + expect(aiModelInfoSchema.safeParse({ id: 'm', label: 'm', supportsTools: true }).success).toBe(true); + }); + + it('AIVIS-002: the acknowledgement of page images is its own scope; no other scope exists', () => { + const ack = { kind: 'anthropic', host: 'api.anthropic.com' }; + expect(aiPrivacyAckRequestSchema.safeParse(ack).success).toBe(true); + expect(aiPrivacyAckRequestSchema.safeParse({ ...ack, scope: 'studio_images' }).success).toBe(true); + expect(aiPrivacyAckRequestSchema.safeParse({ ...ack, scope: 'everything' }).success).toBe(false); + }); +}); diff --git a/shared/src/ai/ai-connections.schema.ts b/shared/src/ai/ai-connections.schema.ts index 15923ae6d..f7e01a61b 100644 --- a/shared/src/ai/ai-connections.schema.ts +++ b/shared/src/ai/ai-connections.schema.ts @@ -48,6 +48,8 @@ export const aiModelInfoSchema = z.object({ free: z.boolean().optional(), // The one a new connection starts on: the newest stable model of a capable line. Listed first. recommended: z.boolean().optional(), + // Whether the model reads images (the Studio assistant's page image); left out where the list cannot say. + supportsImages: z.boolean().optional(), }); export type AiModelInfo = z.infer; @@ -88,6 +90,8 @@ export const aiStatusResponseSchema = z.object({ quota: z.object({ used: z.number().int(), limit: z.number().int() }).nullable(), // Whether the user acknowledged that the effective kind@host receives their data. privacyAcked: z.boolean(), + // Whether the user also let kind@host receive images of their Studio pages, with their own photos on them. + studioImagesAcked: z.boolean().optional(), retention: aiRetentionSchema, // OpenRouter's OAuth needs an https callback; without one the client offers the key field instead. openrouterOAuth: z.boolean(), @@ -142,11 +146,15 @@ export type AiAuthStartResponse = z.infer; export const aiModelListResponseSchema = z.object({ models: z.array(aiModelInfoSchema) }); export type AiModelListResponse = z.infer; -/** POST /api/ai/privacy-ack: the user saw that kind@host receives their messages. */ +/** + * POST /api/ai/privacy-ack: the user saw that kind@host receives their + * messages, or (`scope: 'studio_images'`) also images of their Studio pages. + */ export const aiPrivacyAckRequestSchema = z .object({ kind: aiConnectionKindSchema, host: z.string().max(253), + scope: z.enum(['studio_images']).optional(), }) .strict(); export type AiPrivacyAckRequest = z.infer; diff --git a/shared/src/assistant/assistant-vision.schema.spec.ts b/shared/src/assistant/assistant-vision.schema.spec.ts new file mode 100644 index 000000000..e592b52c3 --- /dev/null +++ b/shared/src/assistant/assistant-vision.schema.spec.ts @@ -0,0 +1,85 @@ +import { + ASSISTANT_PAGE_IMAGE, + assistantPageRenderSchema, + assistantPageViewSchema, + assistantRenderUploadRequestSchema, + drawnPhotoIdsOf, + previewBookPageRequestSchema, +} from './assistant-vision.schema'; +import { assistantMessagePartSchema, uiStreamChunkSchema } from './assistant.schema'; + +import { describe, expect, it } from 'vitest'; + +const RENDER = { + requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76', + threadId: 4, + spreadId: 'sp-1', + bookVersion: 7, + hiddenPhotoIds: [12], +}; + +const VIEW = { spreadId: 'sp-1', bookVersion: 7, width: 1280, height: 640, bytes: 182_000, description: 'Page 2' }; + +describe('the Studio page image', () => { + it('ASTVIS-001: a render request names the page, the version and the photos left out; ops are a proposal', () => { + expect(assistantPageRenderSchema.safeParse(RENDER).success).toBe(true); + const ops = [{ op: 'background', color: '#ffffff' }]; + expect(assistantPageRenderSchema.safeParse({ ...RENDER, ops }).success).toBe(true); + expect(assistantPageRenderSchema.safeParse({ ...RENDER, ops: [] }).success).toBe(false); + expect(assistantPageRenderSchema.safeParse({ ...RENDER, requestId: 'r1' }).success).toBe(false); + expect(assistantPageRenderSchema.safeParse({ ...RENDER, spreadId: '../x' }).success).toBe(false); + }); + + it('ASTVIS-002: a page view keeps the size of the image and the description, never the image', () => { + expect(assistantPageViewSchema.parse({ ...VIEW, data: 'aGVsbG8=' })).toEqual(VIEW); + const textOnly = { ...VIEW, width: null, height: null, bytes: null }; + expect(assistantPageViewSchema.safeParse(textOnly).success).toBe(true); + }); + + it('ASTVIS-003: the page view is a stored part; the render request and the page view are stream chunks', () => { + expect(assistantMessagePartSchema.safeParse({ type: 'data-trek-page-view', data: VIEW }).success).toBe(true); + const request = { type: 'data-trek-render-request', data: RENDER, transient: true }; + expect(uiStreamChunkSchema.safeParse(request).success).toBe(true); + expect(uiStreamChunkSchema.safeParse({ type: 'data-trek-page-view', data: VIEW }).success).toBe(true); + }); + + it('ASTVIS-004: an upload reads its text fields as multipart sends them, and its photo ids as a list', () => { + const fields = { bookVersion: '7', spreadId: 'sp-1', drawnPhotoIds: '12,30' }; + expect(assistantRenderUploadRequestSchema.parse(fields)).toEqual({ ...fields, bookVersion: 7 }); + expect(drawnPhotoIdsOf('12,30')).toEqual([12, 30]); + expect(drawnPhotoIdsOf('')).toEqual([]); + expect(drawnPhotoIdsOf(undefined)).toEqual([]); + }); + + it('ASTVIS-006: a preview names a page and may carry the ops of a proposed edit, nothing else', () => { + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1' }).success).toBe(true); + const ops = [{ op: 'background', color: '#ffffff' }]; + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', ops }).success).toBe(true); + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', ops: [] }).success).toBe(false); + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', journeyId: 3 }).success).toBe(false); + }); + + it('ASTVIS-005: an upload refuses ids that are no list of numbers, and fields it does not know', () => { + const base = { bookVersion: '7', spreadId: 'sp-1' }; + expect(assistantRenderUploadRequestSchema.safeParse({ ...base, drawnPhotoIds: '12,,3' }).success).toBe(false); + expect(assistantRenderUploadRequestSchema.safeParse({ ...base, drawnPhotoIds: '1;2' }).success).toBe(false); + expect(assistantRenderUploadRequestSchema.safeParse({ ...base, data: 'x' }).success).toBe(false); + expect(assistantRenderUploadRequestSchema.safeParse({ ...base, bookVersion: '-1' }).success).toBe(false); + }); + + it('ASTVIS-007: a page may name as many photos as its frames and map stops hold, up to the cap', () => { + const many = Array.from({ length: ASSISTANT_PAGE_IMAGE.maxPagePhotos }, (_, i) => i + 1); + expect(assistantPageRenderSchema.safeParse({ ...RENDER, hiddenPhotoIds: many }).success).toBe(true); + const tooMany = [...many, ASSISTANT_PAGE_IMAGE.maxPagePhotos + 1]; + expect(assistantPageRenderSchema.safeParse({ ...RENDER, hiddenPhotoIds: tooMany }).success).toBe(false); + const fields = { bookVersion: '7', spreadId: 'sp-1', drawnPhotoIds: many.join(',') }; + expect(assistantRenderUploadRequestSchema.safeParse(fields).success).toBe(true); + }); + + it('ASTVIS-008: a preview and a render request may name a design draft, by an id that is no path', () => { + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', designId: 'd-1a2b' }).success).toBe(true); + expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', designId: '../d' }).success).toBe(false); + expect(assistantPageRenderSchema.parse({ ...RENDER, designId: 'd-1a2b' }).designId).toBe('d-1a2b'); + expect(assistantPageRenderSchema.safeParse({ ...RENDER, designId: 'd/1' }).success).toBe(false); + }); +}); diff --git a/shared/src/assistant/assistant-vision.schema.ts b/shared/src/assistant/assistant-vision.schema.ts new file mode 100644 index 000000000..e2554431b --- /dev/null +++ b/shared/src/assistant/assistant-vision.schema.ts @@ -0,0 +1,110 @@ +import { bookSpreadIdSchema } from '../book/book-op-ids'; +import { bookDesignIdSchema, editBookPageArgs } from '../book/book-ops.schema'; +import { idSchema } from '../common/primitives.schema'; + +import { z } from 'zod'; + +/** + * The Studio assistant looking at a page (journey-studio.md section 3). The + * server cannot draw a page, so it asks the Studio tab that holds the turn's + * stream for one: a `data-trek-render-request` chunk names the page, the book + * version and the photos to leave out, the tab draws that one page and + * uploads it as an image, and the turn hands it to a model that reads images, + * beside the page's description. Nothing of the image is kept: the message + * stores what was looked at (`data-trek-page-view`), never the bytes. + */ + +export const ASSISTANT_PAGE_IMAGE = { + /** The long edge a page is drawn at, in pixels, and the JPEG quality the Studio sends it with. */ + longEdge: 1280, + jpegQuality: 0.82, + /** What the server takes at most: the bytes, the long edge and the pixels of one image. */ + maxBytes: 1024 * 1024, + maxEdge: 2048, + maxPixels: 4_000_000, + /** Photos one page may name as hidden or drawn: its frames and the pictures its maps mark stops with. */ + maxPagePhotos: 1000, + /** How long a turn waits for a page image before it goes on with the description alone. */ + timeoutMs: 15_000, + /** Images one model request carries at most, one per tool answer; earlier ones go as their text. */ + perRequest: 3, + /** Previews of a proposed change one step and one turn may ask for. */ + previewsPerStep: 3, + previewsPerTurn: 8, +} as const; + +/** + * A page the server asks the Studio to draw, streamed while the turn waits. + * `ops` are a proposal to draw on an unsaved copy (a preview); without them + * the page is drawn as the book stands at `bookVersion`. The photos in + * `hiddenPhotoIds` are drawn as empty frames, and a map stop showing one as + * a plain dot: only the asking user's own photos leave the browser. + */ +export const assistantPageRenderSchema = z.object({ + requestId: z.uuid(), + threadId: idSchema, + spreadId: bookSpreadIdSchema, + bookVersion: z.number().int().nonnegative(), + ops: editBookPageArgs.ops.optional(), + /** A page of a design draft (design_book) rather than of the book: the Studio draws it from the draft. */ + designId: bookDesignIdSchema.optional(), + hiddenPhotoIds: z.array(idSchema).max(ASSISTANT_PAGE_IMAGE.maxPagePhotos), +}); +export type AssistantPageRender = z.infer; + +/** + * What a turn looked at, as the message keeps it: the page, the version, the + * image's size when one reached the model (null when the model read the + * description alone), and the description itself. Never the image. + */ +export const assistantPageViewSchema = z.object({ + spreadId: bookSpreadIdSchema, + bookVersion: z.number().int().nonnegative(), + width: z.number().int().positive().nullable(), + height: z.number().int().positive().nullable(), + bytes: z.number().int().positive().nullable(), + description: z.string().max(20_000), +}); +export type AssistantPageView = z.infer; + +/** + * `preview_book_page`, a read of the Studio turn's own (no MCP tool): one page + * as it stands, or with `ops` as an `edit_book_page` with them would leave it, + * nothing saved. It answers the page's description and findings, and a model + * that reads images also gets the page as the Studio draws it. + */ +export const previewBookPageRequestSchema = z.strictObject({ + spreadId: bookSpreadIdSchema, + ops: editBookPageArgs.ops.optional(), + /** Look at the page of a design draft instead of the book's (design_book's design_id). */ + designId: bookDesignIdSchema.optional(), +}); +export type PreviewBookPageRequest = z.infer; + +/** + * POST /api/journeys/:journeyId/assistant/threads/:threadId/renders/:requestId, + * multipart: the image in the field `image`, these as text fields beside it. + * `drawnPhotoIds` lists the photos the image shows, comma-separated; the + * server refuses the image when one of them is not the user's to show. + */ +export const assistantRenderUploadRequestSchema = z + .object({ + bookVersion: z.coerce.number().int().nonnegative(), + spreadId: bookSpreadIdSchema, + drawnPhotoIds: z + .string() + .max(ASSISTANT_PAGE_IMAGE.maxPagePhotos * 13) + .regex(/^(?:\d{1,12}(?:,\d{1,12})*)?$/) + .optional(), + }) + .strict(); +export type AssistantRenderUploadRequest = z.infer; + +/** The answer to an image the turn took; anything it did not take is refused with an error instead. */ +export const assistantRenderUploadResponseSchema = z.object({ accepted: z.literal(true) }); +export type AssistantRenderUploadResponse = z.infer; + +/** The photo ids of an upload's `drawnPhotoIds` field. */ +export function drawnPhotoIdsOf(field: string | undefined): number[] { + return field ? field.split(',').map(Number) : []; +}