feat(assistant): models that read images see the live page

The Studio draws the page on request and uploads it once, each provider gets
the image where it expects it, and a refusal falls back to the text
description. No image is stored.
This commit is contained in:
Maurice
2026-10-22 21:25:46 +00:00
parent 499458f8b2
commit 8b93898b73
38 changed files with 2714 additions and 123 deletions
@@ -2,7 +2,7 @@ import type { AiProviderState } from '../../db/json-columns';
import type { AiConnectionKind } from '@trek/shared';
export type { AiProviderState };
export { AiProviderError } from './providers/provider-errors';
export { AiProviderError, isImageRefusal } from './providers/provider-errors';
/** A tool the model may call: its name, what it is for and its arguments as JSON Schema. */
export interface AiToolSpec {
@@ -18,16 +18,30 @@ export interface AiToolCall {
args: Record<string, unknown>;
}
/**
* An image a message carries to a model that reads images (the Studio page
* the assistant looks at), base64 without line breaks. It lives for one turn
* in memory; nothing stores it.
*/
export interface AiImage {
mimeType: 'image/jpeg' | 'image/png' | 'image/webp';
data: string;
detail?: 'low' | 'high';
}
/**
* The provider-neutral history a turn sends. A tool result answers the call
* with the same id; an assistant message may carry the items its provider
* needs replayed verbatim (`providerState`), which an adapter for another
* provider ignores and rebuilds from the text and the calls instead.
* provider ignores and rebuilds from the text and the calls instead. A user
* message or a tool result may carry images; each adapter places them where
* its provider reads them, and the assistant sends none to a model that reads
* no images.
*/
export type AiChatMessage =
| { role: 'user'; text: string }
| { role: 'user'; text: string; images?: AiImage[] }
| { role: 'assistant'; text: string; toolCalls: AiToolCall[]; providerState?: AiProviderState | null }
| { role: 'tool'; toolCallId: string; name: string; output: string; isError?: boolean };
| { role: 'tool'; toolCallId: string; name: string; output: string; isError?: boolean; images?: AiImage[] };
export interface AiChatRequest {
/** The standing instructions, the same on every turn of a conversation (cached where the provider can). */
@@ -34,6 +34,13 @@ export const AI_DEFAULT_RETENTION: AiRetention = { mode: 'days', days: 90 };
/** The user setting that records which `kind@host` a user was told receives their messages. */
export const AI_PRIVACY_ACK_SETTING = 'assistant_privacy_ack';
/**
* The user setting that records which `kind@host` a user let receive images of
* their Studio pages (journey-studio.md 3.4), asked for on its own: the
* planner's notice says no files leave, which a page image would contradict.
*/
export const AI_STUDIO_IMAGES_ACK_SETTING = 'assistant_privacy_ack_studio_images';
/**
* Why a connection request was refused, as the client reads it from the body's
* `code`: the assistant's error codes, plus the ones only setting up a
@@ -12,6 +12,7 @@ import { pickConnection } from './ai-connection.resolver';
import {
AI_BASE_URLS,
AI_PRIVACY_ACK_SETTING,
AI_STUDIO_IMAGES_ACK_SETTING,
aiHostOf,
aiRefusal,
openrouterOAuthAvailable,
@@ -128,6 +129,7 @@ export class AiConnectionsService implements SharedAiConnectionPort {
? { used: await this.usage.instanceTurnsToday(userId), limit: settings.dailyTurns }
: null,
privacyAcked: acked,
studioImagesAcked: !!effective && (await this.studioImagesAckedFor(userId, `${kindOf(effective)}@${host}`)),
retention: settings.retention,
openrouterOAuth: openrouterOAuthAvailable(),
};
@@ -226,9 +228,23 @@ export class AiConnectionsService implements SharedAiConnectionPort {
return viewOf(await this.ownRow(userId));
}
/** The user saw that `kind@host` receives their messages on the instance's connection. */
/**
* The user saw that `kind@host` receives their messages on the instance's
* connection, or (`scope: 'studio_images'`) that it may also receive images
* of their Studio pages, on any connection.
*/
async ackPrivacy(userId: number, req: AiPrivacyAckRequest): Promise<void> {
await this.settings.upsertSetting(userId, AI_PRIVACY_ACK_SETTING, `${req.kind}@${req.host}`);
const key = req.scope === 'studio_images' ? AI_STUDIO_IMAGES_ACK_SETTING : AI_PRIVACY_ACK_SETTING;
await this.settings.upsertSetting(userId, key, `${req.kind}@${req.host}`);
}
/** Whether the user let this connection receive images of their Studio pages; without it a turn stays text. */
async studioImagesAcked(userId: number, connection: Pick<ResolvedAiConnection, 'kind' | 'host'>): Promise<boolean> {
return this.studioImagesAckedFor(userId, `${connection.kind}@${connection.host}`);
}
private async studioImagesAckedFor(userId: number, target: string): Promise<boolean> {
return (await this.settings.getDecryptedUserSetting(userId, AI_STUDIO_IMAGES_ACK_SETTING)) === target;
}
// ── The instance's connection (admin) ──────────────────────────────────────
@@ -35,6 +35,34 @@ const CHATGPT_FALLBACK_MODELS = [
/** OpenAI's list holds every model of the account; these are the ones a chat with tools cannot use. */
const OPENAI_NOT_FOR_CHAT = /(audio|realtime|transcribe|tts|image|search|instruct|embedding|moderation|codex)/;
/**
* OpenAI chat models that read no images: the first GPT-4 and its dated
* snapshots, GPT-3.5 and the small reasoning models of the o1 and o3 lines.
* Every other chat model on the list reads them. A model this misses is
* caught at run time: the provider refuses the image and the assistant goes
* on without it.
*/
const OPENAI_TEXT_ONLY = /^(?:gpt-3\.5|gpt-4(?:-0\d{3})?$|o1-mini|o1-preview|o3-mini)/;
/** Whether an OpenAI chat model reads images. */
function openAiReadsImages(id: string): boolean {
return !OPENAI_TEXT_ONLY.test(id);
}
/**
* Whether the model a turn runs on reads images, without asking the provider:
* every current Anthropic and Gemini model and every model of a ChatGPT plan
* do, OpenAI's by name. OpenRouter's and a custom server's cannot be told
* from the id alone, so they stay on the page description (journey-studio.md
* 6, cut 5); the model list still says so for OpenRouter where it can. A
* model this takes for one that reads images and that refuses one is caught
* at run time and remembered as text-only.
*/
export function modelReadsImages(kind: AiConnectionKind, model: string): boolean {
if (kind === 'openai') return openAiReadsImages(model);
return kind === 'anthropic' || kind === 'gemini' || kind === 'chatgpt';
}
const openAiListSchema = z.object({ data: z.array(z.object({ id: z.string() })) });
const anthropicListSchema = z.object({
data: z.array(z.object({ id: z.string(), display_name: z.string().optional() })),
@@ -45,6 +73,7 @@ const openRouterListSchema = z.object({
id: z.string(),
name: z.string().optional(),
supported_parameters: z.array(z.string()).optional(),
architecture: z.object({ input_modalities: z.array(z.string()).optional() }).optional(),
pricing: z.object({ prompt: z.string().optional(), completion: z.string().optional() }).optional(),
}),
),
@@ -150,6 +179,7 @@ export class ModelCatalogService {
id: m.id,
label: m.display_name ?? m.id,
supportsTools: true,
supportsImages: true,
}));
}
case 'openai': {
@@ -158,14 +188,14 @@ export class ModelCatalogService {
.data.map((m) => m.id)
.filter((id) => (id.startsWith('gpt-') || /^o\d/.test(id)) && !OPENAI_NOT_FOR_CHAT.test(id))
.sort()
.map((id) => ({ id, label: id, supportsTools: true }));
.map((id) => ({ id, label: id, supportsTools: true, supportsImages: openAiReadsImages(id) }));
}
case 'gemini': {
const body = await this.getJson(target.kind, `${AI_BASE_URLS.gemini}/models`, bearer(secret));
return parsed(openAiListSchema, body)
.data.map((m) => m.id.replace(/^models\//, ''))
.filter((id) => id.startsWith('gemini') && !/(embedding|image|tts|audio)/.test(id))
.map((id) => ({ id, label: id, supportsTools: true }));
.map((id) => ({ id, label: id, supportsTools: true, supportsImages: true }));
}
case 'openrouter': {
// The model list is public; the key is checked on its own first.
@@ -178,6 +208,8 @@ export class ModelCatalogService {
label: m.name ?? m.id,
supportsTools: true,
free: m.pricing?.prompt === '0' && m.pricing?.completion === '0',
// A model OpenRouter lists without its input kinds is taken as one that reads no images.
supportsImages: m.architecture?.input_modalities?.includes('image') ?? false,
}));
}
case 'chatgpt':
@@ -193,12 +225,12 @@ export class ModelCatalogService {
const body = await this.getJson('chatgpt', `${AI_BASE_URLS.chatgpt}/models`, bearer(accessToken));
listed = parsed(chatgptListSchema, body)
.models.filter((m) => m.visibility === 'list' && m.supported_in_api !== false)
.map((m) => ({ id: m.slug, label: m.display_name ?? m.slug, supportsTools: true }));
.map((m) => chatgptModel(m.slug, m.display_name));
} catch (err) {
// A refused sign-in is the answer; anything else falls back to the known list.
if (err instanceof AiProviderError && !err.retryable && err.code !== 'generic') throw err;
}
return listed.length > 0 ? listed : CHATGPT_FALLBACK_MODELS.map((id) => ({ id, label: id, supportsTools: true }));
return listed.length > 0 ? listed : CHATGPT_FALLBACK_MODELS.map((id) => chatgptModel(id));
}
private async customModels(baseUrl: string | null, apiKey: string | null): Promise<AiModelInfo[]> {
@@ -231,6 +263,11 @@ export class ModelCatalogService {
}
}
/** A model of a ChatGPT plan: every one reads images through the Responses API. */
function chatgptModel(id: string, label?: string): AiModelInfo {
return { id, label: label ?? id, supportsTools: true, supportsImages: true };
}
function bearer(secret: string): Record<string, string> {
return { authorization: `Bearer ${secret}` };
}
@@ -4,6 +4,7 @@ import type {
AiChatMessage,
AiChatRequest,
AiFinishReason,
AiImage,
AiToolCall,
} from '../ai-chat-client.interface';
import { AI_BASE_URLS } from '../ai-connections.constants';
@@ -32,10 +33,18 @@ const MAX_TOKENS = 32_000;
type ContentBlock = Record<string, unknown>;
type AnthropicMessage = { role: 'user' | 'assistant'; content: ContentBlock[] };
/** An image as a Messages API block: base64 in the request, never a URL. */
function imageBlock(image: AiImage): ContentBlock {
return { type: 'image', source: { type: 'base64', media_type: image.mimeType, data: image.data } };
}
/**
* The provider-neutral history as Messages API turns: a tool result is a
* `tool_result` block of a user turn, and turns of the same role that follow
* each other merge into one, since the API wants them alternating.
* each other merge into one, since the API wants them alternating. An image
* of a tool result goes inside that `tool_result`, after its text, where the
* model reads it as what the tool produced; a user's images go before their
* words.
*/
export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage[] {
const out: AnthropicMessage[] = [];
@@ -47,18 +56,20 @@ export function toAnthropicMessages(messages: AiChatMessage[]): AnthropicMessage
};
for (const message of messages) {
if (message.role === 'user') {
push('user', [{ type: 'text', text: message.text }]);
push('user', [...(message.images ?? []).map(imageBlock), { type: 'text', text: message.text }]);
} else if (message.role === 'assistant') {
push('assistant', [
...(message.text ? [{ type: 'text', text: message.text }] : []),
...message.toolCalls.map((call) => ({ type: 'tool_use', id: call.id, name: call.name, input: call.args })),
]);
} else {
const images = (message.images ?? []).map(imageBlock);
const output = images.length > 0 ? [{ type: 'text', text: message.output }, ...images] : message.output;
push('user', [
{
type: 'tool_result',
tool_use_id: message.toolCallId,
content: message.output,
content: output,
...(message.isError ? { is_error: true } : {}),
},
]);
@@ -4,6 +4,7 @@ import type {
AiChatMessage,
AiChatRequest,
AiFinishReason,
AiImage,
AiToolCall,
} from '../ai-chat-client.interface';
import { AiProviderError, errorCodeOf, providerFailure } from './provider-errors';
@@ -17,20 +18,43 @@ import {
toProviderSchema,
type AiFetch,
} from './provider-http';
import { imageDataUrl, toolImageCaption, toolImagesEndingAt } from './provider-images';
import { readSseEvents } from './sse-lines';
import type { AiConnectionKind } from '@trek/shared';
/** The end of an OpenAI-compatible stream. */
const DONE = '[DONE]';
/** The provider-neutral history as Chat Completions messages, the instructions and the trip card first. */
/**
* The provider-neutral history as Chat Completions messages, the instructions
* and the trip card first. A tool message takes no image, so the images of a
* step's tool results follow its last result as one user message.
*/
export function toChatMessages(request: AiChatRequest): Record<string, unknown>[] {
const system = [request.system, request.context].filter(Boolean).join('\n\n');
return [{ role: 'system', content: system }, ...request.messages.map(toChatMessage)];
const out: Record<string, unknown>[] = [{ role: 'system', content: system }];
request.messages.forEach((message, index) => {
out.push(toChatMessage(message));
const withImages = toolImagesEndingAt(request.messages, index);
if (withImages.length === 0) return;
const content = withImages.flatMap((tool) => [
{ type: 'text', text: toolImageCaption(tool) },
...(tool.images ?? []).map(imagePart),
]);
out.push({ role: 'user', content });
});
return out;
}
function imagePart(image: AiImage): Record<string, unknown> {
return { type: 'image_url', image_url: { url: imageDataUrl(image), detail: image.detail ?? 'high' } };
}
function toChatMessage(message: AiChatMessage): Record<string, unknown> {
if (message.role === 'user') return { role: 'user', content: message.text };
if (message.role === 'user') {
if (!message.images?.length) return { role: 'user', content: message.text };
return { role: 'user', content: [{ type: 'text', text: message.text }, ...message.images.map(imagePart)] };
}
if (message.role === 'tool') return { role: 'tool', tool_call_id: message.toolCallId, content: message.output };
return {
role: 'assistant',
@@ -4,6 +4,7 @@ import type {
AiChatEvent,
AiChatMessage,
AiChatRequest,
AiImage,
AiProviderState,
} from '../ai-chat-client.interface';
import { AI_BASE_URLS } from '../ai-connections.constants';
@@ -18,6 +19,7 @@ import {
toProviderSchema,
type AiFetch,
} from './provider-http';
import { imageDataUrl, toolImageCaption, toolImagesEndingAt } from './provider-images';
import { readSseEvents } from './sse-lines';
/** The namespace the tools are grouped in, as Sign in with ChatGPT asks for. */
@@ -40,13 +42,28 @@ type ChatgptTokenSource = (rejected?: string) => Promise<string>;
export function toResponsesInput(request: AiChatRequest): Record<string, unknown>[] {
const input: Record<string, unknown>[] = [];
if (request.context) input.push({ type: 'message', role: 'developer', content: request.context });
for (const message of request.messages) input.push(...toItems(message));
request.messages.forEach((message, index) => {
input.push(...toItems(message));
// A call's output is text; the images of a step's results follow its last one as a user message.
const withImages = toolImagesEndingAt(request.messages, index);
if (withImages.length === 0) return;
const content = withImages.flatMap((tool) => [
{ type: 'input_text', text: toolImageCaption(tool) },
...(tool.images ?? []).map(inputImage),
]);
input.push({ type: 'message', role: 'user', content });
});
return input;
}
function inputImage(image: AiImage): Record<string, unknown> {
return { type: 'input_image', image_url: imageDataUrl(image), detail: image.detail ?? 'high' };
}
function toItems(message: AiChatMessage): Record<string, unknown>[] {
if (message.role === 'user') {
return [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: message.text }] }];
const images = (message.images ?? []).map(inputImage);
return [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: message.text }, ...images] }];
}
if (message.role === 'tool') {
return [{ type: 'function_call_output', call_id: message.toolCallId, output: message.output }];
@@ -83,3 +83,40 @@ export function errorCodeOf(body: unknown): string | undefined {
}
return undefined;
}
/**
* The code a refusal carries when the provider said the request's image was
* the problem: a model that reads no images, a format it does not take. The
* assistant then sends the step again without images and remembers the model
* as one that reads none.
*/
export const IMAGE_INPUT_REFUSED = 'image_input_refused';
/** How far into an error body's message the image check reads. */
const MAX_MESSAGE_CHARS = 2000;
/** The providers' wording for an image they refuse ("image_url is only supported by certain models"). */
const NAMES_IMAGE = /\b(?:image|images|image_url|input_image|vision)\b/i;
/** The message out of a provider's JSON error body, wherever it puts it; '' when it has none. */
function errorMessageOf(body: unknown): string {
if (!body || typeof body !== 'object') return '';
const record = body as Record<string, unknown>;
const inner = record.error;
if (inner && typeof inner === 'object' && typeof (inner as Record<string, unknown>).message === 'string') {
return (inner as Record<string, string>).message;
}
if (typeof record.message === 'string') return record.message;
return typeof record.detail === 'string' ? record.detail : '';
}
/** Whether a refused request was refused over its image, read from the status and the error body. */
export function namesImageInput(status: number, body: unknown): boolean {
if (status !== 400 && status !== 415 && status !== 422) return false;
return NAMES_IMAGE.test(errorMessageOf(body).slice(0, MAX_MESSAGE_CHARS));
}
/** Whether a provider error is the refusal of an image (see IMAGE_INPUT_REFUSED). */
export function isImageRefusal(err: unknown): boolean {
return err instanceof AiProviderError && err.providerCode === IMAGE_INPUT_REFUSED;
}
@@ -1,6 +1,6 @@
import { readCappedJson } from '../../../utils/cappedFetch';
import { logWarn } from '../../audit/audit-log.logger';
import { AiProviderError, errorCodeOf, providerFailure } from './provider-errors';
import { AiProviderError, errorCodeOf, IMAGE_INPUT_REFUSED, namesImageInput, providerFailure } from './provider-errors';
import type { AiConnectionKind } from '@trek/shared';
/**
@@ -54,7 +54,11 @@ export async function openProviderStream(
/** The AiProviderError a non-success response stands for, read from its status and its error body. */
export async function refusalOf(kind: AiConnectionKind, res: Response): Promise<AiProviderError> {
const body = await readCappedJson<unknown>(res, MAX_ERROR_BYTES).catch(() => undefined);
const code = errorCodeOf(body);
const named = errorCodeOf(body);
// A refused image is named by its own code, so the turn can send the step again without it; a
// refusal the provider's own code already explains (a plan that does not cover it) keeps that code.
const image = namesImageInput(res.status, body) && providerFailure(kind, res.status, named).code === 'generic';
const code = image ? IMAGE_INPUT_REFUSED : named;
// Status and code only: an error body can quote the request, and the request holds the trip.
logWarn(`[ai] ${kind} refused the request: HTTP ${res.status}${code ? ` (${code})` : ''}`);
return providerFailure(kind, res.status, code);
@@ -0,0 +1,29 @@
import type { AiChatMessage, AiImage } from '../ai-chat-client.interface';
type ToolMessage = Extract<AiChatMessage, { role: 'tool' }>;
/** An image as the data URL the OpenAI-style APIs read: base64 in the request, never a link to TREK. */
export function imageDataUrl(image: AiImage): string {
return `data:${image.mimeType};base64,${image.data}`;
}
/** The line an image of a tool result is introduced with, where it has to travel in a user message. */
export function toolImageCaption(message: ToolMessage): string {
return `The image the call ${message.name} (${message.toolCallId}) produced:`;
}
/**
* The tool results with images whose run ends at `index`: an API that takes
* no image inside a tool result (Chat Completions, Responses) gets them in
* one user message after the last result of the step, since the results must
* follow the assistant's calls directly. Empty while the run goes on, or when
* nothing in it carried an image.
*/
export function toolImagesEndingAt(messages: readonly AiChatMessage[], index: number): ToolMessage[] {
if (messages[index]?.role !== 'tool' || messages[index + 1]?.role === 'tool') return [];
let start = index;
while (start > 0 && messages[start - 1]?.role === 'tool') start--;
return messages
.slice(start, index + 1)
.filter((message): message is ToolMessage => message.role === 'tool' && (message.images?.length ?? 0) > 0);
}
+105 -16
View File
@@ -1,8 +1,10 @@
import type { AiProviderState } from '../../db/json-columns';
import {
AiProviderError,
isImageRefusal,
type AiChatClient,
type AiChatMessage,
type AiImage,
type AiToolCall,
type AiToolSpec,
} from '../ai-connections/ai-chat-client.interface';
@@ -15,22 +17,42 @@ import type { AssistantToolKind, ToolRun } from './assistant-tools.service';
import type { AnswerInput } from './cards/card-types';
import { answerCardOf } from './cards/registry';
import type { UiStreamWriter } from './ui-stream.writer';
import type {
AssistantChangePreview,
AssistantErrorCode,
AssistantMessagePart,
AssistantWeatherDay,
TrekCard,
import {
ASSISTANT_PAGE_IMAGE,
type AssistantChangePreview,
type AssistantErrorCode,
type AssistantMessagePart,
type AssistantPageView,
type AssistantWeatherDay,
type TrekCard,
} from '@trek/shared';
type ToolArgs = Record<string, unknown>;
/** The tools of the turn, as the loop uses them (an AssistantToolSession). */
interface AgentToolPort {
/**
* What a tool call came back with; a look at a Studio page also brings the
* page image for the model (in memory for this turn only) and what the
* message keeps of the look.
*/
export interface AgentToolRun extends ToolRun {
images?: AiImage[];
view?: AssistantPageView;
}
/** The tools of the turn, as the loop uses them (an AssistantToolSession, in the Studio with its page preview). */
export interface AgentToolPort {
specs(): AiToolSpec[];
kindOf(name: string): AssistantToolKind | null;
pinArgs(name: string, args: ToolArgs): { args: ToolArgs } | { error: string };
call(name: string, args: ToolArgs): Promise<ToolRun>;
call(name: string, args: ToolArgs): Promise<AgentToolRun>;
}
/** A Studio turn's page images, as the loop uses them (assistant-vision.ts). */
export interface AgentVision {
/** The provider refused an image: the rest of the turn, and later turns on this model, go without. */
refused(): void;
/** The history a turn starts from, with views of the pages the changes just applied left. */
afterApply(history: readonly AiChatMessage[]): Promise<AiChatMessage[]>;
}
/** What one step leaves in its message row. */
@@ -71,6 +93,13 @@ export interface AgentDeps {
today: string;
/** The card a read's answer shows; the registry's (`answerCardOf`) unless a test hands in its own. */
answerCard?: (tool: string, input: AnswerInput) => TrekCard | null;
/**
* Why a proposed change goes back to the model as its call's error instead
* of to the user (a Studio turn's book check); null holds it as usual.
*/
refusalOf?: (tool: string, args: ToolArgs) => Promise<string | null>;
/** A Studio turn's page images; without it no image ever goes to the model. */
vision?: AgentVision | null;
}
export interface AgentInput {
@@ -103,6 +132,8 @@ interface StepParts {
knownPlaceIds: Set<string>;
/** The answer cards the step has shown; past MAX_CARDS_PER_STEP a read shows none. */
cards: number;
/** The page images of the step's reads, by call id, for the model's next request only. */
images: Map<string, AiImage[]>;
}
/** The longest call id a card is kept under; a read under a longer one shows no card. */
@@ -169,6 +200,12 @@ async function runRead(
...(run.ok ? {} : { isError: true }),
});
if (!run.ok) return;
// A look at a Studio page: the message keeps what was looked at, the model gets one image of it.
if (run.view) {
w.pageView(run.view);
step.parts.push({ type: 'data-trek-page-view', data: run.view });
}
if (run.images?.length) step.images.set(id, run.images.slice(0, 1));
const input = { toolCallId: id, args, value: run.value, card: deps.card, today: deps.today };
const card = id.length <= MAX_CARD_CALL_ID ? (deps.answerCard ?? answerCardOf)(name, input) : null;
if (card && step.cards++ < MAX_CARDS_PER_STEP) {
@@ -221,6 +258,39 @@ interface StepAnswer {
cutOff: boolean;
}
/**
* The history as one model request carries it: the page images of the
* newest `max` messages that have any, one image each; older messages go as
* their text alone, which holds the page's description.
*/
function limitImages(history: readonly AiChatMessage[], max: number): AiChatMessage[] {
let kept = 0;
return history
.slice()
.reverse()
.map((message) => {
if (message.role === 'assistant' || !message.images?.length) return message;
return { ...message, images: kept++ < max ? message.images.slice(0, 1) : undefined };
})
.reverse();
}
/**
* A provider that refused the request over its images (a model that reads
* none after all): the turn's history loses its images, the vision remembers
* the model as text only, and the step is asked again, once. False for any
* other failure, or when there was no image to drop.
*/
function dropImagesAfter(turn: Turn, err: unknown): boolean {
const withImages = turn.history.some((message) => message.role !== 'assistant' && !!message.images?.length);
if (!turn.deps.vision || !withImages || !isImageRefusal(err)) return false;
turn.deps.vision.refused();
turn.history.forEach((message, index) => {
if (message.role !== 'assistant' && message.images) turn.history[index] = { ...message, images: undefined };
});
return true;
}
/** Streams one answer of the model; a failure or a stop ends the turn here, saved as far as it got. */
async function streamAnswer(turn: Turn, rowId: number): Promise<StepAnswer | AgentOutcome> {
const { deps, input, w, signal } = turn;
@@ -230,7 +300,7 @@ async function streamAnswer(turn: Turn, rowId: number): Promise<StepAnswer | Age
{
system: input.system,
context: input.context,
messages: turn.history,
messages: limitImages(turn.history, ASSISTANT_PAGE_IMAGE.perRequest),
tools: deps.tools.specs(),
cacheKey: input.cacheKey,
},
@@ -250,6 +320,9 @@ async function streamAnswer(turn: Turn, rowId: number): Promise<StepAnswer | Age
}
return answer;
} catch (err) {
if (!signal.aborted && !answer.text && answer.received.length === 0 && dropImagesAfter(turn, err)) {
return streamAnswer(turn, rowId);
}
const partial: StepRecord = {
parts: answer.text ? [{ type: 'text', text: answer.text }] : [],
providerState: null,
@@ -282,6 +355,10 @@ async function runCalls(turn: Turn, calls: AiToolCall[], collected: StepParts):
let prepared = kind ? deps.tools.pinArgs(call.name, call.args) : { error: `Unknown tool: ${call.name}` };
const unknownPlace = 'error' in prepared ? null : placeIdRefusal(call.name, prepared.args, turn.knownPlaceIds);
if (unknownPlace) prepared = { error: unknownPlace };
if (kind === 'write' && !('error' in prepared)) {
const refused = await deps.refusalOf?.(call.name, prepared.args);
if (refused) prepared = { error: refused };
}
if (kind === 'read' && !('error' in prepared)) reads += 1;
const overCap = kind === 'write' ? writes > MAX_WRITES_PER_STEP : reads > MAX_READS_PER_STEP;
if ('error' in prepared || overCap) {
@@ -334,19 +411,30 @@ async function holdChanges(turn: Turn, rowId: number, step: StepRecord, held: He
return outcomeOf(turn, 'awaiting');
}
/** Records a step that ran its reads, and hands their answers back for the next one. */
async function carryOn(turn: Turn, rowId: number, step: StepRecord, text: string, calls: AiToolCall[]): Promise<void> {
/**
* Records a step that ran its reads, and hands their answers back for the
* next one, a page image beside the answer of the look that brought it.
*/
async function carryOn(
turn: Turn,
rowId: number,
step: StepRecord,
answer: { text: string; calls: AiToolCall[]; images: ReadonlyMap<string, AiImage[]> },
): Promise<void> {
await turn.deps.store.saveStep(rowId, step, 'complete');
turn.w.finishStep();
const { text, calls, images } = answer;
turn.history.push({ role: 'assistant', text, toolCalls: calls, providerState: step.providerState });
for (const part of step.parts) {
if (part.type !== 'tool-call') continue;
const seen = images.get(part.toolCallId);
turn.history.push({
role: 'tool',
toolCallId: part.toolCallId,
name: part.toolName,
output: outputText(part.result),
...(part.isError ? { isError: true } : {}),
...(seen ? { images: seen } : {}),
});
}
}
@@ -387,7 +475,7 @@ async function runStep(turn: Turn, step: number): Promise<AgentOutcome | null> {
}
const calls = received.map((call) => ({ ...call, id: uniqueCallId(call, input.usedCallIds, deps.newId) }));
const collected: StepParts = { parts, weather: [], knownPlaceIds: turn.knownPlaceIds, cards: 0 };
const collected: StepParts = { parts, weather: [], knownPlaceIds: turn.knownPlaceIds, cards: 0, images: new Map() };
const held = await runCalls(turn, calls, collected);
// One weather strip per step, whatever the step held: it is told before the changes.
const weather = weatherOf(collected.weather);
@@ -403,7 +491,7 @@ async function runStep(turn: Turn, step: number): Promise<AgentOutcome | null> {
return outcomeOf(turn, 'aborted');
}
if (held.length > 0) return holdChanges(turn, rowId, { parts, providerState }, held);
await carryOn(turn, rowId, { parts, providerState }, text, calls);
await carryOn(turn, rowId, { parts, providerState }, { text, calls, images: collected.images });
return null;
}
@@ -428,12 +516,13 @@ async function stepsFrom(turn: Turn, step: number): Promise<AgentOutcome> {
* before its held changes are announced, so a card the client shows always
* exists on the server.
*/
export function runAgentLoop(
export async function runAgentLoop(
deps: AgentDeps,
input: AgentInput,
w: UiStreamWriter,
signal: AbortSignal,
): Promise<AgentOutcome> {
const history = deps.vision ? await deps.vision.afterApply(input.history) : [...input.history];
const knownPlaceIds = new Set(
[...deps.card.places.values()].flatMap((place) => (place.googlePlaceId ? [place.googlePlaceId] : [])),
);
@@ -442,7 +531,7 @@ export function runAgentLoop(
input,
w,
signal,
history: [...input.history],
history,
usage: { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 },
knownPlaceIds,
nudged: false,
@@ -0,0 +1,101 @@
import type { AiToolSpec } from '../ai-connections/ai-chat-client.interface';
import type { AgentToolPort, AgentToolRun } from './assistant-agent';
import { PREVIEW_BOOK_PAGE_TOOL } from './assistant-tools.constants';
import type { TurnVision } from './assistant-vision';
import { ASSISTANT_PAGE_IMAGE, previewBookPageRequestSchema } from '@trek/shared';
/**
* `preview_book_page`, offered in every Studio turn beside the session's own
* tools (journey-studio.md 4.3, step 2): a read that answers one page in words
* with its findings, as it stands or as an `edit_book_page` with the given
* ops would leave it, and for a model that reads images the page as the
* Studio draws it (AssistantVision). Nothing is saved; the change itself is
* proposed afterwards like any other.
*
* Its `ops` take exactly the schema `edit_book_page` takes in this session,
* so a preview and the edit it checks are one shape. A session without that
* tool (a viewer's) previews pages as they stand. Previews are capped per
* step and per turn, since each can wait seconds for the Studio and costs a
* model an image.
*/
const EDIT_TOOL = 'edit_book_page';
const DESCRIPTION =
'Look at one page of the book before you change it or after: its elements, its findings (overlaps, words past ' +
'their frame, empty frames, low resolution, content in the fold) and, when your model reads images, the page as ' +
'the Studio draws it, with photos of other contributors left as empty frames. With ops (the same ops ' +
'edit_book_page takes) you see the page as that edit would leave it, without saving anything. Preview every ' +
'page you are about to change once, fix what the findings or the image show, then propose the edit. ' +
'With designId (from design_book) you see the page of that design draft instead of the book. ' +
`At most ${ASSISTANT_PAGE_IMAGE.previewsPerStep} previews per step and ${ASSISTANT_PAGE_IMAGE.previewsPerTurn} ` +
'per turn.';
/** The draft a preview may look at instead of the book, by design_book's design_id. */
const DESIGN_ID = { type: 'string', pattern: '^[\\w-]{1,40}$', description: 'A design draft, by its design_id.' };
const TOO_MANY =
`Too many previews: at most ${ASSISTANT_PAGE_IMAGE.previewsPerStep} in one step and ` +
`${ASSISTANT_PAGE_IMAGE.previewsPerTurn} in one turn. Propose the edit and let its card show the findings.`;
/** The preview's spec, its ops taken from the session's `edit_book_page` when it has that tool. */
function previewSpec(specs: readonly AiToolSpec[]): AiToolSpec {
const edit = specs.find((spec) => spec.name === EDIT_TOOL);
const properties = (edit?.parameters.properties ?? {}) as Record<string, unknown>;
const ops = properties.ops;
// Shared parts the ops point at, where the schema keeps them apart.
const shared: Record<string, unknown> = {};
for (const key of ['$defs', 'definitions']) if (edit?.parameters[key]) shared[key] = edit.parameters[key];
return {
name: PREVIEW_BOOK_PAGE_TOOL,
description: DESCRIPTION,
parameters: {
type: 'object',
properties: {
spreadId: { type: 'string', pattern: '^[\\w-]{1,40}$', description: 'The page, by its id from get_book.' },
// A design draft is an editor's: a session without the edit tool previews the book only.
...(ops ? { ops, designId: DESIGN_ID } : {}),
},
required: ['spreadId'],
additionalProperties: false,
...(ops ? shared : {}),
},
};
}
/** The session's tools with the preview beside them, answered through the turn's vision. */
export function withPagePreview(tools: AgentToolPort, vision: TurnVision): AgentToolPort {
let inStep = 0;
let inTurn = 0;
const preview = async (args: Record<string, unknown>): Promise<AgentToolRun> => {
if (inStep >= ASSISTANT_PAGE_IMAGE.previewsPerStep || inTurn >= ASSISTANT_PAGE_IMAGE.previewsPerTurn) {
return { ok: false, value: TOO_MANY };
}
const parsed = previewBookPageRequestSchema.safeParse(args);
if (!parsed.success) {
const issue = parsed.error.issues[0];
const detail = issue ? [issue.path.join('.'), issue.message].join(' ') : '';
return { ok: false, value: `Invalid arguments: ${detail}` };
}
inStep += 1;
inTurn += 1;
const { spreadId, ops, designId } = parsed.data;
return designId ? vision.look(spreadId, ops, designId) : vision.look(spreadId, ops);
};
return {
// The loop asks for the specs once at the start of every step: a new step may preview again.
specs: () => {
inStep = 0;
const listed = tools.specs();
return [...listed, previewSpec(listed)];
},
kindOf: (name) => (name === PREVIEW_BOOK_PAGE_TOOL ? 'read' : tools.kindOf(name)),
pinArgs: (name, args) => {
if (name !== PREVIEW_BOOK_PAGE_TOOL) return tools.pinArgs(name, args);
// The journey is the conversation's; the preview takes no other argument for it.
const { journeyId: _journeyId, ...rest } = args;
return { args: rest };
},
call: (name, args) => (name === PREVIEW_BOOK_PAGE_TOOL ? preview(args) : tools.call(name, args)),
};
}
@@ -0,0 +1,84 @@
/**
* The Studio page images a running turn waits for (journey-studio.md 3.1), as
* an injectable port. The turn streams a `data-trek-render-request` and waits
* here; the Studio tab that holds the stream draws the page and uploads it to
* the renders route, which hands it over. A turn waits a bounded time and
* stops waiting with its own abort; an upload nobody waits for (too late,
* another user's, never asked) is refused. The image lives in memory for that
* one hand-over: nothing here keeps it. In memory today, like the running
* turns themselves (AssistantStreamRegistry).
*/
/** A page image as the Studio uploaded it, checked by the renders route. */
export interface PageImage {
mimeType: 'image/jpeg' | 'image/png' | 'image/webp';
data: Buffer;
width: number;
height: number;
}
/** Whose request an upload answers: the user, their journey and thread, and the request's own id. */
interface RenderKey {
userId: number;
journeyId: number;
threadId: number;
requestId: string;
}
/** A request a turn streamed: whose it is, and the page and book version it asked for. */
export interface RenderWait extends RenderKey {
spreadId: string;
bookVersion: number;
}
/**
* What became of an upload: `taken` by the turn waiting for it, `unknown`
* when no turn of this user waits for that request, `mismatch` when it shows
* another page or version than the one asked for (the turn goes on waiting).
*/
type RenderDelivery = 'taken' | 'unknown' | 'mismatch';
export abstract class AssistantRenderRegistry {
/** The image of a streamed request, or null once `timeoutMs` passed or `signal` aborted. */
abstract wait(request: RenderWait, signal: AbortSignal, timeoutMs: number): Promise<PageImage | null>;
/** Hands an upload to the turn waiting for it. */
abstract deliver(key: RenderKey, page: { spreadId: string; bookVersion: number }, image: PageImage): RenderDelivery;
}
interface Waiting {
request: RenderWait;
settle: (image: PageImage | null) => void;
}
export class InMemoryAssistantRenderRegistry extends AssistantRenderRegistry {
private readonly waiting = new Map<string, Waiting>();
wait(request: RenderWait, signal: AbortSignal, timeoutMs: number): Promise<PageImage | null> {
if (signal.aborted) return Promise.resolve(null);
return new Promise((resolve) => {
let timer: ReturnType<typeof setTimeout> | undefined;
const settle = (image: PageImage | null) => {
clearTimeout(timer);
signal.removeEventListener('abort', onAbort);
if (this.waiting.get(request.requestId)?.settle === settle) this.waiting.delete(request.requestId);
resolve(image);
};
const onAbort = () => settle(null);
timer = setTimeout(() => settle(null), timeoutMs);
signal.addEventListener('abort', onAbort, { once: true });
this.waiting.set(request.requestId, { request, settle });
});
}
deliver(key: RenderKey, page: { spreadId: string; bookVersion: number }, image: PageImage): RenderDelivery {
const entry = this.waiting.get(key.requestId);
const asked = entry?.request;
if (!entry || !asked) return 'unknown';
if (asked.userId !== key.userId || asked.journeyId !== key.journeyId || asked.threadId !== key.threadId) {
return 'unknown';
}
if (asked.spreadId !== page.spreadId || asked.bookVersion !== page.bookVersion) return 'mismatch';
entry.settle(image);
return 'taken';
}
}
@@ -0,0 +1,123 @@
import { ADDON_IDS } from '../../addons';
import type { User } from '../../types';
import { AddonsService } from '../addons/addons.service';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { CurrentUser } from '../auth-core/current-user.decorator';
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
import { readImageHeader } from '../common/image-header';
import { isManagedBlocked, MANAGED_FORBIDDEN_ERROR, ManagedForbidden } from '../common/managed';
import { ResponseContract } from '../common/response-contract';
import { JourneyAiPhotosService } from '../journey/journey-ai-photos.service';
import { AssistantBookContext } from './assistant-book-context';
import { AssistantRenderRegistry } from './assistant-render.registry';
import { AssistantRenderUploadDto } from './assistant.dto';
import {
Body,
Controller,
HttpCode,
HttpException,
Param,
Post,
UploadedFile,
UseGuards,
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import {
ASSISTANT_PAGE_IMAGE,
assistantRenderUploadResponseSchema,
drawnPhotoIdsOf,
type AssistantRenderUploadResponse,
} from '@trek/shared';
import { memoryStorage } from 'multer';
const UPLOAD = {
storage: memoryStorage(),
limits: { fileSize: ASSISTANT_PAGE_IMAGE.maxBytes, files: 1 },
};
/** A positive integer route id, or null. */
function routeId(value: string): number | null {
const id = Number(value);
return Number.isInteger(id) && id > 0 ? id : null;
}
/**
* `POST /api/journeys/:journeyId/assistant/threads/:threadId/renders/:requestId`:
* the Studio page a running turn asked for (journey-studio.md 3.1), drawn by
* the Studio tab that holds the turn's stream and uploaded as one image in
* the field `image`.
*
* It takes a multipart upload, so the addons, the managed install and the
* journey are checked in the handler, as for a file in the trip chat
* (AssistantAttachmentsController): a guard would answer before multer read
* the body. Then the image itself (at most 1 MB by multer, a JPEG, PNG or
* WebP by its first bytes, at most 2048 px on its long edge and 4 MP), the
* photos it says it shows (only the user's own, JourneyAiPhotosService), and
* that a turn of this user on this journey and thread still waits for exactly
* this page and version. The bytes go on as received, decoded by nobody here,
* and are kept nowhere.
*/
@Controller('api/journeys/:journeyId/assistant')
@UseGuards(JwtAuthGuard)
export class AssistantRendersController {
constructor(
private readonly registry: AssistantRenderRegistry,
private readonly books: AssistantBookContext,
private readonly photos: JourneyAiPhotosService,
private readonly addons: AddonsService,
private readonly env: RuntimeEnvService,
) {}
@ManagedForbidden('the operator decides which model a managed install talks to, and pays for it', {
enforcedInHandler: true,
})
@Post('threads/:threadId/renders/:requestId')
@HttpCode(200)
@ResponseContract(assistantRenderUploadResponseSchema)
@UseInterceptors(FileInterceptor('image', UPLOAD))
async upload(
@CurrentUser() user: User,
@Param('journeyId') journeyParam: string,
@Param('threadId') threadParam: string,
@Param('requestId') requestId: string,
@UploadedFile() file: Express.Multer.File | undefined,
@Body() body: AssistantRenderUploadDto,
): Promise<AssistantRenderUploadResponse> {
if (!(await this.addons.isAddonEnabled(ADDON_IDS.AI_ASSISTANT))) {
throw new HttpException({ error: 'AI assistant addon is not enabled' }, 404);
}
if (isManagedBlocked(this.env)) throw new HttpException(MANAGED_FORBIDDEN_ERROR, 403);
if (!(await this.addons.isAddonEnabled(ADDON_IDS.JOURNEY))) {
throw new HttpException({ error: 'Journey addon is not enabled' }, 404);
}
const journeyId = routeId(journeyParam);
if (journeyId === null || !(await this.books.canOpen(user.id, journeyId))) {
throw new HttpException({ error: 'Journey not found' }, 404);
}
const threadId = routeId(threadParam);
if (threadId === null) throw new HttpException({ error: 'No page was asked for' }, 404);
if (!file) throw new HttpException({ error: 'No image uploaded' }, 400);
const header = readImageHeader(file.buffer);
if (!header) throw new HttpException({ error: 'The upload is not a JPEG, PNG or WebP image' }, 415);
const { maxEdge, maxPixels } = ASSISTANT_PAGE_IMAGE;
if (Math.max(header.width, header.height) > maxEdge || header.width * header.height > maxPixels) {
throw new HttpException({ error: `The image is larger than ${maxEdge} px or ${maxPixels / 1_000_000} MP` }, 413);
}
const own = await this.photos.ownPhotoIds(journeyId, user.id);
if (drawnPhotoIdsOf(body.drawnPhotoIds).some((photoId) => !own.has(photoId))) {
throw new HttpException({ error: 'The image shows a photo another member contributed' }, 403);
}
const delivered = this.registry.deliver(
{ userId: user.id, journeyId, threadId, requestId },
{ spreadId: body.spreadId, bookVersion: body.bookVersion },
{ mimeType: header.mimeType, data: file.buffer, width: header.width, height: header.height },
);
if (delivered === 'unknown') throw new HttpException({ error: 'No page was asked for' }, 404);
if (delivered === 'mismatch') {
throw new HttpException({ error: 'The image shows another page or version than the one asked for' }, 409);
}
return { accepted: true };
}
}
@@ -0,0 +1,291 @@
import type { AiChatMessage, AiImage } from '../ai-connections/ai-chat-client.interface';
import { AiConnectionsService } from '../ai-connections/ai-connections.service';
import type { ResolvedAiConnection } from '../ai-connections/ai-connections.types';
import { modelReadsImages } from '../ai-connections/model-catalog.service';
import { logDebug } from '../audit/audit-log.logger';
import { DomainError } from '../common/domain-error';
import { withRequestContext } from '../database/request-context';
import { JourneyAiPhotosService } from '../journey/journey-ai-photos.service';
import { JourneyBookEditService, type BookTrialBase } from '../journey/journey-book-edit.service';
import type { AgentToolPort, AgentToolRun, AgentVision } from './assistant-agent';
import { withPagePreview } from './assistant-book-preview';
import { AssistantRenderRegistry, type PageImage } from './assistant-render.registry';
import type { UiStreamWriter } from './ui-stream.writer';
import { MikroORM } from '@mikro-orm/core';
import { Injectable } from '@nestjs/common';
import {
applyBookOps,
ASSISTANT_PAGE_IMAGE,
bookWriteResultSchema,
describeSpread,
type AssistantPageView,
type AssistantScope,
type BookDocument,
type BookElement,
type BookPageOp,
} from '@trek/shared';
import { randomUUID } from 'node:crypto';
/**
* The Studio assistant looking at a page (journey-studio.md 3): the page in
* words for every model, and for a model that reads images the page as the
* Studio draws it. The server cannot draw a page, so a look streams a render
* request to the Studio tab that holds the turn and waits a bounded time for
* its upload (AssistantRenderRegistry); no image in time is the description
* alone, never an error.
*
* Images go only when all of this holds for the turn: the client draws pages
* (the Studio's chat says so with `renders`), the model reads images (by its
* kind and id, and not refused one before in this process), and the user let
* this connection receive page images (their Studio acknowledgement). Only the
* user's own photos are drawn; the others go as empty frames, before anything
* is rasterised, and the description says so. Nothing of an image is stored:
* the message keeps what was looked at (`data-trek-page-view`), the bytes live
* for the turn's next model request.
*/
/** One Studio turn's view of its pages. */
export interface TurnVision extends AgentVision {
/** Whether this turn sends page images to the model, as of now. */
sendsImages(): boolean;
/**
* One page as it stands or, with `ops`, as an `edit_book_page` with them
* would leave it, nothing saved; with `designId` the page of that design
* draft (design_book) instead of the book's.
*/
look(spreadId: string, ops?: BookPageOp[], designId?: string): Promise<AgentToolRun>;
}
/** What a Studio turn brings to its pages. */
export interface VisionTurnInput {
userId: number;
journeyId: number;
threadId: number;
connection: Pick<ResolvedAiConnection, 'kind' | 'model' | 'host'>;
/** The client draws pages for this turn (the Studio's chat). */
renders: boolean;
w: UiStreamWriter;
signal: AbortSignal;
}
const NO_BOOK = 'This journey has no book yet: create_book makes one.';
const LATE = 'The Studio did not send the page image in time; check the page on its description.';
/** A page image as a model request carries it. */
function asAiImage(image: PageImage): AiImage {
return { mimeType: image.mimeType, data: image.data.toString('base64') };
}
/** The photos an element shows: a photo frame's own, and the pictures a map draws its stops with. */
function photosOf(el: BookElement): number[] {
if (el.kind === 'photo') return el.photoId === null ? [] : [el.photoId];
if (el.kind === 'map') return el.points.flatMap((point) => (point.photoId === null ? [] : [point.photoId]));
return [];
}
/** The photos on a page that the user did not contribute: drawn as empty frames (or plain map stops). */
function hiddenOn(doc: BookDocument, spreadId: string, own: ReadonlySet<number>): number[] {
const spread = doc.spreads.find((s) => s.id === spreadId);
const ids = new Set<number>();
for (const el of spread?.elements ?? []) {
for (const photoId of photosOf(el)) if (!own.has(photoId)) ids.add(photoId);
}
return [...ids];
}
/** The page a successful book write left, from its answer as the model read it; null for anything else. */
function writtenPage(message: AiChatMessage): string | null {
if (message.role !== 'tool' || message.isError) return null;
let parsed: unknown;
try {
parsed = JSON.parse(message.output);
} catch {
return null;
}
const result = bookWriteResultSchema.safeParse(parsed);
return result.success ? (result.data.spread_ids[0] ?? null) : null;
}
/** The page as a proposed edit would leave it (the stored one without ops), or why the book refuses the edit. */
function pageAfter(
doc: BookDocument,
spreadId: string,
ops: BookPageOp[] | undefined,
base: BookTrialBase,
): BookDocument | string {
if (!ops) return doc;
const made = applyBookOps(doc, { spreadId, ops }, base.ctx);
if (made.ok) return made.doc;
const at = made.error.opIndex === null ? '' : ` (operation ${made.error.opIndex})`;
return `Not a change the book takes${at}: ${made.error.code}: ${made.error.message}`;
}
/** What the message keeps of a look: the page, the version, the image's size when one came, the description. */
function viewOf(spreadId: string, bookVersion: number, text: string, image: PageImage | null): AssistantPageView {
return {
spreadId,
bookVersion,
width: image?.width ?? null,
height: image?.height ?? null,
bytes: image ? image.data.length : null,
description: text.slice(0, 20_000),
};
}
/** Why a look found nothing, in the words a tool would answer. */
function reason(err: unknown): string {
if (err instanceof DomainError) return err.mcpMessage ?? err.publicMessage;
return err instanceof Error ? err.message : String(err);
}
/** Builds each Studio turn's vision, and remembers the models a provider refused an image for. */
@Injectable()
export class AssistantVision {
constructor(
private readonly connections: AiConnectionsService,
private readonly books: JourneyBookEditService,
private readonly photos: JourneyAiPhotosService,
private readonly registry: AssistantRenderRegistry,
private readonly orm: MikroORM,
) {}
/** `kind:model` of the models a provider refused an image for, while this process runs. */
private readonly textOnly = new Set<string>();
/** A turn's tools and its vision: a Studio turn's with the page preview beside them, a trip turn's as they are. */
async turnTools(
scope: AssistantScope,
session: AgentToolPort,
input: Omit<VisionTurnInput, 'journeyId'>,
): Promise<{ tools: AgentToolPort; vision: TurnVision | null }> {
if (scope.kind !== 'journey') return { tools: session, vision: null };
const vision = await this.forTurn({ ...input, journeyId: scope.journeyId });
return { tools: withPagePreview(session, vision), vision };
}
async forTurn(input: VisionTurnInput): Promise<TurnVision> {
const model = `${input.connection.kind}:${input.connection.model}`;
let images =
input.renders &&
modelReadsImages(input.connection.kind, input.connection.model) &&
!this.textOnly.has(model) &&
(await withRequestContext(this.orm, () => this.connections.studioImagesAcked(input.userId, input.connection)));
const vision: TurnVision = {
sendsImages: () => images,
look: (spreadId, ops, designId) => this.look(input, images, spreadId, { ops, designId }),
refused: () => {
images = false;
this.textOnly.add(model);
},
afterApply: (history) => (images ? this.afterApply(vision, history) : Promise.resolve([...history])),
};
return vision;
}
/**
* The history a decision's turn starts from, with a view of each page the
* applied book writes left attached to the write's own answer (4.3, step
* 3), so the model checks what the user took before it goes on. At most as
* many as one request carries, one image each, the same page once.
*/
private async afterApply(vision: TurnVision, history: readonly AiChatMessage[]): Promise<AiChatMessage[]> {
const out = [...history];
let start = out.length;
while (start > 0 && out[start - 1]?.role === 'tool') start--;
const seen = new Set<string>();
const targets: Array<{ index: number; spreadId: string }> = [];
for (let index = out.length - 1; index >= start && targets.length < ASSISTANT_PAGE_IMAGE.perRequest; index--) {
const message = out[index];
const spreadId = message ? writtenPage(message) : null;
if (spreadId && !seen.has(spreadId)) {
seen.add(spreadId);
targets.push({ index, spreadId });
}
}
// One after another: the Studio draws one page at a time.
await targets.reduce<Promise<void>>(async (previous, { index, spreadId }) => {
await previous;
const seenPage = await vision.look(spreadId);
const message = out[index];
if (seenPage.images?.length && message?.role === 'tool') out[index] = { ...message, images: seenPage.images };
}, Promise.resolve());
return out;
}
private async look(
input: VisionTurnInput,
withImage: boolean,
spreadId: string,
{ ops, designId }: { ops?: BookPageOp[]; designId?: string },
): Promise<AgentToolRun> {
let base: BookTrialBase;
let own: ReadonlySet<number> | null;
const { journeyId, userId } = input;
try {
// A draft's page is read from the draft at the version it was made against; the book's as it stands.
base = await withRequestContext(this.orm, () =>
designId ? this.books.trialBase(journeyId, userId, designId) : this.books.trialBase(journeyId, userId),
);
// Whose photos the image may show; the journey can have gone away since the turn began.
own = withImage
? await withRequestContext(this.orm, () => this.photos.ownPhotoIds(input.journeyId, input.userId))
: null;
} catch (err) {
return { ok: false, value: reason(err) };
}
if (!base.doc || base.version === null) return { ok: false, value: NO_BOOK };
const doc = pageAfter(base.doc, spreadId, ops, base);
if (typeof doc === 'string') return { ok: false, value: doc };
const hidden = own ? hiddenOn(doc, spreadId, own) : [];
// A page naming more photos than a render request may list is checked on its description alone.
const draws = withImage && hidden.length <= ASSISTANT_PAGE_IMAGE.maxPagePhotos;
const photos = (base.ctx.photos ?? []).map((p) => (hidden.includes(p.photoId) ? { ...p, hidden: true } : p));
const described = describeSpread(doc, spreadId, photos);
if (!described) return { ok: false, value: `This book has no page ${spreadId}: get_book lists its pages.` };
const image = draws
? await this.render(input, { spreadId, bookVersion: base.version, ops, designId, hidden })
: null;
const value = {
spreadId,
index: described.index,
role: described.role,
proposed: !!ops,
description: described.text,
findings: described.findings,
...(withImage ? { image: image ? 'The page as the Studio draws it is attached.' : LATE } : {}),
};
const view = viewOf(spreadId, base.version, described.text, image);
return { ok: true, value, view, ...(image ? { images: [asAiImage(image)] } : {}) };
}
/** Asks the Studio tab for the page and waits for it; null when none came in time or the turn stopped. */
private async render(
input: VisionTurnInput,
page: { spreadId: string; bookVersion: number; ops?: BookPageOp[]; designId?: string; hidden: number[] },
): Promise<PageImage | null> {
const requestId = randomUUID();
const { userId, journeyId, threadId, signal } = input;
const { spreadId, bookVersion, ops, designId, hidden } = page;
// Waiting before asking: an upload that comes back at once finds its turn.
const waiting = this.registry.wait(
{ userId, journeyId, threadId, requestId, spreadId, bookVersion },
signal,
ASSISTANT_PAGE_IMAGE.timeoutMs,
);
// A draft's page: the Studio draws it from the draft (GET .../book-designs/:designId).
const asked = {
requestId,
threadId,
spreadId,
bookVersion,
hiddenPhotoIds: hidden,
...(designId ? { designId } : {}),
};
input.w.renderRequest(ops ? { ...asked, ops } : asked);
const image = await waiting;
// Size and dimensions only: the page shows the user's photos.
if (image) logDebug(`[assistant] page image ${image.width}x${image.height}, ${image.data.length} bytes`);
return image;
}
}
@@ -6,6 +6,8 @@ import {
type AssistantCard,
type AssistantChangePreview,
type AssistantErrorCode,
type AssistantPageRender,
type AssistantPageView,
type AssistantPlaces,
type AssistantWeather,
type UiStreamChunk,
@@ -108,6 +110,17 @@ export class UiStreamWriter {
this.send({ type: 'data-trek-activity', data: activity, transient: true });
}
/** A Studio page the turn asks the Studio tab to draw and upload; only streamed, the message never keeps it. */
renderRequest(request: AssistantPageRender): void {
this.endText();
this.send({ type: 'data-trek-render-request', data: request, transient: true });
}
/** A Studio page the turn looked at: what it was and its description, never the image. */
pageView(view: AssistantPageView): void {
this.send({ type: 'data-trek-page-view', data: view });
}
quota(used: number, limit: number): void {
this.send({ type: 'data-trek-quota', data: { used, limit } });
}
+86
View File
@@ -0,0 +1,86 @@
/**
* What an image is and how large, read from its first bytes without decoding
* it. Shared by the domains that take an image from outside and must check it
* before anything decodes or sends it: the photo imports of AI parsing and
* the Studio page images the assistant hands a model.
*/
export interface ImageHeader {
mimeType: 'image/png' | 'image/jpeg' | 'image/webp';
width: number;
height: number;
}
/** JPEG start-of-frame markers: every SOFn except DHT (C4), JPG (C8) and DAC (CC). */
function isStartOfFrame(marker: number): boolean {
return marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
}
function jpegSize(data: Buffer): { width: number; height: number } | null {
let i = 2;
while (i + 4 <= data.length) {
// Stray bytes between segments, which some cameras write, are stepped over.
if (data[i] !== 0xff) {
i += 1;
continue;
}
const marker = data[i + 1];
// Fill bytes, and the markers that carry no length.
if (marker === 0xff) {
i += 1;
continue;
}
if (marker === 0x01 || marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7)) {
i += 2;
continue;
}
// The scan or the end came before any frame header.
if (marker === 0xd9 || marker === 0xda) return null;
const length = data.readUInt16BE(i + 2);
if (length < 2) return null;
if (isStartOfFrame(marker)) {
if (i + 9 > data.length) return null;
return { height: data.readUInt16BE(i + 5), width: data.readUInt16BE(i + 7) };
}
i += 2 + length;
}
return null;
}
function webpSize(data: Buffer): { width: number; height: number } | null {
const chunk = data.toString('latin1', 12, 16);
if (chunk === 'VP8 ' && data.length >= 30) {
return { width: data.readUInt16LE(26) & 0x3fff, height: data.readUInt16LE(28) & 0x3fff };
}
if (chunk === 'VP8L' && data.length >= 25) {
const bits = data.readUInt32LE(21);
return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
}
if (chunk === 'VP8X' && data.length >= 30) {
return { width: data.readUIntLE(24, 3) + 1, height: data.readUIntLE(27, 3) + 1 };
}
return null;
}
/**
* PNG from IHDR, JPEG from its frame header, WebP from its VP8, VP8L or VP8X
* chunk. Null when the bytes are none of the three, or do not say.
*/
export function readImageHeader(data: Buffer): ImageHeader | null {
let found: ImageHeader | null = null;
if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47 && data.toString('latin1', 12, 16) === 'IHDR') {
found = { mimeType: 'image/png', width: data.readUInt32BE(16), height: data.readUInt32BE(20) };
} else if (data.length >= 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
const size = jpegSize(data);
found = size && { mimeType: 'image/jpeg', ...size };
} else if (
data.length >= 16 &&
data.toString('latin1', 0, 4) === 'RIFF' &&
data.toString('latin1', 8, 12) === 'WEBP'
) {
const size = webpSize(data);
found = size && { mimeType: 'image/webp', ...size };
}
// A zero edge is a header that defers its size (a JPEG's DNL), which says nothing.
return found && found.width > 0 && found.height > 0 ? found : null;
}
@@ -97,11 +97,19 @@ export const MANAGED_ROUTE_ALLOW_LIST: string[] = [
'AssistantController.thread',
'AssistantController.threads',
'AssistantController.undo',
'AssistantRendersController.upload',
'AuthController.createMcpToken',
'AuthController.validateKeys',
'BackupController.restore',
'BackupController.updateAutoSettings',
'BackupController.uploadRestore',
'JourneyAssistantController.chat',
'JourneyAssistantController.decide',
'JourneyAssistantController.deleteThread',
'JourneyAssistantController.stop',
'JourneyAssistantController.thread',
'JourneyAssistantController.threads',
'JourneyAssistantController.undo',
'LlmLocalController.models',
'LlmLocalController.pull',
'NotificationsController.testSmtp',
@@ -0,0 +1,35 @@
import { JourneyPhotos } from '../../db/entities/JourneyPhotos.entity';
import type { JourneyPhotosRepository } from '../../db/repositories/JourneyPhotos.repository';
import { DomainError } from '../common/domain-error';
import { JourneyDomainService } from './journey-domain.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
/**
* Which photos of a journey's gallery the Studio assistant may show a model
* that reads images, for the user asking (journey-studio.md 3.4).
*
* TREK cannot tell who is in a photo; it knows who contributed it. So the
* rule is strict: only the photos the asking user contributed, as
* `trek_photos.owner_id` says, with the journey's owner standing in for a
* photo that names nobody (the fallback the photo proxy uses). Every other
* photo is drawn as an empty frame before the page is rasterised, and an
* uploaded page image that says it shows one is refused.
*/
@Injectable()
export class JourneyAiPhotosService {
constructor(
private readonly journey: JourneyDomainService,
@InjectRepository(JourneyPhotos) private readonly photosRepo: JourneyPhotosRepository,
) {}
/** The gallery photos the user contributed; refused like the journey routes for a journey they cannot open. */
async ownPhotoIds(journeyId: number, userId: number): Promise<Set<number>> {
const journey = await this.journey.canAccessJourney(journeyId, userId);
if (!journey) {
throw new DomainError(404, 'Journey not found', { mcpMessage: 'Journey not found or access denied.' });
}
const gallery = await this.photosRepo.galleryRead(journeyId);
return new Set(gallery.filter((p) => (p.owner_id || journey.user_id) === userId).map((p) => p.photo_id));
}
}
+1 -81
View File
@@ -1,3 +1,4 @@
import { readImageHeader } from '../common/image-header';
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared';
import { Jimp } from 'jimp';
@@ -46,87 +47,6 @@ export const IMAGE_MAX_BYTES = Math.floor(3.5 * 1024 * 1024);
/** A photo TREK will not decode or send: the message says why, for the import's warning. */
export class ImageTooLargeError extends Error {}
export interface ImageHeader {
mimeType: 'image/png' | 'image/jpeg' | 'image/webp';
width: number;
height: number;
}
/** JPEG start-of-frame markers: every SOFn except DHT (C4), JPG (C8) and DAC (CC). */
function isStartOfFrame(marker: number): boolean {
return marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
}
function jpegSize(data: Buffer): { width: number; height: number } | null {
let i = 2;
while (i + 4 <= data.length) {
// Stray bytes between segments, which some cameras write, are stepped over.
if (data[i] !== 0xff) {
i += 1;
continue;
}
const marker = data[i + 1];
// Fill bytes, and the markers that carry no length.
if (marker === 0xff) {
i += 1;
continue;
}
if (marker === 0x01 || marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7)) {
i += 2;
continue;
}
// The scan or the end came before any frame header.
if (marker === 0xd9 || marker === 0xda) return null;
const length = data.readUInt16BE(i + 2);
if (length < 2) return null;
if (isStartOfFrame(marker)) {
if (i + 9 > data.length) return null;
return { height: data.readUInt16BE(i + 5), width: data.readUInt16BE(i + 7) };
}
i += 2 + length;
}
return null;
}
function webpSize(data: Buffer): { width: number; height: number } | null {
const chunk = data.toString('latin1', 12, 16);
if (chunk === 'VP8 ' && data.length >= 30) {
return { width: data.readUInt16LE(26) & 0x3fff, height: data.readUInt16LE(28) & 0x3fff };
}
if (chunk === 'VP8L' && data.length >= 25) {
const bits = data.readUInt32LE(21);
return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
}
if (chunk === 'VP8X' && data.length >= 30) {
return { width: data.readUIntLE(24, 3) + 1, height: data.readUIntLE(27, 3) + 1 };
}
return null;
}
/**
* What a photo is and how large, read from its first bytes without decoding it:
* PNG from IHDR, JPEG from its frame header, WebP from its VP8, VP8L or VP8X
* chunk. Null when the bytes are none of the three, or do not say.
*/
export function readImageHeader(data: Buffer): ImageHeader | null {
let found: ImageHeader | null = null;
if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47 && data.toString('latin1', 12, 16) === 'IHDR') {
found = { mimeType: 'image/png', width: data.readUInt32BE(16), height: data.readUInt32BE(20) };
} else if (data.length >= 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
const size = jpegSize(data);
found = size && { mimeType: 'image/jpeg', ...size };
} else if (
data.length >= 16 &&
data.toString('latin1', 0, 4) === 'RIFF' &&
data.toString('latin1', 8, 12) === 'WEBP'
) {
const size = webpSize(data);
found = size && { mimeType: 'image/webp', ...size };
}
// A zero edge is a header that defers its size (a JPEG's DNL), which says nothing.
return found && found.width > 0 && found.height > 0 ? found : null;
}
/** Bytes sent as they came, as long as a provider takes them at that size. */
function asTheyCame(data: Buffer, mimeType: string): { data: Buffer; mimeType: string } {
if (data.length > IMAGE_MAX_BYTES) {
@@ -0,0 +1,343 @@
/**
* The Studio page image reaching the model, per provider kind, end to end:
* the whole app (buildApp, every global guard, the Zod pipe, the real MCP
* registry, the real agent and vision) with the provider's own adapter
* (Anthropic Messages, Chat Completions, the Responses API of a ChatGPT plan)
* talking to a scripted fetch instead of the network.
*
* What is stood in for, and why: the connection the turn resolves names the
* kind under test (an instance can only point a `custom` connection at
* loopback), the user's Studio acknowledgement is given, and the Studio tab's
* upload is the render registry answering at once with a page image (the
* upload route itself is ASTRNE2E's). Everything between, from the look to
* the request on the wire, is the real code.
*
* Pinned: a model that reads images gets the page in the place its provider
* reads it (inside the tool_result for Anthropic, a user message after the
* step's tool answers for Chat Completions and the Responses API), the turn
* streams one render request and keeps the image's size, never its bytes; a
* kind that cannot be told to read images (OpenRouter) asks for no image and
* checks the page on its description.
*/
import { buildApp } from '../../src/bootstrap';
import { AiChatClientService } from '../../src/nest/ai-connections/ai-chat-client.service';
import { AiConnectionsService } from '../../src/nest/ai-connections/ai-connections.service';
import type { ResolvedAiConnection } from '../../src/nest/ai-connections/ai-connections.types';
import { AnthropicMessagesClient } from '../../src/nest/ai-connections/providers/anthropic-messages.client';
import { OpenAiChatClient } from '../../src/nest/ai-connections/providers/openai-chat.client';
import { OpenAiResponsesClient } from '../../src/nest/ai-connections/providers/openai-responses.client';
import type { AiFetch } from '../../src/nest/ai-connections/providers/provider-http';
import { AssistantRenderRegistry, type PageImage } from '../../src/nest/assistant/assistant-render.registry';
import { JourneyBookEditService } from '../../src/nest/journey/journey-book-edit.service';
import { makeJourney } from '../helpers/factories/journeys';
import { setAddonEnabled } from '../helpers/factories/settings';
import { makeUser } from '../helpers/factories/users';
import { resetRateLimits } from '../helpers/test-db';
import { sessionCookie } from './harness';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import type { AiConnectionKind, UiStreamChunk } from '@trek/shared';
import { createServer, type IncomingMessage, type Server as HttpServer, type ServerResponse } from 'node:http';
import type { AddressInfo } from 'node:net';
import request from 'supertest';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
// ── The page the Studio "uploads" ────────────────────────────────────────────
const JPEG = Buffer.from('ffd8ffe000104a46494600010100ffd9', 'hex');
const PAGE: PageImage = { mimeType: 'image/jpeg', data: JPEG, width: 1280, height: 640 };
const DATA_URL = `data:image/jpeg;base64,${JPEG.toString('base64')}`;
// ── The provider, per kind: what each adapter streams back ───────────────────
/** An SSE answer from whole events, as the adapters read it. */
function sse(events: unknown[]): Response {
const text = events.map((e) => `data: ${typeof e === 'string' ? e : JSON.stringify(e)}\n\n`).join('');
return new Response(text, { status: 200, headers: { 'content-type': 'text/event-stream' } });
}
type Wire = 'anthropic' | 'chat' | 'responses';
const PREVIEW = 'preview_book_page';
/** The model looks at a page with one call, as each provider streams a tool call. */
function lookAt(wire: Wire, args: Record<string, unknown>): Response {
const json = JSON.stringify(args);
if (wire === 'anthropic') {
return sse([
{ type: 'message_start', message: { usage: { input_tokens: 10, output_tokens: 1 } } },
{
type: 'content_block_start',
index: 0,
content_block: { type: 'tool_use', id: 'call_look', name: PREVIEW },
},
{ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: json } },
{ type: 'content_block_stop', index: 0 },
{ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 20 } },
{ type: 'message_stop' },
]);
}
if (wire === 'chat') {
const call = { index: 0, id: 'call_look', type: 'function', function: { name: PREVIEW, arguments: json } };
return sse([
{ choices: [{ index: 0, delta: { tool_calls: [call] } }] },
{ choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }] },
{ choices: [], usage: { prompt_tokens: 12, completion_tokens: 4 } },
'[DONE]',
]);
}
const item = { type: 'function_call', id: 'fc_1', call_id: 'call_look', name: PREVIEW, arguments: json };
return sse([
{ type: 'response.created', response: {} },
{ type: 'response.output_item.done', item: { ...item, namespace: 'trek' } },
{ type: 'response.completed', response: { usage: { input_tokens: 50, output_tokens: 6 } } },
]);
}
/** The model's last words, as each provider streams text. */
function says(wire: Wire, text: string): Response {
if (wire === 'anthropic') {
return sse([
{ type: 'message_start', message: { usage: { input_tokens: 10, output_tokens: 1 } } },
{ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } },
{ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } },
{ type: 'content_block_stop', index: 0 },
{ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 4 } },
{ type: 'message_stop' },
]);
}
if (wire === 'chat') {
return sse([
{ choices: [{ index: 0, delta: { content: text } }] },
{ choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] },
{ choices: [], usage: { prompt_tokens: 12, completion_tokens: 4 } },
'[DONE]',
]);
}
return sse([
{ type: 'response.created', response: {} },
{ type: 'response.output_text.delta', delta: text },
{ type: 'response.completed', response: { usage: { input_tokens: 50, output_tokens: 6 } } },
]);
}
/** What the adapters sent, request by request, and what they are answered next. */
const wire = { bodies: [] as Array<Record<string, unknown>>, answers: [] as Response[] };
const scripted: AiFetch = async (_url, init) => {
wire.bodies.push(JSON.parse(String(init.body)) as Record<string, unknown>);
const next = wire.answers.shift();
if (!next) throw new Error('the scripted provider has no answer left');
return next;
};
/** The real adapter of a kind, on the scripted fetch. */
function adapterFor(kind: AiConnectionKind, model: string) {
if (kind === 'anthropic') return new AnthropicMessagesClient(model, 'sk-ant-test', scripted);
if (kind === 'chatgpt') return new OpenAiResponsesClient(model, async () => 'at-test', scripted);
return new OpenAiChatClient(kind, model, 'https://provider.invalid/v1', 'sk-test', scripted);
}
// ── A loopback server, only so the instance connection can be saved ─────────
async function answerModels(req: IncomingMessage, res: ServerResponse): Promise<void> {
if (req.method === 'GET' && req.url?.startsWith('/api/tags')) {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify({ models: [{ name: 'fake-model', size: 1 }] }));
return;
}
res.writeHead(404).end();
}
// ── The app ──────────────────────────────────────────────────────────────────
let nestApp: INestApplication;
let orm: MikroORM;
let fake: HttpServer;
let books: JourneyBookEditService;
let connections: AiConnectionsService;
let chatClients: AiChatClientService;
let registry: AssistantRenderRegistry;
beforeAll(async () => {
fake = createServer((req, res) => {
answerModels(req, res).catch(() => res.destroy());
});
await new Promise<void>((resolve) => fake.listen(0, '127.0.0.1', resolve));
nestApp = await buildApp();
orm = nestApp.get(MikroORM);
books = nestApp.get(JourneyBookEditService);
connections = nestApp.get(AiConnectionsService);
chatClients = nestApp.get(AiChatClientService);
registry = nestApp.get(AssistantRenderRegistry);
await setAddonEnabled(orm, 'ai_assistant', true);
await setAddonEnabled(orm, 'journey', true);
const adminId = (await makeUser(orm, { role: 'admin' })).user.id;
const { port } = fake.address() as AddressInfo;
const res = await request(nestApp.getHttpServer())
.put('/api/admin/ai/connection')
.set('Cookie', sessionCookie(adminId))
.send({ kind: 'custom', baseUrl: `http://127.0.0.1:${port}/v1`, model: 'fake-model' });
expect(res.status).toBe(200);
});
beforeEach(async () => {
await resetRateLimits(nestApp);
vi.restoreAllMocks();
wire.bodies = [];
wire.answers = [];
});
afterAll(async () => {
vi.restoreAllMocks();
await nestApp.close();
await new Promise<void>((resolve) => fake.close(() => resolve()));
});
/** The turn runs on `kind`'s adapter as if that connection were saved, the Studio images acknowledged. */
function runOn(kind: AiConnectionKind, model: string) {
const resolve = connections.resolveForUser.bind(connections);
vi.spyOn(connections, 'resolveForUser').mockImplementation(async (userId: number): Promise<ResolvedAiConnection> => ({
...(await resolve(userId)),
kind,
model,
}));
vi.spyOn(connections, 'studioImagesAcked').mockResolvedValue(true);
vi.spyOn(chatClients, 'create').mockImplementation(() => adapterFor(kind, model));
return vi.spyOn(registry, 'wait').mockResolvedValue(PAGE);
}
// ── Helpers ──────────────────────────────────────────────────────────────────
const CTX = { locale: 'en', timeZone: 'Europe/Lisbon' };
async function stream(path: string, userId: number, body: object): Promise<UiStreamChunk[]> {
const res = await request(nestApp.getHttpServer())
.post(path)
.set('Cookie', sessionCookie(userId))
.send(body)
.buffer(true)
.parse((s, done) => {
let text = '';
s.setEncoding('utf8');
s.on('data', (chunk: string) => (text += chunk));
s.on('end', () => done(null, text));
});
expect(res.status).toBe(200);
return (res.body as string)
.split('\n\n')
.filter((event) => event.startsWith('data: ') && event !== 'data: [DONE]')
.map((event) => JSON.parse(event.slice('data: '.length)) as UiStreamChunk);
}
const ofType = <T extends UiStreamChunk['type']>(chunks: UiStreamChunk[], type: T) =>
chunks.filter((chunk): chunk is Extract<UiStreamChunk, { type: T }> => chunk.type === type);
/** A user with a journey of their own and a blank book in it: its double page is `inner`. */
async function ownerWithBook() {
const { user } = await makeUser(orm);
const journey = await makeJourney(orm, user.id, { title: 'Portugal by train' });
const created = await books.createBook(journey.id, user.id, {});
const inner = created.pages.find((page) => page.role === 'inner')?.id ?? 'no inner page';
return { userId: user.id, journeyId: journey.id, inner, base: `/api/journeys/${journey.id}/assistant` };
}
/** One Studio turn on `kind`: the model previews warmer paper on the inner page, then answers. */
async function lookTurn(kind: AiConnectionKind, model: string, via: Wire) {
const j = await ownerWithBook();
const waited = runOn(kind, model);
const warmer = { spreadId: j.inner, ops: [{ op: 'background', color: '#f4efe6' }] };
wire.answers = [lookAt(via, warmer), says(via, 'Fine.')];
const turn = await stream(`${j.base}/chat`, j.userId, { ...CTX, threadId: null, text: 'Warmer?', renders: true });
expect(wire.bodies).toHaveLength(2);
return { j, turn, waited, sent: wire.bodies[1] ?? {} };
}
/** A list in a request body, empty when that key holds none. */
const listIn = (body: Record<string, unknown>, key: string): Array<Record<string, unknown>> => {
const value = body[key];
return Array.isArray(value) ? (value as Array<Record<string, unknown>>) : [];
};
/** The turn asked the Studio once for the page and kept the image's size, never its bytes. */
function expectOneImage(turn: UiStreamChunk[], spreadId: string) {
const [asked, ...more] = ofType(turn, 'data-trek-render-request');
expect(more).toEqual([]);
expect(asked?.data).toMatchObject({ spreadId, bookVersion: 1, hiddenPhotoIds: [] });
const [view] = ofType(turn, 'data-trek-page-view');
expect(view?.data).toMatchObject({ spreadId, bookVersion: 1, width: 1280, height: 640, bytes: JPEG.length });
expect(JSON.stringify(ofType(turn, 'tool-output-available'))).toContain('is attached');
expect(JSON.stringify(turn)).not.toContain(JPEG.toString('base64'));
}
// ── Cases ────────────────────────────────────────────────────────────────────
describe('the Studio page image per provider kind', () => {
it('ASTIMGE2E-001: Anthropic gets the page as an image block inside the tool_result of the look', async () => {
const { j, turn, waited, sent } = await lookTurn('anthropic', 'claude-sonnet-4-5', 'anthropic');
expect(waited).toHaveBeenCalledTimes(1);
expectOneImage(turn, j.inner);
const results = listIn(sent, 'messages').flatMap((message) =>
Array.isArray(message.content) ? (message.content as Array<Record<string, unknown>>) : [],
);
const result = results.find((block) => block.type === 'tool_result' && block.tool_use_id === 'call_look');
expect(result?.content).toEqual([
{ type: 'text', text: expect.stringContaining('is attached') },
{ type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: JPEG.toString('base64') } },
]);
});
/** Chat Completions: the tool answer stays text, the page follows in a user message of its own. */
async function expectChatImage(kind: AiConnectionKind, model: string) {
const { j, turn, sent } = await lookTurn(kind, model, 'chat');
expectOneImage(turn, j.inner);
const messages = listIn(sent, 'messages');
const answer = messages.findIndex((m) => m.role === 'tool' && m.tool_call_id === 'call_look');
expect(answer).toBeGreaterThan(0);
expect(typeof messages[answer]?.content).toBe('string');
expect(messages[answer + 1]).toEqual({
role: 'user',
content: [
{ type: 'text', text: 'The image the call preview_book_page (call_look) produced:' },
{ type: 'image_url', image_url: { url: DATA_URL, detail: 'high' } },
],
});
}
it('ASTIMGE2E-002: OpenAI keeps the tool answer text and sends the page in a user message after it', async () => {
await expectChatImage('openai', 'gpt-4o');
});
it('ASTIMGE2E-005: Gemini, on the same Chat Completions shape, gets the page the same way', async () => {
await expectChatImage('gemini', 'gemini-2.5-flash');
});
it('ASTIMGE2E-003: a ChatGPT plan (Responses API) gets the output as text, then the page as an image', async () => {
const { j, turn, sent } = await lookTurn('chatgpt', 'gpt-5', 'responses');
expectOneImage(turn, j.inner);
const input = listIn(sent, 'input');
const output = input.findIndex((item) => item.type === 'function_call_output' && item.call_id === 'call_look');
expect(output).toBeGreaterThan(0);
expect(typeof input[output]?.output).toBe('string');
expect(input[output + 1]).toEqual({
type: 'message',
role: 'user',
content: [
{ type: 'input_text', text: 'The image the call preview_book_page (call_look) produced:' },
{ type: 'input_image', image_url: DATA_URL, detail: 'high' },
],
});
});
it('ASTIMGE2E-004: OpenRouter (vision unknown by id) asks for no image and reads the page in words', async () => {
const { j, turn, waited, sent } = await lookTurn('openrouter', 'some-lab/some-model', 'chat');
expect(waited).not.toHaveBeenCalled();
expect(ofType(turn, 'data-trek-render-request')).toEqual([]);
const [view] = ofType(turn, 'data-trek-page-view');
expect(view?.data).toMatchObject({ spreadId: j.inner, width: null, height: null, bytes: null });
expect(JSON.stringify(sent)).not.toContain('image_url');
const answer = listIn(sent, 'messages').find((m) => m.role === 'tool' && m.tool_call_id === 'call_look');
expect(String(answer?.content)).toContain('Findings');
});
});
@@ -0,0 +1,193 @@
/**
* The Studio page images, end to end: the whole app (buildApp, every global
* guard, the Zod pipe, the response contracts) with a turn's wait primed on
* the container's own AssistantRenderRegistry, and the page uploaded the way
* the Studio tab uploads it (multipart, the image in `image`, the page and the
* photos it shows as text fields).
*
* Covered: the gates in the handler (the AI addon, the managed install, the
* journey addon, the journey itself), the image checks (none, not an image,
* over the edge or byte cap), a photo of another member refused, and the
* hand-over: only to the waiting turn of the same user and thread, for the
* page and version asked for, once.
*/
import { buildApp } from '../../src/bootstrap';
import { JourneyPhotos } from '../../src/db/entities/JourneyPhotos.entity';
import { TrekPhotos } from '../../src/db/entities/TrekPhotos.entity';
import { AssistantRenderRegistry, type PageImage } from '../../src/nest/assistant/assistant-render.registry';
import { addJourneyContributor, makeJourney } from '../helpers/factories/journeys';
import { insertRow } from '../helpers/factories/rows';
import { setAddonEnabled } from '../helpers/factories/settings';
import { makeUser } from '../helpers/factories/users';
import { resetRateLimits } from '../helpers/test-db';
import { sessionCookie } from './harness';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import { randomUUID } from 'node:crypto';
import request from 'supertest';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
let nestApp: INestApplication;
let orm: MikroORM;
let registry: AssistantRenderRegistry;
beforeAll(async () => {
nestApp = await buildApp();
orm = nestApp.get(MikroORM);
registry = nestApp.get(AssistantRenderRegistry);
await setAddonEnabled(orm, 'ai_assistant', true);
await setAddonEnabled(orm, 'journey', true);
});
beforeEach(async () => {
await resetRateLimits(nestApp);
});
afterAll(async () => {
await nestApp.close();
});
/** The first bytes of a baseline JPEG of w x h: all the route reads. */
function jpeg(w: number, h: number, size = 64): Buffer {
const out = Buffer.alloc(size);
Buffer.from([0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08]).copy(out, 0);
out.writeUInt16BE(h, 7);
out.writeUInt16BE(w, 9);
return out;
}
async function photoOf(journeyId: number, ownerId: number): Promise<number> {
const photoId = await insertRow(orm, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg' });
await insertRow(orm, JourneyPhotos, { journey: journeyId, photo: photoId, sort_order: 0, created_at: Date.now() });
return photoId;
}
async function studio() {
const { user } = await makeUser(orm);
const { user: friend } = await makeUser(orm);
const journey = await makeJourney(orm, user.id, { title: 'Portugal by train' });
await addJourneyContributor(orm, journey.id, friend.id, 'editor');
const own = await photoOf(journey.id, user.id);
const friends = await photoOf(journey.id, friend.id);
return { userId: user.id, friendId: friend.id, journeyId: journey.id, own, friends };
}
interface Upload {
userId: number;
journeyId: number | string;
threadId?: number;
requestId: string;
image?: Buffer;
fields?: Record<string, string>;
}
function upload(u: Upload) {
const path = `/api/journeys/${u.journeyId}/assistant/threads/${u.threadId ?? 4}/renders/${u.requestId}`;
let req = request(nestApp.getHttpServer()).post(path).set('Cookie', sessionCookie(u.userId));
for (const [key, value] of Object.entries(u.fields ?? { bookVersion: '3', spreadId: 'sp-2' })) {
req = req.field(key, value);
}
if (u.image === undefined) return req;
return req.attach('image', u.image, { filename: 'page.jpg', contentType: 'image/jpeg' });
}
/** A turn of the user waiting for page sp-2 at version 3 of their journey, in thread 4. */
function waitFor(userId: number, journeyId: number) {
const requestId = randomUUID();
const stop = new AbortController();
const image = registry.wait(
{ userId, journeyId, threadId: 4, requestId, spreadId: 'sp-2', bookVersion: 3 },
stop.signal,
15_000,
);
return { requestId, image, stop };
}
describe('Studio page images e2e', () => {
it('ASTRNE2E-001: the addons, the managed install and the journey are checked before the image', async () => {
const s = await studio();
const { user: stranger } = await makeUser(orm);
const turn = waitFor(s.userId, s.journeyId);
const page = { requestId: turn.requestId, image: jpeg(1280, 640) };
await setAddonEnabled(orm, 'ai_assistant', false);
try {
expect((await upload({ ...page, userId: s.userId, journeyId: s.journeyId })).status).toBe(404);
} finally {
await setAddonEnabled(orm, 'ai_assistant', true);
}
vi.stubEnv('TREK_MANAGED', 'true');
try {
const managed = await upload({ ...page, userId: s.userId, journeyId: s.journeyId });
expect(managed.status).toBe(403);
expect(managed.body).toMatchObject({ code: 'MANAGED_FORBIDDEN' });
} finally {
vi.unstubAllEnvs();
}
await setAddonEnabled(orm, 'journey', false);
try {
const off = await upload({ ...page, userId: s.userId, journeyId: s.journeyId });
expect([off.status, off.body]).toEqual([404, { error: 'Journey addon is not enabled' }]);
} finally {
await setAddonEnabled(orm, 'journey', true);
}
const outsider = await upload({ ...page, userId: stranger.id, journeyId: s.journeyId });
expect([outsider.status, outsider.body]).toEqual([404, { error: 'Journey not found' }]);
expect((await upload({ ...page, userId: s.userId, journeyId: 'abc' })).status).toBe(404);
const path = `/api/journeys/${s.journeyId}/assistant/threads/4/renders/x`;
expect((await request(nestApp.getHttpServer()).post(path)).status).toBe(401);
turn.stop.abort();
await expect(turn.image).resolves.toBeNull();
});
it('ASTRNE2E-002: an upload that is no image, too large or without its fields is refused', async () => {
const s = await studio();
const turn = waitFor(s.userId, s.journeyId);
const base = { requestId: turn.requestId, userId: s.userId, journeyId: s.journeyId };
expect((await upload({ ...base })).status).toBe(400);
const text = await upload({ ...base, image: Buffer.from('not an image at all') });
expect([text.status, text.body]).toEqual([415, { error: 'The upload is not a JPEG, PNG or WebP image' }]);
expect((await upload({ ...base, image: jpeg(4000, 1000) })).status).toBe(413);
expect((await upload({ ...base, image: jpeg(2048, 2048) })).status).toBe(413);
expect((await upload({ ...base, image: jpeg(1280, 640, 1024 * 1024 + 1) })).status).toBe(413);
const noPage = await upload({ ...base, image: jpeg(1280, 640), fields: { bookVersion: '3' } });
expect(noPage.status).toBe(400);
const extra = { bookVersion: '3', spreadId: 'sp-2', data: 'x' };
expect((await upload({ ...base, image: jpeg(1280, 640), fields: extra })).status).toBe(400);
turn.stop.abort();
await expect(turn.image).resolves.toBeNull();
});
it('ASTRNE2E-003: an image that shows a photo of another member is refused; the turn goes on waiting', async () => {
const s = await studio();
const turn = waitFor(s.userId, s.journeyId);
const base = { requestId: turn.requestId, userId: s.userId, journeyId: s.journeyId, image: jpeg(1280, 640) };
const fields = { bookVersion: '3', spreadId: 'sp-2', drawnPhotoIds: `${s.own},${s.friends}` };
const foreign = await upload({ ...base, fields });
expect([foreign.status, foreign.body]).toEqual([
403,
{ error: 'The image shows a photo another member contributed' },
]);
const mine = await upload({ ...base, fields: { ...fields, drawnPhotoIds: String(s.own) } });
expect([mine.status, mine.body]).toEqual([200, { accepted: true }]);
const image = (await turn.image) as PageImage;
expect(image).toMatchObject({ mimeType: 'image/jpeg', width: 1280, height: 640 });
expect(image.data.equals(jpeg(1280, 640))).toBe(true);
});
it('ASTRNE2E-004: only the waiting turn of this user and thread takes the page it asked for, once', async () => {
const s = await studio();
const turn = waitFor(s.userId, s.journeyId);
const base = { requestId: turn.requestId, journeyId: s.journeyId, image: jpeg(1280, 640) };
// The friend may open the journey, but the turn is not theirs.
expect((await upload({ ...base, userId: s.friendId })).status).toBe(404);
expect((await upload({ ...base, userId: s.userId, threadId: 5 })).status).toBe(404);
expect((await upload({ ...base, userId: s.userId, requestId: randomUUID() })).status).toBe(404);
const stale = await upload({ ...base, userId: s.userId, fields: { bookVersion: '2', spreadId: 'sp-2' } });
expect(stale.status).toBe(409);
expect((await upload({ ...base, userId: s.userId })).status).toBe(200);
await expect(turn.image).resolves.toMatchObject({ width: 1280, height: 640 });
const again = await upload({ ...base, userId: s.userId });
expect([again.status, again.body]).toEqual([404, { error: 'No page was asked for' }]);
});
});
@@ -267,6 +267,27 @@ describe('the instance connection', () => {
expect((await service().status(id)).privacyAcked).toBe(false);
});
it('AICS-016: images of Studio pages are acknowledged on their own, per kind@host, on any connection', async () => {
const admin = await user();
const id = await user();
await service().saveInstance(admin, { kind: 'anthropic', apiKey: 'sk-ant-instance' });
const target = { kind: 'anthropic' as const, host: 'api.anthropic.com' };
expect((await service().status(id)).studioImagesAcked).toBe(false);
// The planner's acknowledgement says nothing about images.
await service().ackPrivacy(id, target);
expect(await service().studioImagesAcked(id, target)).toBe(false);
await service().ackPrivacy(id, { ...target, scope: 'studio_images' });
expect(await readUserSetting(orm(), id, 'assistant_privacy_ack_studio_images')).toContain(
'anthropic@api.anthropic.com',
);
expect(await service().studioImagesAcked(id, target)).toBe(true);
expect(await service().status(id)).toMatchObject({ privacyAcked: true, studioImagesAcked: true });
// Another provider asks again.
expect(await service().studioImagesAcked(id, { kind: 'openai', host: 'api.openai.com' })).toBe(false);
await service().saveInstance(admin, { kind: 'openai', apiKey: 'sk-proj-other' });
expect((await service().status(id)).studioImagesAcked).toBe(false);
});
it('AICS-014: settings are read with their defaults and written back', async () => {
const admin = await user();
expect(await service().readSettings()).toEqual({
@@ -6,6 +6,7 @@
*/
import { AiConnectionsModule } from '../../../../src/nest/ai-connections/ai-connections.module';
import {
modelReadsImages,
ModelCatalogService,
pickDefaultModel,
withRecommendation,
@@ -66,7 +67,7 @@ describe('ModelCatalogService', () => {
safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'claude-sonnet-5', display_name: 'Claude Sonnet 5' }] }));
const key = secret();
expect(await catalog().list({ kind: 'anthropic', baseUrl: null, secret: key })).toEqual([
{ id: 'claude-sonnet-5', label: 'Claude Sonnet 5', supportsTools: true, recommended: true },
{ id: 'claude-sonnet-5', label: 'Claude Sonnet 5', supportsTools: true, supportsImages: true, recommended: true },
]);
const [url, init] = safeFetch.mock.calls[0] as [string, RequestInit, { maxBytes: number }];
expect(url).toBe('https://api.anthropic.com/v1/models?limit=1000');
@@ -109,6 +110,7 @@ describe('ModelCatalogService', () => {
id: 'anthropic/claude-sonnet-5',
name: 'Sonnet',
supported_parameters: ['tools'],
architecture: { input_modalities: ['text', 'image'] },
pricing: { prompt: '0.000003', completion: '0.000015' },
},
{ id: 'meta/llama:free', supported_parameters: ['tools'], pricing: { prompt: '0', completion: '0' } },
@@ -117,8 +119,15 @@ describe('ModelCatalogService', () => {
}),
);
expect(await catalog().list({ kind: 'openrouter', baseUrl: null, secret: secret() })).toEqual([
{ id: 'anthropic/claude-sonnet-5', label: 'Sonnet', supportsTools: true, free: false, recommended: true },
{ id: 'meta/llama:free', label: 'meta/llama:free', supportsTools: true, free: true },
{
id: 'anthropic/claude-sonnet-5',
label: 'Sonnet',
supportsTools: true,
free: false,
supportsImages: true,
recommended: true,
},
{ id: 'meta/llama:free', label: 'meta/llama:free', supportsTools: true, free: true, supportsImages: false },
]);
expect(safeFetch.mock.calls.map((call) => call[0])).toEqual([
'https://openrouter.ai/api/v1/key',
@@ -167,7 +176,7 @@ describe('ModelCatalogService', () => {
}),
);
expect(await catalog().list({ kind: 'chatgpt', baseUrl: null, secret: secret() })).toEqual([
{ id: 'gpt-6.1-sol', label: 'GPT-6.1 Sol', supportsTools: true, recommended: true },
{ id: 'gpt-6.1-sol', label: 'GPT-6.1 Sol', supportsTools: true, supportsImages: true, recommended: true },
]);
safeFetch.mockResolvedValueOnce(json(500, {}));
expect((await catalog().list({ kind: 'chatgpt', baseUrl: null, secret: secret() }))[0]?.id).toBe('gpt-6.1-sol');
@@ -218,6 +227,30 @@ describe('ModelCatalogService', () => {
expect(pickDefaultModel('openai', list('gpt-4.1-mini', 'gpt-5.5-mini', 'gpt-5-mini'))).toBe('gpt-5.5-mini');
});
it('MCAT-011: says which models read images: OpenAI by name, Gemini all, OpenRouter by its input kinds', async () => {
safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'gpt-5-mini' }, { id: 'gpt-4' }, { id: 'o3-mini' }] }));
const openai = await catalog().list({ kind: 'openai', baseUrl: null, secret: secret() });
expect(Object.fromEntries(openai.map((m) => [m.id, m.supportsImages]))).toEqual({
'gpt-4': false,
'gpt-5-mini': true,
'o3-mini': false,
});
safeFetch.mockResolvedValueOnce(json(200, { data: [{ id: 'models/gemini-3-flash' }] }));
expect((await catalog().list({ kind: 'gemini', baseUrl: null, secret: secret() }))[0]?.supportsImages).toBe(true);
});
it('MCAT-012: a turn tells from the kind and the id alone whether its model reads images', () => {
expect(modelReadsImages('anthropic', 'claude-sonnet-5')).toBe(true);
expect(modelReadsImages('gemini', 'gemini-3-flash')).toBe(true);
expect(modelReadsImages('chatgpt', 'gpt-6.1-sol')).toBe(true);
expect(modelReadsImages('openai', 'gpt-4o')).toBe(true);
expect(modelReadsImages('openai', 'gpt-4-0613')).toBe(false);
expect(modelReadsImages('openai', 'gpt-3.5-turbo')).toBe(false);
// OpenRouter and a custom server cannot be told from the id: the turn stays on the description.
expect(modelReadsImages('openrouter', 'anthropic/claude-sonnet-5')).toBe(false);
expect(modelReadsImages('custom', 'llava')).toBe(false);
});
it('MCAT-010: a picker lists the recommended model first and marked, the rest newest first', () => {
const list = (...ids: string[]) => ids.map((id) => ({ id, label: id, supportsTools: true }));
expect(
@@ -19,6 +19,9 @@ import {
AiProviderError,
CHATGPT_USAGE_URL,
errorCodeOf,
IMAGE_INPUT_REFUSED,
isImageRefusal,
namesImageInput,
providerFailure,
} from '../../../../src/nest/ai-connections/providers/provider-errors';
import { toProviderSchema, type AiFetch } from '../../../../src/nest/ai-connections/providers/provider-http';
@@ -542,3 +545,91 @@ describe('OpenAiResponsesClient', () => {
});
});
});
describe('images for a model that reads them', () => {
const PAGE = { mimeType: 'image/jpeg' as const, data: 'aGVsbG8=' };
const WITH_IMAGES: AiChatRequest = {
...REQUEST,
messages: [
{ role: 'user', text: 'Tidy page 2', images: [PAGE] },
{
role: 'assistant',
text: '',
toolCalls: [
{ id: 'call_1', name: 'preview_book_page', args: { spreadId: 'sp-2' } },
{ id: 'call_2', name: 'get_book', args: {} },
],
},
{ role: 'tool', toolCallId: 'call_1', name: 'preview_book_page', output: 'Page 2', images: [PAGE] },
{ role: 'tool', toolCallId: 'call_2', name: 'get_book', output: '{"pages":4}' },
],
};
it('PIMG-001: Anthropic puts a tool image in its tool_result after the text, a user image before the words', () => {
const turns = toAnthropicMessages(WITH_IMAGES.messages);
const image = { type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: 'aGVsbG8=' } };
expect(turns[0]?.content).toEqual([image, { type: 'text', text: 'Tidy page 2' }]);
expect(turns[2]?.content).toEqual([
{ type: 'tool_result', tool_use_id: 'call_1', content: [{ type: 'text', text: 'Page 2' }, image] },
{ type: 'tool_result', tool_use_id: 'call_2', content: '{"pages":4}' },
]);
});
it('PIMG-002: Chat Completions keeps the tool message text and sends the step images after its last result', () => {
const messages = toChatMessages(WITH_IMAGES);
const part = { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,aGVsbG8=', detail: 'high' } };
expect(messages[1]).toEqual({ role: 'user', content: [{ type: 'text', text: 'Tidy page 2' }, part] });
expect(messages.slice(3)).toEqual([
{ role: 'tool', tool_call_id: 'call_1', content: 'Page 2' },
{ role: 'tool', tool_call_id: 'call_2', content: '{"pages":4}' },
{
role: 'user',
content: [{ type: 'text', text: 'The image the call preview_book_page (call_1) produced:' }, part],
},
]);
});
it('PIMG-003: the Responses API sends the output as text, then the image in a user message', () => {
const input = toResponsesInput(WITH_IMAGES);
const image = { type: 'input_image', image_url: 'data:image/jpeg;base64,aGVsbG8=', detail: 'high' };
expect(dig(input, 1, 'content', 1)).toEqual(image);
expect(input.slice(-3)).toEqual([
{ type: 'function_call_output', call_id: 'call_1', output: 'Page 2' },
{ type: 'function_call_output', call_id: 'call_2', output: '{"pages":4}' },
{
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: 'The image the call preview_book_page (call_1) produced:' }, image],
},
]);
});
it('PIMG-004: a history without images reads as it always did, with no extra user message', () => {
expect(toChatMessages(REQUEST).filter((message) => Array.isArray(message.content))).toEqual([]);
expect(toAnthropicMessages(REQUEST.messages)[2]?.content).toEqual([
{ type: 'tool_result', tool_use_id: 'call_1', content: '{"id":7}' },
{ type: 'text', text: 'Thanks' },
]);
});
it('PIMG-005: a refusal over the image carries its own code; other refusals and plan limits do not', async () => {
const refused = (status: number, body: unknown) => {
const http = fetchAnswering(jsonResponse(status, body));
const client = new OpenAiChatClient('openai', 'gpt-4', 'https://o', 'k', http.fetcher);
return failure(client.stream(WITH_IMAGES, signal()));
};
const imageRefusal = await refused(400, { error: { message: 'image_url is only supported by certain models.' } });
expect(imageRefusal).toMatchObject({ code: 'generic', providerCode: IMAGE_INPUT_REFUSED });
expect(isImageRefusal(imageRefusal)).toBe(true);
const other = await refused(400, { error: { code: 'model_not_found', message: 'No such model' } });
expect(isImageRefusal(other)).toBe(false);
expect(isImageRefusal(await refused(500, { error: { message: 'image service down' } }))).toBe(false);
const code = 'subscription_sharing_unsupported_capability';
const plan = await refused(400, { error: { code, message: 'Image input is not part of this plan' } });
expect(plan).toMatchObject({ code: 'not_eligible', providerCode: code });
expect(namesImageInput(422, { message: 'This model does not support vision' })).toBe(true);
expect(namesImageInput(400, { detail: 'Invalid input_image' })).toBe(true);
expect(namesImageInput(400, 'image')).toBe(false);
expect(isImageRefusal(new Error('image'))).toBe(false);
});
});
@@ -7,6 +7,7 @@
import type {
AiChatClient,
AiChatEvent,
AiChatMessage,
AiChatRequest,
} from '../../../../src/nest/ai-connections/ai-chat-client.interface';
import { AiProviderError } from '../../../../src/nest/ai-connections/ai-chat-client.interface';
@@ -14,6 +15,7 @@ import {
runAgentLoop,
type AgentDeps,
type AgentInput,
type AgentToolRun,
type HeldCall,
type StepRecord,
} from '../../../../src/nest/assistant/assistant-agent';
@@ -559,6 +561,34 @@ describe('runAgentLoop', () => {
]);
});
it("ASTAGT-023: a change its gate sends back is that call's error for the model, never held; the next one is", async () => {
const refusalOf = vi.fn(async (_tool: string, args: Record<string, unknown>) =>
args.name === 'Broken' ? 'Not proposed yet: two photos overlap' : null,
);
const t = setup(
[
[
{ type: 'tool-call', call: { id: 'w1', name: 'create_place', args: { name: 'Broken' } } },
{ type: 'tool-call', call: { id: 'r1', name: 'list_places', args: {} } },
],
[{ type: 'tool-call', call: { id: 'w2', name: 'create_place', args: { name: 'Fixed' } } }],
],
{ refusalOf },
);
const outcome = await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal);
expect(outcome.end).toBe('awaiting');
// Only changes are asked about, each with the arguments it would be held with.
expect(refusalOf.mock.calls).toEqual([
['create_place', { name: 'Broken' }],
['create_place', { name: 'Fixed' }],
]);
expect(t.deps.describe).toHaveBeenCalledTimes(1);
const sentBack = t.requests[1].messages.find((m) => m.role === 'tool' && m.toolCallId === 'w1');
expect(sentBack).toMatchObject({ isError: true, output: expect.stringContaining('two photos overlap') });
expect(t.chunks).toContainEqual({ type: 'tool-output-error', toolCallId: 'w1', errorText: expect.any(String) });
expect(t.memory.held[0]?.held.map((call) => call.args)).toEqual([{ name: 'Fixed' }]);
});
describe('answer cards', () => {
/** A card as a package would build it; its fields do not matter to the loop. */
const CARD = { kind: 'tourList' } as TrekCard;
@@ -623,4 +653,83 @@ describe('runAgentLoop', () => {
expect(t.memory.saved[0].step.parts.map((part) => part.type)).toEqual(['tool-call']);
});
});
describe('page images', () => {
const PAGE = { mimeType: 'image/jpeg' as const, data: 'aGVsbG8=' };
const VIEW = { spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 5, description: 'Page 2' };
const LOOKED: AgentToolRun = { ok: true, value: { description: 'Page 2' }, images: [PAGE, PAGE], view: VIEW };
const look = (id: string) => ({ type: 'tool-call' as const, call: { id, name: 'list_places', args: {} } });
const done = (): Step => [{ type: 'text-delta', delta: 'Looks good.' }];
const vision = () => ({
refused: vi.fn(),
afterApply: vi.fn(async (history: readonly AiChatMessage[]) => [...history]),
});
const imagesOf = (messages: AiChatMessage[]) =>
messages.filter((m) => m.role === 'tool').map((m) => (m.role === 'tool' ? (m.images?.length ?? 0) : 0));
it('ASTAGT-030: a look keeps what it looked at in its step and streams it; one image goes to the next request', async () => {
const t = setup([[look('c1')], done()], { vision: vision() });
t.tools.call.mockImplementation(async () => LOOKED);
await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal);
const parts = t.memory.saved[0]?.step.parts ?? [];
expect(parts.map((part) => part.type)).toEqual(['tool-call', 'data-trek-page-view']);
expect(parts[1]).toEqual({ type: 'data-trek-page-view', data: VIEW });
expect(t.chunks).toContainEqual({ type: 'data-trek-page-view', data: VIEW });
// Never stored, never streamed: the bytes live for the next request.
expect(JSON.stringify([t.memory.saved, t.chunks])).not.toContain('aGVsbG8=');
expect(t.requests[1]?.messages.find((m) => m.role === 'tool')).toMatchObject({
toolCallId: 'c1',
images: [PAGE],
});
});
it('ASTAGT-031: a request carries the images of the newest three looks, one each; older looks go as their text', async () => {
const t = setup([[look('c1')], [look('c2')], [look('c3')], [look('c4')], done()], { vision: vision() });
t.tools.call.mockImplementation(async () => LOOKED);
await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal);
expect(imagesOf(t.requests[1]?.messages ?? [])).toEqual([1]);
expect(imagesOf(t.requests[4]?.messages ?? [])).toEqual([0, 1, 1, 1]);
});
it('ASTAGT-032: a refused image drops the images, tells the vision and asks the step again, once', async () => {
const refusedImage = new AiProviderError('generic', 400, false, undefined, 'image_input_refused');
const seeing = vision();
const t = setup([[look('c1')], refusedImage, done()], { vision: seeing });
t.tools.call.mockImplementation(async () => LOOKED);
const outcome = await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal);
expect(outcome.end).toBe('stop');
expect(seeing.refused).toHaveBeenCalledOnce();
expect(t.requests).toHaveLength(3);
expect(imagesOf(t.requests[1]?.messages ?? [])).toEqual([1]);
expect(imagesOf(t.requests[2]?.messages ?? [])).toEqual([0]);
expect(t.memory.saved.map((row) => row.status)).toEqual(['complete', 'complete']);
});
it('ASTAGT-033: another refusal, or one without a vision, ends the turn as before, asked once', async () => {
const otherRefusal = new AiProviderError('generic', 400, false, undefined, 'model_not_found');
const seeing = vision();
const t = setup([[look('c1')], otherRefusal], { vision: seeing });
t.tools.call.mockImplementation(async () => LOOKED);
expect((await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal)).end).toBe('failed');
expect(seeing.refused).not.toHaveBeenCalled();
expect(t.requests).toHaveLength(2);
const refusedImage = new AiProviderError('generic', 400, false, undefined, 'image_input_refused');
const plain = setup([[look('c1')], refusedImage]);
plain.tools.call.mockImplementation(async () => LOOKED);
const plainOutcome = await runAgentLoop(plain.deps, plain.input, plain.writer, new AbortController().signal);
expect(plainOutcome.end).toBe('failed');
expect(plain.requests).toHaveLength(2);
});
it('ASTAGT-034: a turn starts from the history the vision gives back, with views of applied pages', async () => {
const seeing = vision();
const seen = { role: 'user' as const, text: 'seen', images: [PAGE] };
seeing.afterApply.mockImplementation(async (history) => [...history, seen]);
const t = setup([done()], { vision: seeing });
await runAgentLoop(t.deps, t.input, t.writer, new AbortController().signal);
expect(seeing.afterApply).toHaveBeenCalledWith(t.input.history);
expect(t.requests[0]?.messages.at(-1)).toEqual(seen);
});
});
});
@@ -0,0 +1,110 @@
/**
* withPagePreview (ASTPVW-*): a Studio turn's tools with `preview_book_page`
* beside them. The preview takes the session's own `edit_book_page` ops
* schema, runs as a read through the turn's vision, refuses arguments it does
* not take, and is capped per step and per turn.
*/
import type { AgentToolPort, AgentToolRun } from '../../../../src/nest/assistant/assistant-agent';
import { withPagePreview } from '../../../../src/nest/assistant/assistant-book-preview';
import type { TurnVision } from '../../../../src/nest/assistant/assistant-vision';
import { ASSISTANT_PAGE_IMAGE } from '@trek/shared';
import { describe, expect, it, vi } from 'vitest';
const OPS_SCHEMA = { type: 'array', items: { $ref: '#/$defs/op' } };
const EDIT = {
name: 'edit_book_page',
description: 'Edit a page',
parameters: { type: 'object', properties: { spreadId: { type: 'string' }, ops: OPS_SCHEMA }, $defs: { op: {} } },
};
function setup(specs = [EDIT]) {
const session: AgentToolPort = {
specs: () => specs,
kindOf: (name) => (name === 'edit_book_page' ? 'write' : 'read'),
pinArgs: vi.fn((_name: string, args: Record<string, unknown>) => ({ args })),
call: vi.fn(async (): Promise<AgentToolRun> => ({ ok: true, value: 'session' })),
};
const looked: AgentToolRun = { ok: true, value: { description: 'Page 2' } };
const vision = {
sendsImages: () => true,
look: vi.fn(async () => looked),
refused: vi.fn(),
afterApply: vi.fn(async () => []),
};
return { session, vision, looked, tools: withPagePreview(session, vision as TurnVision) };
}
describe('withPagePreview', () => {
it('ASTPVW-001: offers the preview after the session tools, its ops as the session edit_book_page takes them', () => {
const { tools } = setup();
const specs = tools.specs();
expect(specs.map((spec) => spec.name)).toEqual(['edit_book_page', 'preview_book_page']);
expect(specs[1]?.parameters).toMatchObject({
properties: { spreadId: { type: 'string' }, ops: OPS_SCHEMA },
required: ['spreadId'],
additionalProperties: false,
$defs: { op: {} },
});
// A viewer has no edit tool: the preview shows pages as they stand.
const viewer = setup([]).tools.specs()[0];
expect(viewer?.parameters.properties).toEqual({ spreadId: expect.objectContaining({ type: 'string' }) });
expect(viewer?.parameters).not.toHaveProperty('$defs');
});
it('ASTPVW-002: the preview is a read through the vision; every other tool stays the session one', async () => {
const { tools, session, vision, looked } = setup();
expect(tools.kindOf('preview_book_page')).toBe('read');
expect(tools.kindOf('edit_book_page')).toBe('write');
const pinned = tools.pinArgs('preview_book_page', { spreadId: 'sp-2', journeyId: 12 });
expect(pinned).toEqual({ args: { spreadId: 'sp-2' } });
expect(session.pinArgs).not.toHaveBeenCalled();
tools.pinArgs('edit_book_page', { spreadId: 'sp-2' });
expect(session.pinArgs).toHaveBeenCalledWith('edit_book_page', { spreadId: 'sp-2' });
const ops = [{ op: 'background', color: '#112233' }];
expect(await tools.call('preview_book_page', { spreadId: 'sp-2', ops })).toBe(looked);
expect(vision.look).toHaveBeenCalledWith('sp-2', ops);
expect(await tools.call('get_book', {})).toEqual({ ok: true, value: 'session' });
expect(session.call).toHaveBeenCalledWith('get_book', {});
});
it('ASTPVW-003: arguments the preview does not take are refused without a look', async () => {
const { tools, vision } = setup();
const refused = await tools.call('preview_book_page', { spreadId: '../x' });
expect(refused).toMatchObject({ ok: false, value: expect.stringContaining('Invalid arguments: spreadId') });
expect((await tools.call('preview_book_page', { spreadId: 'sp-2', ops: [] })).ok).toBe(false);
expect(vision.look).not.toHaveBeenCalled();
});
it('ASTPVW-004: previews are capped per step, which the next step resets, and per turn', async () => {
const { tools, vision } = setup();
expect(ASSISTANT_PAGE_IMAGE).toMatchObject({ previewsPerStep: 3, previewsPerTurn: 8 });
const preview = () => tools.call('preview_book_page', { spreadId: 'sp-2' });
/** `count` previews one after another, as the loop runs a step's calls; whether each was answered. */
const previews = (count: number) =>
Array.from({ length: count }).reduce<Promise<boolean[]>>(
async (done) => [...(await done), (await preview()).ok],
Promise.resolve([]),
);
tools.specs();
expect(await previews(4)).toEqual([true, true, true, false]);
expect(await preview()).toMatchObject({ ok: false, value: expect.stringContaining('Too many previews') });
tools.specs();
expect(await previews(3)).toEqual([true, true, true]);
tools.specs();
expect(await previews(3)).toEqual([true, true, false]);
expect(vision.look).toHaveBeenCalledTimes(8);
});
it('ASTPVW-005: a session that edits may preview a page of a design draft by its designId; a viewer is not offered it', async () => {
const { tools, vision, looked } = setup();
expect(tools.specs()[1]?.parameters.properties).toHaveProperty('designId');
expect(setup([]).tools.specs()[0]?.parameters.properties).not.toHaveProperty('designId');
expect(await tools.call('preview_book_page', { spreadId: 'sp-2', designId: 'd-1' })).toBe(looked);
expect(vision.look).toHaveBeenCalledWith('sp-2', undefined, 'd-1');
const refused = await tools.call('preview_book_page', { spreadId: 'sp-2', designId: '../d' });
expect(refused).toMatchObject({ ok: false, value: expect.stringContaining('designId') });
});
});
@@ -275,6 +275,31 @@ describe('toChatHistory', () => {
'The hike "Herzogstand" was not added; the user moved on without deciding.',
);
});
it('ASTHIS-014: a page the Studio drew is replayed as its description; no image is stored or sent again', () => {
const view = { spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 182_000, description: 'Page 2' };
const looked = { description: 'Page 2, a double spread', findings: [] };
const args = { spreadId: 'sp-2' };
const rows = [
row('user', [{ type: 'text', text: 'Check page 2' }]),
row('assistant', [
{ type: 'tool-call', toolCallId: 'p1', toolName: 'preview_book_page', args, result: looked },
// Whatever reaches the store is what the shared part keeps: no bytes.
{ type: 'data-trek-page-view', data: { ...view, data: 'aGVsbG8=' } as typeof view },
]),
row('assistant', [{ type: 'text', text: 'It looks balanced.' }]),
];
const history = toChatHistory(rows, new Map());
expect(history).toEqual([
{ role: 'user', text: 'Check page 2' },
{ role: 'assistant', text: '', toolCalls: [{ id: 'p1', name: 'preview_book_page', args }], providerState: null },
{ role: 'tool', toolCallId: 'p1', name: 'preview_book_page', output: JSON.stringify(looked) },
{ role: 'assistant', text: 'It looks balanced.', toolCalls: [], providerState: null },
]);
expect(JSON.stringify(history)).not.toContain('aGVsbG8=');
const [, answer] = toThreadMessages(rows, new Map(), NOW);
expect(answer?.parts[1]).toEqual({ type: 'data-trek-page-view', data: view });
});
});
describe('toThreadMessages', () => {
@@ -0,0 +1,81 @@
/**
* InMemoryAssistantRenderRegistry (ASTRND-*): a turn waits for the page image
* it asked the Studio for; the upload is handed over only to the turn of the
* same user, journey and thread, for the page and version asked for, and only
* while the turn still waits.
*/
import {
InMemoryAssistantRenderRegistry,
type PageImage,
type RenderWait,
} from '../../../../src/nest/assistant/assistant-render.registry';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const REQUEST: RenderWait = {
userId: 7,
journeyId: 12,
threadId: 4,
requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76',
spreadId: 'sp-2',
bookVersion: 3,
};
const PAGE = { spreadId: 'sp-2', bookVersion: 3 };
const IMAGE: PageImage = { mimeType: 'image/jpeg', data: Buffer.from('jpeg'), width: 1280, height: 640 };
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
describe('InMemoryAssistantRenderRegistry', () => {
it('ASTRND-001: the upload of the page asked for reaches the waiting turn once', async () => {
const registry = new InMemoryAssistantRenderRegistry();
const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000);
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken');
await expect(waiting).resolves.toBe(IMAGE);
// Taken: a second upload for the same request finds nobody waiting.
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown');
});
it('ASTRND-002: a turn stops waiting after the timeout, and an upload after it is too late', async () => {
const registry = new InMemoryAssistantRenderRegistry();
const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000);
await vi.advanceTimersByTimeAsync(15_000);
await expect(waiting).resolves.toBeNull();
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown');
});
it('ASTRND-003: a stopped turn stops waiting, also when it was stopped before it asked', async () => {
const registry = new InMemoryAssistantRenderRegistry();
const stop = new AbortController();
const waiting = registry.wait(REQUEST, stop.signal, 15_000);
stop.abort();
await expect(waiting).resolves.toBeNull();
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown');
await expect(registry.wait(REQUEST, stop.signal, 15_000)).resolves.toBeNull();
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('unknown');
});
it('ASTRND-004: another user, journey or thread cannot answer the request; the turn goes on waiting', async () => {
const registry = new InMemoryAssistantRenderRegistry();
const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000);
expect(registry.deliver({ ...REQUEST, userId: 8 }, PAGE, IMAGE)).toBe('unknown');
expect(registry.deliver({ ...REQUEST, journeyId: 13 }, PAGE, IMAGE)).toBe('unknown');
expect(registry.deliver({ ...REQUEST, threadId: 5 }, PAGE, IMAGE)).toBe('unknown');
expect(registry.deliver({ ...REQUEST, requestId: 'other' }, PAGE, IMAGE)).toBe('unknown');
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken');
await expect(waiting).resolves.toBe(IMAGE);
});
it('ASTRND-005: an image of another page or version is refused, and the right one still comes through', async () => {
const registry = new InMemoryAssistantRenderRegistry();
const waiting = registry.wait(REQUEST, new AbortController().signal, 15_000);
expect(registry.deliver(REQUEST, { ...PAGE, spreadId: 'sp-3' }, IMAGE)).toBe('mismatch');
expect(registry.deliver(REQUEST, { ...PAGE, bookVersion: 2 }, IMAGE)).toBe('mismatch');
expect(registry.deliver(REQUEST, PAGE, IMAGE)).toBe('taken');
await expect(waiting).resolves.toBe(IMAGE);
});
});
@@ -0,0 +1,258 @@
/**
* AssistantVision (ASTVSN-*): a Studio turn looking at a page. Images go only
* when the client draws pages, the model reads images and the user let the
* connection receive them; only the user's own photos are drawn; no image in
* time is the description alone; a refused image makes the model text-only
* for the process; a decision's turn starts with views of the pages the
* applied book writes left. The book, the photos and the acknowledgement are
* fakes; the registry is the real in-memory one, the request context a real
* ORM over a snapshot test db.
*/
import type { AiChatMessage } from '../../../../src/nest/ai-connections/ai-chat-client.interface';
import type { AiConnectionsService } from '../../../../src/nest/ai-connections/ai-connections.service';
import { InMemoryAssistantRenderRegistry } from '../../../../src/nest/assistant/assistant-render.registry';
import type { AssistantToolSession } from '../../../../src/nest/assistant/assistant-tools.service';
import { AssistantVision, type VisionTurnInput } from '../../../../src/nest/assistant/assistant-vision';
import type { UiStreamWriter } from '../../../../src/nest/assistant/ui-stream.writer';
import { DomainError } from '../../../../src/nest/common/domain-error';
import type { JourneyAiPhotosService } from '../../../../src/nest/journey/journey-ai-photos.service';
import type { JourneyBookEditService } from '../../../../src/nest/journey/journey-book-edit.service';
import { createSnapshotTestDb } from '../../../helpers/db-mock';
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
import {
blankBook,
bookMapElementSchema,
bookPhotoElementSchema,
type AssistantPageRender,
type BookDocument,
} from '@trek/shared';
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
const testDb = createSnapshotTestDb();
let orm: TestOrm;
beforeAll(async () => {
orm = await createTestOrm(testDb);
});
afterAll(async () => {
await orm.close();
testDb.close();
});
const JPEG = Buffer.from('jpeg-bytes');
/** A book whose inner spread holds photo 1 (the user's) and photo 2 (another member's). */
function book(): { doc: BookDocument; inner: string } {
let n = 0;
const doc = blankBook({ title: 'Lisbon', newId: (prefix) => `${prefix}-${++n}` });
const inner = doc.spreads[2];
if (!inner) throw new Error('no inner spread');
const photo = (id: string, photoId: number, x: number) =>
bookPhotoElementSchema.parse({ id, kind: 'photo', frame: { x, y: 20, w: 120, h: 90 }, photoId });
doc.spreads[2] = { ...inner, elements: [photo('p-own', 1, 20), photo('p-other', 2, 260)] };
return { doc, inner: inner.id };
}
interface Fakes {
acked?: boolean;
doc?: BookDocument | null;
/** What the Studio does with a render request: upload the page, or nothing (the turn is stopped instead). */
upload?: boolean;
}
function setup(fakes: Fakes = {}) {
const registry = new InMemoryAssistantRenderRegistry();
const connections = { studioImagesAcked: vi.fn(async () => fakes.acked ?? true) };
const fixture = book();
const books = {
trialBase: vi.fn(async () => ({
doc: fakes.doc === undefined ? fixture.doc : fakes.doc,
version: fakes.doc === null ? null : 4,
ctx: { photos: [{ photoId: 1 }, { photoId: 2 }] },
})),
};
const photos = { ownPhotoIds: vi.fn(async () => new Set([1])) };
const vision = new AssistantVision(
connections as unknown as AiConnectionsService,
books as unknown as JourneyBookEditService,
photos as unknown as JourneyAiPhotosService,
registry,
orm,
);
const stop = new AbortController();
const asked: AssistantPageRender[] = [];
const w = {
renderRequest: vi.fn((request: AssistantPageRender) => {
asked.push(request);
if (fakes.upload === false) {
stop.abort();
return;
}
const key = { userId: 7, journeyId: 12, threadId: 4, requestId: request.requestId };
registry.deliver(key, request, { mimeType: 'image/jpeg', data: JPEG, width: 1280, height: 640 });
}),
};
const input: VisionTurnInput = {
userId: 7,
journeyId: 12,
threadId: 4,
connection: { kind: 'anthropic', model: 'claude-sonnet-5', host: 'api.anthropic.com' },
renders: true,
w: w as unknown as UiStreamWriter,
signal: stop.signal,
};
return { vision, connections, books, photos, asked, input, inner: fixture.inner };
}
describe('AssistantVision', () => {
it('ASTVSN-001: images go only when the Studio draws, the model reads images and the user said yes', async () => {
const t = setup();
expect((await t.vision.forTurn(t.input)).sendsImages()).toBe(true);
expect(t.connections.studioImagesAcked).toHaveBeenCalledWith(7, t.input.connection);
expect((await t.vision.forTurn({ ...t.input, renders: false })).sendsImages()).toBe(false);
const textModel = { kind: 'custom' as const, model: 'llama3', host: 'ollama:11434' };
expect((await t.vision.forTurn({ ...t.input, connection: textModel })).sendsImages()).toBe(false);
const notAcked = setup({ acked: false });
expect((await notAcked.vision.forTurn(notAcked.input)).sendsImages()).toBe(false);
});
it('ASTVSN-002: a look draws only the user own photos; the image goes to the model, never stored', async () => {
const t = setup();
const run = await (await t.vision.forTurn(t.input)).look(t.inner);
expect(t.asked).toEqual([
{ requestId: expect.any(String), threadId: 4, spreadId: t.inner, bookVersion: 4, hiddenPhotoIds: [2] },
]);
expect(run.ok).toBe(true);
expect(run.images).toEqual([{ mimeType: 'image/jpeg', data: JPEG.toString('base64') }]);
expect(run.view).toMatchObject({ spreadId: t.inner, bookVersion: 4, width: 1280, height: 640, bytes: JPEG.length });
expect(run.view?.description).toContain('photo 2, by another contributor, not shown');
expect(run.value).toMatchObject({ spreadId: t.inner, proposed: false, findings: expect.any(Array) });
expect(JSON.stringify([run.value, run.view])).not.toContain(JPEG.toString('base64'));
});
it('ASTVSN-003: a text-only turn describes the page without asking the Studio or hiding anything', async () => {
const t = setup();
const run = await (await t.vision.forTurn({ ...t.input, renders: false })).look(t.inner);
expect(t.asked).toEqual([]);
expect(t.photos.ownPhotoIds).not.toHaveBeenCalled();
expect(run.images).toBeUndefined();
expect(run.view).toMatchObject({ width: null, height: null, bytes: null });
expect(run.view?.description).not.toContain('not shown');
expect(run.value).not.toHaveProperty('image');
});
it('ASTVSN-004: a proposal is drawn on a copy with its ops; no image in time leaves the description', async () => {
const t = setup({ upload: false });
const ops = [{ op: 'background' as const, color: '#112233' }];
const run = await (await t.vision.forTurn(t.input)).look(t.inner, ops);
expect(t.asked[0]).toMatchObject({ spreadId: t.inner, bookVersion: 4, ops });
expect(run.ok).toBe(true);
expect(run.images).toBeUndefined();
expect(run.value).toMatchObject({ proposed: true, image: expect.stringContaining('did not send') });
expect(t.books.trialBase).toHaveBeenCalledWith(12, 7);
});
it('ASTVSN-005: a look at nothing says why: no book, no such page, an op the book refuses, no access', async () => {
const t = setup();
const vision = await t.vision.forTurn(t.input);
expect(await vision.look('sp-missing')).toMatchObject({ ok: false, value: expect.stringContaining('no page') });
const locked = await vision.look(t.inner, [{ op: 'remove', elementId: 'nope' } as never]);
expect(locked).toMatchObject({ ok: false, value: expect.stringContaining('Not a change the book takes') });
const empty = setup({ doc: null });
const noBook = await (await empty.vision.forTurn(empty.input)).look('sp-1');
expect(noBook).toEqual({ ok: false, value: 'This journey has no book yet: create_book makes one.' });
t.books.trialBase.mockRejectedValueOnce(new DomainError(404, 'Journey not found'));
expect(await vision.look(t.inner)).toEqual({ ok: false, value: 'Journey not found' });
expect(t.asked).toEqual([]);
});
it('ASTVSN-006: a refused image makes the model text-only, for this turn and the next', async () => {
const t = setup();
const vision = await t.vision.forTurn(t.input);
vision.refused();
expect(vision.sendsImages()).toBe(false);
expect((await vision.look(t.inner)).images).toBeUndefined();
expect((await t.vision.forTurn(t.input)).sendsImages()).toBe(false);
const other = { ...t.input.connection, model: 'claude-opus-5' };
expect((await t.vision.forTurn({ ...t.input, connection: other })).sendsImages()).toBe(true);
});
it('ASTVSN-007: a decision turn starts with a view of each page the applied book writes left', async () => {
const t = setup();
const written = JSON.stringify({ version: 5, spread_ids: [t.inner], pages: [], findings: [], dropped: [] });
const history: AiChatMessage[] = [
{ role: 'user', text: 'Tidy it' },
{ role: 'assistant', text: '', toolCalls: [] },
{ role: 'tool', toolCallId: 'a', name: 'edit_book_page', output: written },
{ role: 'tool', toolCallId: 'b', name: 'edit_book_page', output: written },
{ role: 'tool', toolCallId: 'c', name: 'get_book', output: '{"exists":true}' },
{ role: 'tool', toolCallId: 'd', name: 'edit_book_page', output: written, isError: true },
];
const seen = await (await t.vision.forTurn(t.input)).afterApply(history);
expect(t.asked).toHaveLength(1);
expect(seen.map((m) => (m.role === 'tool' ? (m.images?.length ?? 0) : null))).toEqual([null, null, 0, 1, 0, 0]);
expect(history[3]).not.toHaveProperty('images');
const textOnly = setup();
const unchanged = await (await textOnly.vision.forTurn({ ...textOnly.input, renders: false })).afterApply(history);
expect(unchanged).toEqual(history);
expect(textOnly.books.trialBase).not.toHaveBeenCalled();
});
it('ASTVSN-008: a Studio turn gets the page preview beside its tools; a trip turn its session as it is', async () => {
const t = setup();
const session = {
specs: () => [],
kindOf: () => null,
pinArgs: (_name: string, args: Record<string, unknown>) => ({ args }),
call: async () => ({ ok: true, value: null }),
} as unknown as AssistantToolSession;
const trip = await t.vision.turnTools({ kind: 'trip', tripId: 3 }, session, t.input);
expect(trip).toEqual({ tools: session, vision: null });
const studio = await t.vision.turnTools({ kind: 'journey', journeyId: 12 }, session, t.input);
expect(studio.vision?.sendsImages()).toBe(true);
expect(studio.tools.specs().map((spec) => spec.name)).toEqual(['preview_book_page']);
});
it('ASTVSN-009: a map that marks its stops with photos draws only the user own ones', async () => {
const t = setup();
const { doc, inner } = book();
const spread = doc.spreads[2];
if (!spread) throw new Error('no inner spread');
const point = (photoId: number | null) => ({ lat: 38.7, lng: -9.1, label: '', photoId });
const map = bookMapElementSchema.parse({
id: 'm-1',
kind: 'map',
frame: { x: 20, y: 140, w: 200, h: 120 },
pinStyle: 'photo',
points: [point(1), point(3), point(null), point(3)],
});
doc.spreads[2] = { ...spread, elements: [...spread.elements, map] };
t.books.trialBase.mockResolvedValueOnce({ doc, version: 4, ctx: { photos: [{ photoId: 1 }, { photoId: 2 }] } });
await (await t.vision.forTurn(t.input)).look(inner);
expect(t.asked[0]?.hiddenPhotoIds).toEqual([2, 3]);
});
it('ASTVSN-010: a journey that went away while the turn ran is a look that says so, with no page drawn', async () => {
const t = setup();
const vision = await t.vision.forTurn(t.input);
t.photos.ownPhotoIds.mockRejectedValueOnce(new DomainError(404, 'Journey not found'));
expect(await vision.look(t.inner)).toEqual({ ok: false, value: 'Journey not found' });
expect(t.asked).toEqual([]);
});
it('ASTVSN-011: a page of a design draft is read from the draft and the Studio is told to draw it from there', async () => {
const t = setup();
const vision = await t.vision.forTurn(t.input);
const run = await vision.look(t.inner, undefined, 'd-1');
expect(t.books.trialBase).toHaveBeenCalledWith(12, 7, 'd-1');
expect(t.asked[0]).toMatchObject({ spreadId: t.inner, bookVersion: 4, designId: 'd-1', hiddenPhotoIds: [2] });
expect(run.ok).toBe(true);
await vision.look(t.inner);
expect(t.books.trialBase).toHaveBeenLastCalledWith(12, 7);
expect(t.asked[1]).not.toHaveProperty('designId');
});
});
@@ -217,4 +217,32 @@ describe('UiStreamWriter', () => {
w.close();
expect(stream.text()).toBe(CARDS_FIXTURE);
});
it('ASTWRT-007: asks the Studio for a page in a transient chunk and tells the page it looked at', () => {
const stream = memoryStream();
const w = new UiStreamWriter(stream);
w.text('Let me look');
w.renderRequest({
requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76',
threadId: 4,
spreadId: 'sp-2',
bookVersion: 3,
hiddenPhotoIds: [12],
});
w.pageView({ spreadId: 'sp-2', bookVersion: 3, width: 1280, height: 640, bytes: 1000, description: 'Page 2' });
const chunks = stream
.text()
.split('\n\n')
.filter((event) => event.startsWith('data: '))
.map((event) => uiStreamChunkSchema.parse(JSON.parse(event.slice('data: '.length))));
expect(chunks.map((chunk) => chunk.type)).toEqual([
'text-start',
'text-delta',
'text-end',
'data-trek-render-request',
'data-trek-page-view',
]);
expect(chunks[3]).toMatchObject({ transient: true, data: { spreadId: 'sp-2', hiddenPhotoIds: [12] } });
expect(chunks[4]).not.toHaveProperty('transient');
});
});
@@ -0,0 +1,57 @@
import { readImageHeader } from '../../../../src/nest/common/image-header';
import { describe, expect, it } from 'vitest';
/** SOI, then the given segments, then a baseline frame header of w x h. */
function jpeg(w: number, h: number, ...before: number[][]): Buffer {
const sof = Buffer.alloc(19);
sof.writeUInt16BE(0xffc0, 0);
sof.writeUInt16BE(17, 2);
sof.writeUInt8(8, 4);
sof.writeUInt16BE(h, 5);
sof.writeUInt16BE(w, 7);
return Buffer.concat([Buffer.from([0xff, 0xd8]), ...before.map((bytes) => Buffer.from(bytes)), sof]);
}
function webp(chunk: string, length: number): Buffer {
const out = Buffer.alloc(length);
out.write('RIFF', 0, 'latin1');
out.write('WEBP', 8, 'latin1');
out.write(chunk, 12, 'latin1');
return out;
}
describe('readImageHeader', () => {
it('IMGHDR-001: steps over stray bytes, fill bytes and the markers that carry no length', () => {
const segments = [
[0x00, 0x12],
[0xff, 0xff],
[0xff, 0x01],
[0xff, 0xd0],
[0xff, 0xe1, 0x00, 0x04, 0, 0],
];
expect(readImageHeader(jpeg(800, 600, ...segments))).toEqual({ mimeType: 'image/jpeg', width: 800, height: 600 });
});
it('IMGHDR-002: a JPEG whose segment length is broken, or whose frame header is cut, says no size', () => {
expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x01, 0, 0]))).toBeNull();
expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 8, 0]))).toBeNull();
expect(readImageHeader(Buffer.from([0xff, 0xd8, 0xff, 0xda, 0x00, 0x08]))).toBeNull();
expect(readImageHeader(jpeg(0, 600))).toBeNull();
});
it('IMGHDR-003: a WebP too short for its chunk, or with a chunk it does not know, says no size', () => {
expect(readImageHeader(webp('VP8 ', 20))).toBeNull();
expect(readImageHeader(webp('VP8L', 22))).toBeNull();
expect(readImageHeader(webp('VP8X', 26))).toBeNull();
expect(readImageHeader(webp('ALPH', 30))).toBeNull();
});
it('IMGHDR-004: a PNG signature without its IHDR chunk is no PNG', () => {
const data = Buffer.alloc(24);
data.writeUInt32BE(0x89504e47, 0);
data.write('IDAT', 12, 'latin1');
expect(readImageHeader(data)).toBeNull();
expect(readImageHeader(Buffer.from([0xff, 0xd8]))).toBeNull();
});
});
@@ -0,0 +1,72 @@
/**
* JourneyAiPhotosService (JAIPH-*): which gallery photos the Studio assistant
* may show a model, taken from JourneyBookModule over a snapshot test db.
* Only the asking user's own photos, the journey's owner standing in for a
* photo that names nobody; a stranger is told the journey is not there.
*/
import { JourneyPhotos } from '../../../src/db/entities/JourneyPhotos.entity';
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
import { JourneyAiPhotosService } from '../../../src/nest/journey/journey-ai-photos.service';
import { JourneyBookModule } from '../../../src/nest/journey/journey-book.module';
import { createSnapshotTestDb } from '../../helpers/db-mock';
import { addJourneyContributor, makeJourney } from '../../helpers/factories/journeys';
import { insertRow } from '../../helpers/factories/rows';
import { makeUser } from '../../helpers/factories/users';
import { resetTestDb } from '../../helpers/test-db';
import { createTestModule, type TestModule } from '../../helpers/test-module';
import { createTestOrm, type TestOrm } from '../../helpers/test-orm';
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
const db = createSnapshotTestDb();
let t: TestModule;
let orm: TestOrm;
beforeAll(async () => {
t = await createTestModule({ db, imports: [JourneyBookModule] });
orm = await createTestOrm(db);
});
beforeEach(() => {
resetTestDb(db);
orm.clear();
});
afterAll(async () => {
await orm.close();
await t.close();
db.close();
});
/** A photo in the journey's gallery, contributed by `ownerId` (null: nobody named); answers its photo id. */
async function galleryPhoto(journeyId: number, ownerId: number | null): Promise<number> {
const photoId = await insertRow(orm, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg' });
await insertRow(orm, JourneyPhotos, { journey: journeyId, photo: photoId, sort_order: 0, created_at: Date.now() });
return photoId;
}
describe('JourneyAiPhotosService', () => {
it('JAIPH-001: each member may show their own photos only; a photo naming nobody counts as the owner', async () => {
const { user: owner } = await makeUser(orm);
const { user: editor } = await makeUser(orm);
const journey = await makeJourney(orm, owner.id);
await addJourneyContributor(orm, journey.id, editor.id, 'editor');
const ownersPhoto = await galleryPhoto(journey.id, owner.id);
const unnamed = await galleryPhoto(journey.id, null);
const editorsPhoto = await galleryPhoto(journey.id, editor.id);
const service = t.get(JourneyAiPhotosService);
expect([...(await service.ownPhotoIds(journey.id, owner.id))].sort()).toEqual([ownersPhoto, unnamed].sort());
expect([...(await service.ownPhotoIds(journey.id, editor.id))]).toEqual([editorsPhoto]);
});
it('JAIPH-002: a photo of another journey never counts; a stranger is told the journey is not there', async () => {
const { user: owner } = await makeUser(orm);
const { user: stranger } = await makeUser(orm);
const journey = await makeJourney(orm, owner.id);
const other = await makeJourney(orm, owner.id);
await galleryPhoto(other.id, owner.id);
const service = t.get(JourneyAiPhotosService);
expect((await service.ownPhotoIds(journey.id, owner.id)).size).toBe(0);
await expect(service.ownPhotoIds(journey.id, stranger.id)).rejects.toThrow('Journey not found');
});
});
@@ -1,10 +1,10 @@
import { readImageHeader } from '../../../../src/nest/common/image-header';
import {
capImage,
IMAGE_MAX_BYTES,
IMAGE_MAX_EDGE,
imageMimeType,
ImageTooLargeError,
readImageHeader,
renderPdfPages,
} from '../../../../src/nest/llm-parse/image-input';
@@ -3,7 +3,9 @@ import {
aiApiKeyConnectRequestSchema,
aiChatgptFinishRequestSchema,
aiInstanceConnectionRequestSchema,
aiModelInfoSchema,
aiOpenrouterStartRequestSchema,
aiPrivacyAckRequestSchema,
aiRetentionSchema,
aiStatusResponseSchema,
} from './ai-connections.schema';
@@ -145,3 +147,22 @@ describe('aiStatusResponseSchema', () => {
).toBe(true);
});
});
describe('the Studio page image', () => {
it('AIVIS-001: a model may say whether it reads images, and a list that cannot say leaves it out', () => {
expect(aiModelInfoSchema.parse({ id: 'm', label: 'm', supportsTools: true, supportsImages: false })).toEqual({
id: 'm',
label: 'm',
supportsTools: true,
supportsImages: false,
});
expect(aiModelInfoSchema.safeParse({ id: 'm', label: 'm', supportsTools: true }).success).toBe(true);
});
it('AIVIS-002: the acknowledgement of page images is its own scope; no other scope exists', () => {
const ack = { kind: 'anthropic', host: 'api.anthropic.com' };
expect(aiPrivacyAckRequestSchema.safeParse(ack).success).toBe(true);
expect(aiPrivacyAckRequestSchema.safeParse({ ...ack, scope: 'studio_images' }).success).toBe(true);
expect(aiPrivacyAckRequestSchema.safeParse({ ...ack, scope: 'everything' }).success).toBe(false);
});
});
+9 -1
View File
@@ -48,6 +48,8 @@ export const aiModelInfoSchema = z.object({
free: z.boolean().optional(),
// The one a new connection starts on: the newest stable model of a capable line. Listed first.
recommended: z.boolean().optional(),
// Whether the model reads images (the Studio assistant's page image); left out where the list cannot say.
supportsImages: z.boolean().optional(),
});
export type AiModelInfo = z.infer<typeof aiModelInfoSchema>;
@@ -88,6 +90,8 @@ export const aiStatusResponseSchema = z.object({
quota: z.object({ used: z.number().int(), limit: z.number().int() }).nullable(),
// Whether the user acknowledged that the effective kind@host receives their data.
privacyAcked: z.boolean(),
// Whether the user also let kind@host receive images of their Studio pages, with their own photos on them.
studioImagesAcked: z.boolean().optional(),
retention: aiRetentionSchema,
// OpenRouter's OAuth needs an https callback; without one the client offers the key field instead.
openrouterOAuth: z.boolean(),
@@ -142,11 +146,15 @@ export type AiAuthStartResponse = z.infer<typeof aiAuthStartResponseSchema>;
export const aiModelListResponseSchema = z.object({ models: z.array(aiModelInfoSchema) });
export type AiModelListResponse = z.infer<typeof aiModelListResponseSchema>;
/** POST /api/ai/privacy-ack: the user saw that kind@host receives their messages. */
/**
* POST /api/ai/privacy-ack: the user saw that kind@host receives their
* messages, or (`scope: 'studio_images'`) also images of their Studio pages.
*/
export const aiPrivacyAckRequestSchema = z
.object({
kind: aiConnectionKindSchema,
host: z.string().max(253),
scope: z.enum(['studio_images']).optional(),
})
.strict();
export type AiPrivacyAckRequest = z.infer<typeof aiPrivacyAckRequestSchema>;
@@ -0,0 +1,85 @@
import {
ASSISTANT_PAGE_IMAGE,
assistantPageRenderSchema,
assistantPageViewSchema,
assistantRenderUploadRequestSchema,
drawnPhotoIdsOf,
previewBookPageRequestSchema,
} from './assistant-vision.schema';
import { assistantMessagePartSchema, uiStreamChunkSchema } from './assistant.schema';
import { describe, expect, it } from 'vitest';
const RENDER = {
requestId: '0b6f7c1e-3a52-4d8e-9a7b-2f1e0c9d8a76',
threadId: 4,
spreadId: 'sp-1',
bookVersion: 7,
hiddenPhotoIds: [12],
};
const VIEW = { spreadId: 'sp-1', bookVersion: 7, width: 1280, height: 640, bytes: 182_000, description: 'Page 2' };
describe('the Studio page image', () => {
it('ASTVIS-001: a render request names the page, the version and the photos left out; ops are a proposal', () => {
expect(assistantPageRenderSchema.safeParse(RENDER).success).toBe(true);
const ops = [{ op: 'background', color: '#ffffff' }];
expect(assistantPageRenderSchema.safeParse({ ...RENDER, ops }).success).toBe(true);
expect(assistantPageRenderSchema.safeParse({ ...RENDER, ops: [] }).success).toBe(false);
expect(assistantPageRenderSchema.safeParse({ ...RENDER, requestId: 'r1' }).success).toBe(false);
expect(assistantPageRenderSchema.safeParse({ ...RENDER, spreadId: '../x' }).success).toBe(false);
});
it('ASTVIS-002: a page view keeps the size of the image and the description, never the image', () => {
expect(assistantPageViewSchema.parse({ ...VIEW, data: 'aGVsbG8=' })).toEqual(VIEW);
const textOnly = { ...VIEW, width: null, height: null, bytes: null };
expect(assistantPageViewSchema.safeParse(textOnly).success).toBe(true);
});
it('ASTVIS-003: the page view is a stored part; the render request and the page view are stream chunks', () => {
expect(assistantMessagePartSchema.safeParse({ type: 'data-trek-page-view', data: VIEW }).success).toBe(true);
const request = { type: 'data-trek-render-request', data: RENDER, transient: true };
expect(uiStreamChunkSchema.safeParse(request).success).toBe(true);
expect(uiStreamChunkSchema.safeParse({ type: 'data-trek-page-view', data: VIEW }).success).toBe(true);
});
it('ASTVIS-004: an upload reads its text fields as multipart sends them, and its photo ids as a list', () => {
const fields = { bookVersion: '7', spreadId: 'sp-1', drawnPhotoIds: '12,30' };
expect(assistantRenderUploadRequestSchema.parse(fields)).toEqual({ ...fields, bookVersion: 7 });
expect(drawnPhotoIdsOf('12,30')).toEqual([12, 30]);
expect(drawnPhotoIdsOf('')).toEqual([]);
expect(drawnPhotoIdsOf(undefined)).toEqual([]);
});
it('ASTVIS-006: a preview names a page and may carry the ops of a proposed edit, nothing else', () => {
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1' }).success).toBe(true);
const ops = [{ op: 'background', color: '#ffffff' }];
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', ops }).success).toBe(true);
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', ops: [] }).success).toBe(false);
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', journeyId: 3 }).success).toBe(false);
});
it('ASTVIS-005: an upload refuses ids that are no list of numbers, and fields it does not know', () => {
const base = { bookVersion: '7', spreadId: 'sp-1' };
expect(assistantRenderUploadRequestSchema.safeParse({ ...base, drawnPhotoIds: '12,,3' }).success).toBe(false);
expect(assistantRenderUploadRequestSchema.safeParse({ ...base, drawnPhotoIds: '1;2' }).success).toBe(false);
expect(assistantRenderUploadRequestSchema.safeParse({ ...base, data: 'x' }).success).toBe(false);
expect(assistantRenderUploadRequestSchema.safeParse({ ...base, bookVersion: '-1' }).success).toBe(false);
});
it('ASTVIS-007: a page may name as many photos as its frames and map stops hold, up to the cap', () => {
const many = Array.from({ length: ASSISTANT_PAGE_IMAGE.maxPagePhotos }, (_, i) => i + 1);
expect(assistantPageRenderSchema.safeParse({ ...RENDER, hiddenPhotoIds: many }).success).toBe(true);
const tooMany = [...many, ASSISTANT_PAGE_IMAGE.maxPagePhotos + 1];
expect(assistantPageRenderSchema.safeParse({ ...RENDER, hiddenPhotoIds: tooMany }).success).toBe(false);
const fields = { bookVersion: '7', spreadId: 'sp-1', drawnPhotoIds: many.join(',') };
expect(assistantRenderUploadRequestSchema.safeParse(fields).success).toBe(true);
});
it('ASTVIS-008: a preview and a render request may name a design draft, by an id that is no path', () => {
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', designId: 'd-1a2b' }).success).toBe(true);
expect(previewBookPageRequestSchema.safeParse({ spreadId: 'sp-1', designId: '../d' }).success).toBe(false);
expect(assistantPageRenderSchema.parse({ ...RENDER, designId: 'd-1a2b' }).designId).toBe('d-1a2b');
expect(assistantPageRenderSchema.safeParse({ ...RENDER, designId: 'd/1' }).success).toBe(false);
});
});
@@ -0,0 +1,110 @@
import { bookSpreadIdSchema } from '../book/book-op-ids';
import { bookDesignIdSchema, editBookPageArgs } from '../book/book-ops.schema';
import { idSchema } from '../common/primitives.schema';
import { z } from 'zod';
/**
* The Studio assistant looking at a page (journey-studio.md section 3). The
* server cannot draw a page, so it asks the Studio tab that holds the turn's
* stream for one: a `data-trek-render-request` chunk names the page, the book
* version and the photos to leave out, the tab draws that one page and
* uploads it as an image, and the turn hands it to a model that reads images,
* beside the page's description. Nothing of the image is kept: the message
* stores what was looked at (`data-trek-page-view`), never the bytes.
*/
export const ASSISTANT_PAGE_IMAGE = {
/** The long edge a page is drawn at, in pixels, and the JPEG quality the Studio sends it with. */
longEdge: 1280,
jpegQuality: 0.82,
/** What the server takes at most: the bytes, the long edge and the pixels of one image. */
maxBytes: 1024 * 1024,
maxEdge: 2048,
maxPixels: 4_000_000,
/** Photos one page may name as hidden or drawn: its frames and the pictures its maps mark stops with. */
maxPagePhotos: 1000,
/** How long a turn waits for a page image before it goes on with the description alone. */
timeoutMs: 15_000,
/** Images one model request carries at most, one per tool answer; earlier ones go as their text. */
perRequest: 3,
/** Previews of a proposed change one step and one turn may ask for. */
previewsPerStep: 3,
previewsPerTurn: 8,
} as const;
/**
* A page the server asks the Studio to draw, streamed while the turn waits.
* `ops` are a proposal to draw on an unsaved copy (a preview); without them
* the page is drawn as the book stands at `bookVersion`. The photos in
* `hiddenPhotoIds` are drawn as empty frames, and a map stop showing one as
* a plain dot: only the asking user's own photos leave the browser.
*/
export const assistantPageRenderSchema = z.object({
requestId: z.uuid(),
threadId: idSchema,
spreadId: bookSpreadIdSchema,
bookVersion: z.number().int().nonnegative(),
ops: editBookPageArgs.ops.optional(),
/** A page of a design draft (design_book) rather than of the book: the Studio draws it from the draft. */
designId: bookDesignIdSchema.optional(),
hiddenPhotoIds: z.array(idSchema).max(ASSISTANT_PAGE_IMAGE.maxPagePhotos),
});
export type AssistantPageRender = z.infer<typeof assistantPageRenderSchema>;
/**
* What a turn looked at, as the message keeps it: the page, the version, the
* image's size when one reached the model (null when the model read the
* description alone), and the description itself. Never the image.
*/
export const assistantPageViewSchema = z.object({
spreadId: bookSpreadIdSchema,
bookVersion: z.number().int().nonnegative(),
width: z.number().int().positive().nullable(),
height: z.number().int().positive().nullable(),
bytes: z.number().int().positive().nullable(),
description: z.string().max(20_000),
});
export type AssistantPageView = z.infer<typeof assistantPageViewSchema>;
/**
* `preview_book_page`, a read of the Studio turn's own (no MCP tool): one page
* as it stands, or with `ops` as an `edit_book_page` with them would leave it,
* nothing saved. It answers the page's description and findings, and a model
* that reads images also gets the page as the Studio draws it.
*/
export const previewBookPageRequestSchema = z.strictObject({
spreadId: bookSpreadIdSchema,
ops: editBookPageArgs.ops.optional(),
/** Look at the page of a design draft instead of the book's (design_book's design_id). */
designId: bookDesignIdSchema.optional(),
});
export type PreviewBookPageRequest = z.infer<typeof previewBookPageRequestSchema>;
/**
* POST /api/journeys/:journeyId/assistant/threads/:threadId/renders/:requestId,
* multipart: the image in the field `image`, these as text fields beside it.
* `drawnPhotoIds` lists the photos the image shows, comma-separated; the
* server refuses the image when one of them is not the user's to show.
*/
export const assistantRenderUploadRequestSchema = z
.object({
bookVersion: z.coerce.number().int().nonnegative(),
spreadId: bookSpreadIdSchema,
drawnPhotoIds: z
.string()
.max(ASSISTANT_PAGE_IMAGE.maxPagePhotos * 13)
.regex(/^(?:\d{1,12}(?:,\d{1,12})*)?$/)
.optional(),
})
.strict();
export type AssistantRenderUploadRequest = z.infer<typeof assistantRenderUploadRequestSchema>;
/** The answer to an image the turn took; anything it did not take is refused with an error instead. */
export const assistantRenderUploadResponseSchema = z.object({ accepted: z.literal(true) });
export type AssistantRenderUploadResponse = z.infer<typeof assistantRenderUploadResponseSchema>;
/** The photo ids of an upload's `drawnPhotoIds` field. */
export function drawnPhotoIdsOf(field: string | undefined): number[] {
return field ? field.split(',').map(Number) : [];
}