mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
fix(booking-import): say why a recognised booking could not be imported
A document the model read correctly could still come back as "No reservations could be extracted". Every type mapper can answer null when the node carries no reservationFor, or one with no name, and both call sites pushed the item only if there was one — with no else. The unknown-@type branch next to them does warn, so a node TREK did not recognise was reported while a node it recognised but could not map vanished: no item, no warning, nothing in the log. Because the file did produce a node, the "no reservations found" fallback was skipped too, and the preview arrived empty and silent. Every recognised type could disappear this way, on both the kitinerary and the AI path. Each of them now warns, naming the file, the index, the type and the reason. Three smaller silences went with it. The debug line labelled "Raw LLM Response" printed the already-parsed array, so its inspect formatting — single quotes, bare keys — read as a provider emitting broken JSON and sent this report at the JSON parser, which has handled that shape since 4.0.0. A single reservation answered on its own, with no array and no wrapper, was dropped by the list unwrapper although the schema downstream would have taken it. And a response nobody could read returned an empty list, which is the same answer as a voucher with no booking in it; it now raises, on the Anthropic client too, where a reply cut off at the token limit produced exactly this report's symptom. The plugin AI surface keeps its old empty answer, since a value is what a plugin contracts for. The retry ladder gained a rung that sends no response_format at all, for providers that support neither json_schema nor json_object. Closes #2375
This commit is contained in:
@@ -375,6 +375,21 @@ function applyCommonMeta(item: ParsedBookingItem, r: KiReservation): ParsedBooki
|
||||
return item;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why a mapper answered null, in the words of the node it refused.
|
||||
*
|
||||
* Every type mapper drops out on one of two things: no `reservationFor` at all,
|
||||
* or one that carries no name. Said out loud, because a recognised booking that
|
||||
* cannot be mapped used to disappear between the mapper and the preview, and an
|
||||
* import that found one weak node looked exactly like a document with nothing in
|
||||
* it (#2375). Shaped like the unknown-type warning below so the two read as one
|
||||
* list.
|
||||
*/
|
||||
function unmappableWarning(r: KiReservation, fileName: string, index: number): string {
|
||||
const reason = r.reservationFor ? 'no name in reservationFor' : 'no reservationFor';
|
||||
return `Incomplete ${r['@type']} in ${fileName}[${index}] (${reason}) — skipped`;
|
||||
}
|
||||
|
||||
export function mapReservations(kiItems: KiReservation[], fileName: string): { items: ParsedBookingItem[]; warnings: string[] } {
|
||||
const items: ParsedBookingItem[] = [];
|
||||
const warnings: string[] = [];
|
||||
@@ -382,7 +397,7 @@ export function mapReservations(kiItems: KiReservation[], fileName: string): { i
|
||||
for (let i = 0; i < kiItems.length; i++) {
|
||||
const r = kiItems[i];
|
||||
const source = { fileName, index: i };
|
||||
let item: ParsedBookingItem | null = null;
|
||||
let item: ParsedBookingItem | null;
|
||||
|
||||
// Group consecutive connecting flight legs that share a PNR into one booking.
|
||||
if (r['@type'] === 'FlightReservation') {
|
||||
@@ -398,7 +413,10 @@ export function mapReservations(kiItems: KiReservation[], fileName: string): { i
|
||||
group.push(kiItems[++i]);
|
||||
}
|
||||
item = group.length > 1 ? mapFlightGroup(group, source) : mapFlight(r, source);
|
||||
// `i` sits on the last leg the group swallowed, so the index comes from
|
||||
// `source` — the node the warning is actually describing.
|
||||
if (item) items.push(applyCommonMeta(item, r));
|
||||
else warnings.push(unmappableWarning(r, fileName, source.index));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -413,9 +431,11 @@ export function mapReservations(kiItems: KiReservation[], fileName: string): { i
|
||||
case 'TouristAttractionVisit': item = mapEvent(r, source); break;
|
||||
default:
|
||||
warnings.push(`Unknown type "${r['@type']}" in ${fileName}[${i}] — skipped`);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item) items.push(applyCommonMeta(item, r));
|
||||
else warnings.push(unmappableWarning(r, fileName, i));
|
||||
}
|
||||
|
||||
return { items, warnings };
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.in
|
||||
import { safeFetchLlm } from '../../../utils/ssrfGuard';
|
||||
import { readEnv } from '../../../app-config';
|
||||
import { toReservationList } from '../lenient-json';
|
||||
import { UnreadableLlmResponse } from './openai-compatible.client';
|
||||
|
||||
const MAX_TOKENS = 8192;
|
||||
const ANTHROPIC_VERSION = '2023-06-01';
|
||||
@@ -83,9 +84,20 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
if (data.stop_reason === 'refusal') {
|
||||
throw new Error('Anthropic declined to process this document');
|
||||
}
|
||||
// A run that hits the cap stops mid-tool-call, so what arrives is a fragment
|
||||
// of the list or nothing at all — and a forced tool that was not called left
|
||||
// no list either. Neither is "this document holds no booking", but both came
|
||||
// back as [] and reached the person as an empty preview with nothing in the
|
||||
// log: the #2375 symptom, on the provider the dropdown offers first.
|
||||
if (data.stop_reason === 'max_tokens') {
|
||||
throw new UnreadableLlmResponse(
|
||||
`the answer was cut off at the ${MAX_TOKENS}-token limit — the document is too long to extract in one pass`,
|
||||
);
|
||||
}
|
||||
|
||||
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === TOOL_NAME);
|
||||
return toReservationList(toolUse?.input?.reservations);
|
||||
if (!toolUse) throw new UnreadableLlmResponse(`the model answered without calling ${TOOL_NAME}`);
|
||||
return toReservationList(toolUse.input?.reservations);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,10 +6,24 @@ import { readEnv } from '../../../app-config';
|
||||
|
||||
const MAX_TOKENS = 4096;
|
||||
|
||||
/**
|
||||
* A response the client could not read at all: prose where JSON was asked for,
|
||||
* an empty body, an answer that stopped mid-sentence.
|
||||
*
|
||||
* Its own type because the two callers want opposite things from it. The booking
|
||||
* import lets it through, so the per-file catch in llm-parse.service.ts names the
|
||||
* file and the person gets a warning instead of an empty preview (#2375); the
|
||||
* plugin AI surface, versioned separately and answering an empty value for a
|
||||
* model that talks prose since it shipped, swallows exactly this one and nothing
|
||||
* else. The Anthropic client raises it too.
|
||||
*/
|
||||
export class UnreadableLlmResponse extends Error {}
|
||||
|
||||
/** What one attempt differs in. Each field is switched on by a 400 that asked for it. */
|
||||
interface RequestShape {
|
||||
tokenParam: 'max_tokens' | 'max_completion_tokens';
|
||||
jsonObject: boolean;
|
||||
noResponseFormat: boolean;
|
||||
omitTemperature: boolean;
|
||||
}
|
||||
|
||||
@@ -44,15 +58,18 @@ function rejectsTemperature(detail: string): boolean {
|
||||
*
|
||||
* Structured output is requested as `json_schema` first; servers that only
|
||||
* support `json_object` (DeepSeek, Mistral, some vLLM/llama.cpp) reject that
|
||||
* with a 400, so the request is retried once in `json_object` mode. Two further
|
||||
* 400s are answered the same way: `max_tokens` becomes `max_completion_tokens`
|
||||
* (#1760), and "temperature is not supported" drops the parameter (#2262).
|
||||
* with a 400, so the request is retried once in `json_object` mode, and a server
|
||||
* that refuses that too gets one last attempt carrying no `response_format` at
|
||||
* all — the shape a proxy's `drop_params` produces, and the only one a provider
|
||||
* without either grammar can answer (#2375). Two further 400s are answered the
|
||||
* same way: `max_tokens` becomes `max_completion_tokens` (#1760), and
|
||||
* "temperature is not supported" drops the parameter (#2262).
|
||||
*
|
||||
* Those retries are a loop over what the server actually said, not a fixed
|
||||
* chain. A reasoning model rejects `max_tokens` AND `temperature`, the API names
|
||||
* only one parameter per response, and it may name either first — a chain of
|
||||
* one-shot ifs survives only one of the two orders. Each remedy applies at most
|
||||
* once, so this adds at most three extra requests.
|
||||
* once, so this adds at most four extra requests.
|
||||
*/
|
||||
export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
async extract(input: LlmExtractionInput): Promise<Record<string, unknown>[]> {
|
||||
@@ -99,7 +116,10 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
{ role: 'user', content: userContent },
|
||||
],
|
||||
};
|
||||
if (nuextract) return baseBody;
|
||||
// NuExtract never carries one, and the last rung of the 400 ladder has
|
||||
// given it up: the system prompt still dictates the shape, and JSON5 reads
|
||||
// back whatever the model made of it.
|
||||
if (nuextract || shape.noResponseFormat) return baseBody;
|
||||
return {
|
||||
...baseBody,
|
||||
response_format: shape.jsonObject
|
||||
@@ -108,8 +128,8 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
};
|
||||
};
|
||||
|
||||
const shape: RequestShape = { tokenParam: 'max_tokens', jsonObject: false, omitTemperature: false };
|
||||
const tried = { tokenParam: false, temperature: false, jsonObject: false };
|
||||
const shape: RequestShape = { tokenParam: 'max_tokens', jsonObject: false, noResponseFormat: false, omitTemperature: false };
|
||||
const tried = { tokenParam: false, temperature: false, jsonObject: false, noResponseFormat: false };
|
||||
|
||||
let res = await this.send(url, buildBody(shape), input.apiKey);
|
||||
let detail = res.ok ? '' : await res.text().catch(() => '');
|
||||
@@ -134,9 +154,14 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
if (!named) {
|
||||
// NuExtract sends no response_format at all, so it has nothing to fall
|
||||
// back to and the 400 is final.
|
||||
if (tried.jsonObject || nuextract) break;
|
||||
shape.jsonObject = true;
|
||||
tried.jsonObject = true;
|
||||
if (nuextract || tried.noResponseFormat) break;
|
||||
if (tried.jsonObject) {
|
||||
shape.noResponseFormat = true;
|
||||
tried.noResponseFormat = true;
|
||||
} else {
|
||||
shape.jsonObject = true;
|
||||
tried.jsonObject = true;
|
||||
}
|
||||
}
|
||||
res = await this.send(url, buildBody(shape), input.apiKey);
|
||||
detail = res.ok ? '' : await res.text().catch(() => '');
|
||||
@@ -176,12 +201,38 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
|
||||
/** Parse a NuExtract response and map its flat template output to KiReservation nodes. */
|
||||
function parseNuExtract(content: string | undefined | null): Record<string, unknown>[] {
|
||||
return nuExtractToKiReservations(parseLenientJson(content));
|
||||
return nuExtractToKiReservations(readAnswer(content));
|
||||
}
|
||||
|
||||
const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
|
||||
|
||||
/** Tolerant parse: strip code fences, JSON(5).parse, pull `reservations`. `[]` on failure. */
|
||||
/** Tolerant parse: strip code fences, JSON(5).parse, pull `reservations`. */
|
||||
function parseReservations(content: string | undefined | null): Record<string, unknown>[] {
|
||||
return toReservationList(parseLenientJson(content));
|
||||
return toReservationList(readAnswer(content));
|
||||
}
|
||||
|
||||
/**
|
||||
* What the model answered, or an `UnreadableLlmResponse` when nothing could read it.
|
||||
*
|
||||
* An empty list is an answer — the document held no booking, and the import says
|
||||
* so. A response nobody could read is not that answer, and returning `[]` for it
|
||||
* too made a provider that replied with prose, or with nothing at all, look
|
||||
* exactly like an empty voucher: no item, no warning, nothing in the log (#2375).
|
||||
* So only the unreadable throws; whatever parsed is handed on, and an answer that
|
||||
* holds no reservation keeps the old "no reservations found" path.
|
||||
*/
|
||||
function readAnswer(content: string | undefined | null): unknown {
|
||||
if (!content?.trim()) throw new UnreadableLlmResponse('the model returned an empty response');
|
||||
const parsed = parseLenientJson(content);
|
||||
if (isReadable(parsed)) return parsed;
|
||||
// The response is the booking, so it stays out of a managed operator's log —
|
||||
// same split as the extracted text in llm-parse.service.ts.
|
||||
const snippet = readEnv().managed.enabled ? '' : `: ${content.trim().slice(0, 200)}`;
|
||||
throw new UnreadableLlmResponse(`the model did not answer with JSON${snippet}`);
|
||||
}
|
||||
|
||||
/** A value the parser actually read back, as opposed to prose or a bare scalar. */
|
||||
function isReadable(value: unknown): boolean {
|
||||
if (typeof value === 'string') return isReadable(parseLenientJson(value));
|
||||
return Array.isArray(value) || (!!value && typeof value === 'object');
|
||||
}
|
||||
|
||||
@@ -50,12 +50,24 @@ export function parseLenientJson(content: string | undefined | null): unknown {
|
||||
* One unwrap of a string, not a loop: a value that is still not a list after
|
||||
* that is genuinely not one, and guessing further would start inventing
|
||||
* bookings out of prose.
|
||||
*
|
||||
* A single reservation answered on its own, with no array and no wrapper around
|
||||
* it, is the third shape seen in the wild:
|
||||
*
|
||||
* {"@type":"LodgingReservation","reservationNumber":"733", …}
|
||||
*
|
||||
* A string `@type` is the same gate `kiReservationSchema` applies, so a node that
|
||||
* would have been accepted inside an array is accepted alone rather than thrown
|
||||
* away without a word (#2375). It is still not guessing: an object without that
|
||||
* key stays refused.
|
||||
*/
|
||||
export function toReservationList(value: unknown): Record<string, unknown>[] {
|
||||
const list = (v: unknown): Record<string, unknown>[] | null => {
|
||||
if (Array.isArray(v)) return v as Record<string, unknown>[];
|
||||
if (v && typeof v === 'object' && Array.isArray((v as { reservations?: unknown }).reservations)) {
|
||||
return (v as { reservations: Record<string, unknown>[] }).reservations;
|
||||
if (v && typeof v === 'object') {
|
||||
const o = v as { reservations?: unknown; '@type'?: unknown };
|
||||
if (Array.isArray(o.reservations)) return o.reservations as Record<string, unknown>[];
|
||||
if (typeof o['@type'] === 'string') return [v as Record<string, unknown>];
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
@@ -116,8 +116,11 @@ export class LlmParseService {
|
||||
try {
|
||||
raw = await createLlmClient(config).extract(input);
|
||||
// Same reason: the model answers with the fields it read out of the document.
|
||||
// Parsed, not raw — this prints the nodes the client already read out of the
|
||||
// response, so its single quotes and bare keys are util.inspect's, not the
|
||||
// provider's, and a bug report that quotes them is quoting TREK (#2375).
|
||||
if (this.env.isManaged()) console.debug(`[DEBUG] LLM response: ${raw.length} item(s)`);
|
||||
else console.debug('[DEBUG] Raw LLM Response: ', raw);
|
||||
else console.debug(`[DEBUG] Parsed LLM response (${raw.length} item(s)): `, raw);
|
||||
} catch (err) {
|
||||
console.error(`[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`, err instanceof Error ? err.message : err);
|
||||
return {
|
||||
|
||||
@@ -9,6 +9,9 @@ import { RealtimeService } from '../../../realtime/realtime.service';
|
||||
import { NotificationsService } from '../../../notifications/notifications.service';
|
||||
import { LlmConfigResolver } from '../../../llm-parse/llm-config.resolver';
|
||||
import { createLlmClient } from '../../../llm-parse/llm-client.factory';
|
||||
import { UnreadableLlmResponse } from '../../../llm-parse/clients/openai-compatible.client';
|
||||
import type { ResolvedLlmConfig } from '../../../llm-parse/llm-config';
|
||||
import type { LlmExtractionInput } from '../../../llm-parse/llm-provider.interface';
|
||||
import { PluginOAuthService } from '../../oauth/plugin-oauth.service';
|
||||
import { stripEmoji } from '../../text-sanitize';
|
||||
|
||||
@@ -125,7 +128,7 @@ export class HostSurfaceRpc {
|
||||
if (prompt.length > AI_TEXT_MAX) throw new BadParams(`prompt exceeds the ${AI_TEXT_MAX}-char cap`);
|
||||
this.takeAiBudget(ctx);
|
||||
const system = typeof params.system === 'string' ? params.system.slice(0, 4000) : undefined;
|
||||
const results = await createLlmClient(config).extract({
|
||||
const results = await this.runModel(config, {
|
||||
prompt: system || 'You are a helpful assistant. Reply with a JSON object of the form {"text": "..."} whose "text" field holds your answer.',
|
||||
jsonSchema: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] },
|
||||
model: config.model,
|
||||
@@ -149,7 +152,7 @@ export class HostSurfaceRpc {
|
||||
}
|
||||
this.takeAiBudget(ctx);
|
||||
const hint = typeof params.prompt === 'string' ? params.prompt.slice(0, 4000) : '';
|
||||
const results = await createLlmClient(config).extract({
|
||||
const results = await this.runModel(config, {
|
||||
prompt: hint || 'Extract structured data from the text into the given JSON schema.',
|
||||
jsonSchema: params.jsonSchema as object,
|
||||
model: config.model,
|
||||
@@ -224,6 +227,24 @@ export class HostSurfaceRpc {
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* The model's answer, without the diagnosis the booking import asked for.
|
||||
*
|
||||
* A client raises `UnreadableLlmResponse` when the model replies with prose or
|
||||
* with nothing at all, so an import can warn about that file (#2375). Plugins
|
||||
* were written against the older contract here — an empty answer, never an
|
||||
* error — and this surface is versioned on its own, so that one stays empty. A
|
||||
* request that actually failed still reaches the plugin as HOST_ERROR.
|
||||
*/
|
||||
private async runModel(config: ResolvedLlmConfig, input: LlmExtractionInput): Promise<Record<string, unknown>[]> {
|
||||
try {
|
||||
return await createLlmClient(config).extract(input);
|
||||
} catch (err) {
|
||||
if (err instanceof UnreadableLlmResponse) return [];
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private takeAiBudget(ctx: PluginRpcContext): void {
|
||||
if (!budgetFor(ctx.pluginId, this.db.connection).take('ai', Date.now())) {
|
||||
throw new BadParams('daily AI budget exhausted (resets at UTC midnight)');
|
||||
|
||||
@@ -65,9 +65,56 @@ describe('OpenAiCompatibleClient', () => {
|
||||
expect(out).toEqual([{ '@type': 'LodgingReservation', checkinTime: '2026-08-28T00:00:00', price: 146.25 }]);
|
||||
});
|
||||
|
||||
it('returns [] on malformed content', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: 'not json' } }] }));
|
||||
expect(await new OpenAiCompatibleClient().extract(baseInput)).toEqual([]);
|
||||
/*
|
||||
* Prose and an honest empty list both came back as `[]`, so a provider that
|
||||
* answered neither JSON nor a reservation looked exactly like a document
|
||||
* holding no booking: no item, no warning, nothing in the log (#2375). Only
|
||||
* what nothing could read throws — a parsed answer that simply holds no
|
||||
* reservation keeps the old empty path.
|
||||
*/
|
||||
it('throws on content the lenient parser cannot read (#2375)', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: 'I could not find a booking in this document.' } }] }));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/did not answer with JSON/);
|
||||
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: '42' } }] }));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/did not answer with JSON/);
|
||||
});
|
||||
|
||||
it('throws when the response carries no content at all (#2375)', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: '' } }] }));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/empty response/);
|
||||
|
||||
// A content filter or a zero-token completion leaves the field off entirely.
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: {} }] }));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/empty response/);
|
||||
});
|
||||
|
||||
it('still answers empty for a document that genuinely holds no booking', async () => {
|
||||
for (const content of ['[]', '{"reservations":[]}', '{}', '{"reservations":null}', JSON.stringify('{"reservations":[]}')]) {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content } }] }));
|
||||
expect(await new OpenAiCompatibleClient().extract(baseInput)).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the unreadable response out of the message on a managed instance', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: 'Booking for Jane Doe, 12 Rue de Rivoli' } }] }));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/Jane Doe/);
|
||||
|
||||
// The response is the booking, and on a managed install the operator's log
|
||||
// is not the place for it — same split as the extracted text.
|
||||
vi.stubEnv('TREK_MANAGED', 'true');
|
||||
try {
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/did not answer with JSON$/);
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it('reads a single reservation the model answered on its own', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"@type":"LodgingReservation","reservationNumber":"733"}' } }] }));
|
||||
expect(await new OpenAiCompatibleClient().extract(baseInput)).toEqual([
|
||||
{ '@type': 'LodgingReservation', reservationNumber: '733' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('throws on non-2xx', async () => {
|
||||
@@ -187,12 +234,37 @@ describe('OpenAiCompatibleClient', () => {
|
||||
expect(second.response_format.type).toBe('json_object');
|
||||
});
|
||||
|
||||
it('throws when the json_object retry also fails (400 twice)', async () => {
|
||||
/*
|
||||
* A provider behind a proxy that supports neither grammar rejected both rungs
|
||||
* and the import failed hard, which is what `drop_params` on the proxy side was
|
||||
* being used to work around (#2375). The last attempt sends the request the
|
||||
* proxy would have produced.
|
||||
*/
|
||||
it('drops response_format entirely once json_schema and json_object are both refused (#2375)', async () => {
|
||||
safeFetchLlmMock
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'no json_schema' }, false, 400))
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'no json_object either' }, false, 400));
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'no json_object either' }, false, 400))
|
||||
.mockResolvedValueOnce(jsonResponse({ choices: [{ message: { content: '{"reservations":[{"@type":"FlightReservation"}]}' } }] }));
|
||||
const out = await new OpenAiCompatibleClient().extract(baseInput);
|
||||
expect(out).toEqual([{ '@type': 'FlightReservation' }]);
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(3);
|
||||
|
||||
const bodies = safeFetchLlmMock.mock.calls.map(c => JSON.parse((c[1] as RequestInit).body as string));
|
||||
expect(bodies[0].response_format.type).toBe('json_schema');
|
||||
expect(bodies[1].response_format).toEqual({ type: 'json_object' });
|
||||
expect('response_format' in bodies[2]).toBe(false);
|
||||
// Only the grammar goes; the prompt, the model and the token cap are untouched.
|
||||
expect(bodies[2].messages).toEqual(bodies[0].messages);
|
||||
expect(bodies[2].max_tokens).toBe(4096);
|
||||
});
|
||||
|
||||
it('throws when the attempt without response_format also fails (400 three times)', async () => {
|
||||
safeFetchLlmMock
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'no json_schema' }, false, 400))
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'no json_object either' }, false, 400))
|
||||
.mockResolvedValueOnce(jsonResponse({ error: 'nothing this model accepts' }, false, 400));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/400/);
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(2);
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('does not retry the NuExtract path on 400', async () => {
|
||||
@@ -255,6 +327,17 @@ describe('OpenAiCompatibleClient — NuExtract path', () => {
|
||||
expect(body.response_format).toBeUndefined();
|
||||
});
|
||||
|
||||
it('throws when NuExtract answered something nothing could read (#2375)', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: 'I am NuExtract, a template filling model.' } }] }));
|
||||
await expect(
|
||||
new OpenAiCompatibleClient().extract({ ...baseInput, model: 'nuextract', text: 'Hotel doc' }),
|
||||
).rejects.toThrow(/did not answer with JSON/);
|
||||
|
||||
// A template that came back filled with nothing is still an answer.
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"type":null,"name":null}' } }] }));
|
||||
expect(await new OpenAiCompatibleClient().extract({ ...baseInput, model: 'nuextract', text: 'Hotel doc' })).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the system prompt and response_format for non-NuExtract models', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
|
||||
await new OpenAiCompatibleClient().extract({ ...baseInput, model: 'qwen2.5:7b' });
|
||||
@@ -306,6 +389,38 @@ describe('AnthropicClient', () => {
|
||||
expect(await new AnthropicClient().extract(baseInput)).toEqual([]);
|
||||
});
|
||||
|
||||
/*
|
||||
* A forced tool that was never called, and a run that stopped at the cap with a
|
||||
* fragment of one, both came back as [] — so a long voucher reached the person
|
||||
* as "no reservations found", with an empty warnings array and nothing in the
|
||||
* log. That is the #2375 symptom on the provider the dropdown offers first, so
|
||||
* the same rule applies here: only what the client could read is an answer.
|
||||
*/
|
||||
it('throws when the answer was cut off at the token cap (#2375)', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'max_tokens', content: [{ type: 'tool_use', name: 'emit_reservations', input: {} }] }),
|
||||
);
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/cut off at the 8192-token limit/);
|
||||
});
|
||||
|
||||
it('throws when the forced tool was never called (#2375)', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'end_turn', content: [{ type: 'text', text: 'I could not find a booking in this document.' }] }),
|
||||
);
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/without calling emit_reservations/);
|
||||
|
||||
// A response carrying no content block at all lands on the same message.
|
||||
mockFetch(() => jsonResponse({ stop_reason: 'end_turn' }));
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/without calling emit_reservations/);
|
||||
});
|
||||
|
||||
it('still answers empty for a tool call that carries an empty list', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }),
|
||||
);
|
||||
expect(await new AnthropicClient().extract(baseInput)).toEqual([]);
|
||||
});
|
||||
|
||||
it('throws on a refusal stop_reason', async () => {
|
||||
mockFetch(() => jsonResponse({ stop_reason: 'refusal', content: [] }));
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/declined/i);
|
||||
|
||||
@@ -79,6 +79,17 @@ describe('toReservationList', () => {
|
||||
expect(toReservationList(JSON.stringify({ reservations: [node] }))).toEqual([node]);
|
||||
});
|
||||
|
||||
/*
|
||||
* A single booking answered on its own, with no array and no wrapper. The
|
||||
* check only knew the two list shapes, so a perfectly good extraction was
|
||||
* dropped without a word — the same silent empty preview as #1968, from a
|
||||
* different direction (#2375).
|
||||
*/
|
||||
it('takes a single reservation answered on its own', () => {
|
||||
expect(toReservationList(node)).toEqual([node]);
|
||||
expect(toReservationList(JSON.stringify(node))).toEqual([node]);
|
||||
});
|
||||
|
||||
it('takes the relaxed JSON a model might emit inside that string', () => {
|
||||
expect(toReservationList("[{'@type': 'FlightReservation', flightNumber: 'LH400',}]"))
|
||||
.toEqual([{ '@type': 'FlightReservation', flightNumber: 'LH400' }]);
|
||||
|
||||
@@ -61,6 +61,7 @@ import { DbRpc } from '../../../src/nest/plugins/host/rpc/db.rpc';
|
||||
import type { PluginUserSettingsService } from '../../../src/nest/plugins/plugin-user-settings.service';
|
||||
import { MetaRpc } from '../../../src/nest/plugins/host/rpc/meta.rpc';
|
||||
import { HostSurfaceRpc } from '../../../src/nest/plugins/host/rpc/host-surface.rpc';
|
||||
import { UnreadableLlmResponse } from '../../../src/nest/llm-parse/clients/openai-compatible.client';
|
||||
import { getPluginDataDb, closePluginDataDb } from '../../../src/nest/plugins/host/plugin-host-state';
|
||||
import { db as mockDb } from '../../../src/db/database';
|
||||
import { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
@@ -472,6 +473,27 @@ describe('HostSurfaceRpc — users, broadcasts, notify, ai, oauth, scheduler', (
|
||||
expect((await call(makeHost('surface', 'ai:invoke'), 'ai.complete', { prompt: 'p' }, 5)).result).toEqual({ text: '' });
|
||||
});
|
||||
|
||||
/*
|
||||
* The clients raise on a response nothing could read so the booking import can
|
||||
* warn per file (#2375). This surface answered an empty value for a model that
|
||||
* talks prose long before that, and it is versioned on its own, so the plugin
|
||||
* keeps the empty value — while a request that genuinely failed still has to
|
||||
* reach it as an error.
|
||||
*/
|
||||
it('HOSTRPC-026a an unreadable answer stays empty here, a failed request does not (#2375)', async () => {
|
||||
const host = makeHost('surface', 'ai:invoke');
|
||||
llmExtract.mockRejectedValueOnce(new UnreadableLlmResponse('the model did not answer with JSON: Sure, happy to help.'));
|
||||
expect((await call(host, 'ai.complete', { prompt: 'p' }, 5)).result).toEqual({ text: '' });
|
||||
|
||||
llmExtract.mockRejectedValueOnce(new UnreadableLlmResponse('the model returned an empty response'));
|
||||
expect((await call(host, 'ai.extract', { text: 't', jsonSchema: { type: 'object' } }, 5)).result).toEqual({ results: [] });
|
||||
|
||||
llmExtract.mockRejectedValueOnce(new Error('LLM request failed (503): upstream is down'));
|
||||
const failed = await call(host, 'ai.complete', { prompt: 'p' }, 5);
|
||||
expect(failed.error?.code).toBe('HOST_ERROR');
|
||||
expect(failed.error?.message).toMatch(/503/);
|
||||
});
|
||||
|
||||
it('HOSTRPC-027 oauth.getToken returns only the acting user\'s access token', async () => {
|
||||
const host = makeHost('surface', 'oauth:client');
|
||||
expect((await call(host, 'oauth.getToken', {}, 5)).result).toEqual({ accessToken: 'tok-5' });
|
||||
|
||||
@@ -107,3 +107,62 @@ describe('kitinerary mapper — printed 12-hour clocks (#2094)', () => {
|
||||
expect(items).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A recognised @type whose mapper cannot build an item used to leave nothing
|
||||
* behind: no item, no warning, no log. With a single node in the file the
|
||||
* service's own "no reservations found" never fired either, so a hotel voucher
|
||||
* the model had read correctly but namelessly came back as an empty preview,
|
||||
* indistinguishable from a document holding no booking at all (#2375).
|
||||
*/
|
||||
describe('kitinerary mapper — a recognised type that cannot be mapped (#2375)', () => {
|
||||
it('warns instead of dropping a lodging whose reservationFor carries no name', () => {
|
||||
const { items, warnings } = mapReservations([
|
||||
{ '@type': 'LodgingReservation', reservationNumber: 'HMTRSX', reservationFor: {} },
|
||||
] as any, 'airbnb.pdf');
|
||||
|
||||
expect(items).toHaveLength(0);
|
||||
expect(warnings).toEqual([
|
||||
'Incomplete LodgingReservation in airbnb.pdf[0] (no name in reservationFor) — skipped',
|
||||
]);
|
||||
});
|
||||
|
||||
it('warns for a flight that carries no reservationFor at all', () => {
|
||||
const { items, warnings } = mapReservations([
|
||||
{ '@type': 'FlightReservation', reservationNumber: 'ABC123' },
|
||||
] as any, 'ticket.eml');
|
||||
|
||||
expect(items).toHaveLength(0);
|
||||
expect(warnings).toEqual([
|
||||
'Incomplete FlightReservation in ticket.eml[0] (no reservationFor) — skipped',
|
||||
]);
|
||||
});
|
||||
|
||||
it('names the type it could not map, TouristAttractionVisit included', () => {
|
||||
const { warnings } = mapReservations([
|
||||
{ '@type': 'TouristAttractionVisit', reservationFor: {} },
|
||||
] as any, 'museum.pdf');
|
||||
|
||||
expect(warnings[0]).toBe('Incomplete TouristAttractionVisit in museum.pdf[0] (no name in reservationFor) — skipped');
|
||||
});
|
||||
|
||||
it('keeps the bookings it could map and warns only about the weak one', () => {
|
||||
const { items, warnings } = mapReservations([
|
||||
flight('MIX1', FRA, BER, '2026-06-11T10:00:00', '2026-06-11T12:00:00', 'LH 800'),
|
||||
{ '@type': 'LodgingReservation', reservationFor: {} },
|
||||
] as any, 'trip.eml');
|
||||
|
||||
expect(items).toHaveLength(1);
|
||||
expect(warnings).toHaveLength(1);
|
||||
expect(warnings[0]).toContain('trip.eml[1]');
|
||||
});
|
||||
|
||||
it('still warns exactly once for an unknown type', () => {
|
||||
const { items, warnings } = mapReservations([
|
||||
{ '@type': 'RocketLaunchReservation', reservationFor: { name: 'Starbase' } },
|
||||
] as any, 'mars.eml');
|
||||
|
||||
expect(items).toHaveLength(0);
|
||||
expect(warnings).toEqual(['Unknown type "RocketLaunchReservation" in mars.eml[0] — skipped']);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user