mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
Cloud models got the first 4,000 characters, so totals and lodging sections further down never reached them, without a word. They now get 24,000 with the start and the end kept and a warning when the middle is left out; a server whose context is too small is asked once more with 8,000. A total the model repeated on every flight of one booking stays on the first flight only, when the document prints it once as a total, and the prompt says so too.
This commit is contained in:
Maurice
@@ -0,0 +1,80 @@
|
||||
import { parseAmount } from './clients/nuextract';
|
||||
import { toMinor } from '@trek/shared';
|
||||
|
||||
type Node = Record<string, unknown>;
|
||||
|
||||
/** Lines that state a sum rather than a fare per leg or a seat surcharge. */
|
||||
const TOTAL_LINE = /total|gesamt|summe|betrag|amount|paid|bezahlt/i;
|
||||
/** One number as printed: digits with their grouping and decimal marks, never two numbers glued together. */
|
||||
const NUMBER = /\d[\d.,]*/g;
|
||||
|
||||
/**
|
||||
* How many lines of the document print this amount (in hundredths), and whether
|
||||
* one of them is a total line. Printed once per leg it is a fare per leg, even
|
||||
* on lines labelled "Amount", not one total the model repeated.
|
||||
*/
|
||||
function amountLines(text: string, minor: number): { lines: number; onTotal: boolean } {
|
||||
let lines = 0;
|
||||
let onTotal = false;
|
||||
for (const line of text.split('\n')) {
|
||||
const printed = (line.match(NUMBER) ?? []).some((token) => {
|
||||
const amount = parseAmount(token);
|
||||
return amount != null && toMinor(amount) === minor;
|
||||
});
|
||||
if (!printed) continue;
|
||||
lines += 1;
|
||||
if (TOTAL_LINE.test(line)) onTotal = true;
|
||||
}
|
||||
return { lines, onTotal };
|
||||
}
|
||||
|
||||
const reference = (node: Node) =>
|
||||
String(node.reservationNumber ?? '')
|
||||
.trim()
|
||||
.toUpperCase();
|
||||
|
||||
const departure = (node: Node) => {
|
||||
const time = (node.reservationFor as { departureTime?: unknown } | undefined)?.departureTime;
|
||||
return typeof time === 'string' ? time : '';
|
||||
};
|
||||
|
||||
const priceOf = (node: Node): number | null => {
|
||||
const amount = parseAmount(node.price);
|
||||
return amount != null && amount > 0 ? toMinor(amount) : null;
|
||||
};
|
||||
|
||||
/**
|
||||
* A model reading a round trip with one total for the whole booking puts that
|
||||
* total on every flight, and saving both legs then books the purchase twice
|
||||
* (#2651). When the flights plainly share it, the price stays on the first
|
||||
* flight to depart and leaves the others.
|
||||
*
|
||||
* Plainly means: only flights, all on one booking reference, at least two of
|
||||
* them priced, every price the same amount in the same currency, and the
|
||||
* document printing that amount on a total line, on fewer lines than there are
|
||||
* priced flights. Fares that differ, separate references, mixed currencies,
|
||||
* other booking types, or no document text to check against (a PDF read
|
||||
* natively, a photo) leave the nodes as they are.
|
||||
*/
|
||||
export function collapseRepeatedBookingTotal(nodes: Node[], documentText: string | undefined): Node[] {
|
||||
if (!documentText || nodes.length < 2) return nodes;
|
||||
if (!nodes.every((node) => node['@type'] === 'FlightReservation')) return nodes;
|
||||
const ref = reference(nodes[0]);
|
||||
if (!ref || !nodes.every((node) => reference(node) === ref)) return nodes;
|
||||
|
||||
const priced = nodes.filter((node) => priceOf(node) != null);
|
||||
const minor = priced.length >= 2 ? priceOf(priced[0]) : null;
|
||||
if (minor == null) return nodes;
|
||||
const currency = priced[0].priceCurrency;
|
||||
if (!priced.every((node) => priceOf(node) === minor && node.priceCurrency === currency)) return nodes;
|
||||
const printed = amountLines(documentText, minor);
|
||||
if (!printed.onTotal || printed.lines >= priced.length) return nodes;
|
||||
|
||||
// The first to depart keeps it; a tie keeps the order the model gave.
|
||||
const keeper = priced.reduce((first, node) => (departure(node) < departure(first) ? node : first));
|
||||
return nodes.map((node) => {
|
||||
if (node === keeper || priceOf(node) == null) return node;
|
||||
const { price: _price, priceCurrency: _currency, ...rest } = node;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { readEnv } from '../../../app-config';
|
||||
import { safeFetchLlm } from '../../../utils/ssrfGuard';
|
||||
import { toReservationList } from '../lenient-json';
|
||||
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
|
||||
import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
|
||||
import { UnreadableLlmResponse } from './openai-compatible.client';
|
||||
|
||||
const MAX_TOKENS = 8192;
|
||||
@@ -79,7 +80,10 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => '');
|
||||
throw new Error(`Anthropic request failed (${res.status}): ${detail.slice(0, 300)}`);
|
||||
const message = `Anthropic request failed (${res.status}): ${detail.slice(0, 300)}`;
|
||||
// A gateway in front of a smaller model can run out of context (#2652, #2655).
|
||||
if (res.status === 400 && isContextOverflow(detail)) throw new LlmContextOverflowError(message);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Raised when a server refuses a request because the prompt does not fit its
|
||||
* context. Only a shorter text can help, so the request is not retried in
|
||||
* another shape: the caller cuts the document down and asks once more (#2652).
|
||||
*/
|
||||
export class LlmContextOverflowError extends Error {}
|
||||
|
||||
/**
|
||||
* The wordings OpenAI, vLLM, llama.cpp, LiteLLM and Anthropic use for a prompt
|
||||
* that is too long for the model's context.
|
||||
*/
|
||||
const CONTEXT_OVERFLOW =
|
||||
/maximum context length|context[_ ]length[_ ]exceeded|context window|prompt is too long|too many tokens|exceeds the context/i;
|
||||
|
||||
export function isContextOverflow(detail: string): boolean {
|
||||
return CONTEXT_OVERFLOW.test(detail);
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { readEnv } from '../../../app-config';
|
||||
import { safeFetchLlm } from '../../../utils/ssrfGuard';
|
||||
import { parseLenientJson, toReservationList } from '../lenient-json';
|
||||
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
|
||||
import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
|
||||
import { isNuExtractModel, buildNuExtractUserText, nuExtractToKiReservations } from './nuextract';
|
||||
|
||||
const MAX_TOKENS = 4096;
|
||||
@@ -153,6 +154,9 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
// system prompt already dictates the exact output shape and mentions JSON,
|
||||
// which json_object mode requires.
|
||||
while (!res.ok && res.status === 400) {
|
||||
// No response_format makes a prompt shorter, so a context overflow ends the ladder.
|
||||
if (isContextOverflow(detail))
|
||||
throw new LlmContextOverflowError(`LLM request failed (400): ${detail.slice(0, 300)}`);
|
||||
let named = false;
|
||||
if (!tried.tokenParam && detail.includes('max_completion_tokens')) {
|
||||
shape.tokenParam = 'max_completion_tokens';
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import type { KiReservation } from '../booking-import/kitinerary.types';
|
||||
import { collapseRepeatedBookingTotal } from './booking-total';
|
||||
import { LlmContextOverflowError } from './clients/context-overflow';
|
||||
import { toIsoCurrency } from './currency-code';
|
||||
import { capImage, imageMimeType, renderPdfPages } from './image-input';
|
||||
import { createLlmClient } from './llm-client.factory';
|
||||
@@ -24,6 +26,13 @@ import {
|
||||
} from './router/extraction-router';
|
||||
import { extractEnforced } from './router/ollama-format.client';
|
||||
import { isPdf, extractText } from './text-extract';
|
||||
import {
|
||||
CLOUD_TEXT_WINDOW,
|
||||
COMPACT_TEXT_WINDOW,
|
||||
fitTextWindow,
|
||||
LOCAL_FLIGHT_TEXT_WINDOW,
|
||||
LOCAL_TEXT_WINDOW,
|
||||
} from './text-window';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { kiReservationSchema, type ReceiptScanResult } from '@trek/shared';
|
||||
|
||||
@@ -155,6 +164,9 @@ export class LlmParseService {
|
||||
baseUrl: config.baseUrl,
|
||||
apiKey: config.apiKey,
|
||||
};
|
||||
// The whole extracted text, kept for a second, shorter try (see extractFitting).
|
||||
let fullText: string | undefined;
|
||||
const longDocument = `${file.originalName}: long document, the middle part was not read`;
|
||||
|
||||
const imageType = imageMimeType(file.originalName);
|
||||
if (imageType && !(await this.visionFor(config))) {
|
||||
@@ -173,18 +185,25 @@ export class LlmParseService {
|
||||
);
|
||||
} else if (config.provider === 'anthropic' && isPdf(file.originalName)) {
|
||||
input.file = { mimeType: MIME_BY_EXT['.pdf'], data: file.buffer };
|
||||
// Not for the model, which reads the PDF itself: only to check its prices against (#2651).
|
||||
fullText = await extractText(file.buffer, file.originalName).catch(() => undefined);
|
||||
console.debug(
|
||||
`[DEBUG] Extracted (native PDF, ${file.buffer.length} bytes) sent to ${config.provider}: ${file.originalName}`,
|
||||
);
|
||||
} else {
|
||||
input.text = await extractText(file.buffer, file.originalName);
|
||||
// Cap the text fed to the model. A flight itinerary lists its legs throughout a long
|
||||
// document, so it keeps a generous window; a single booking has the essentials up top,
|
||||
// so cap it tighter to keep CPU prompt-eval fast (a 11-page rental voucher was ~200s at
|
||||
// 16k, the booking data sits in the first ~2k). Cloud single-shot keeps the tight cap.
|
||||
const MAX_EXTRACT_CHARS =
|
||||
config.provider !== 'local' ? 4000 : detectFlightNumbers(input.text).length > 0 ? 16000 : 6000;
|
||||
if (input.text.length > MAX_EXTRACT_CHARS) input.text = input.text.slice(0, MAX_EXTRACT_CHARS);
|
||||
const extracted = await extractText(file.buffer, file.originalName);
|
||||
fullText = extracted;
|
||||
// Cap the text fed to the model (the windows and why are in text-window.ts). A longer
|
||||
// document loses its middle, never its end, where the total is, and the import says so.
|
||||
const textWindow =
|
||||
config.provider !== 'local'
|
||||
? CLOUD_TEXT_WINDOW
|
||||
: detectFlightNumbers(extracted).length > 0
|
||||
? LOCAL_FLIGHT_TEXT_WINDOW
|
||||
: LOCAL_TEXT_WINDOW;
|
||||
const fitted = fitTextWindow(extracted, textWindow);
|
||||
input.text = fitted.text;
|
||||
if (fitted.omitted > 0) warnings.push(longDocument);
|
||||
// The extracted text IS the booking: traveller name, address, booking
|
||||
// reference. On a centrally administered install that would land in the
|
||||
// operator's log, which is a processing nobody asked for and nobody needs.
|
||||
@@ -253,7 +272,9 @@ export class LlmParseService {
|
||||
|
||||
let raw: Record<string, unknown>[];
|
||||
try {
|
||||
raw = await createLlmClient(config).extract(input);
|
||||
raw = await this.extractFitting(input, config, fullText, () => {
|
||||
if (!warnings.includes(longDocument)) warnings.push(longDocument);
|
||||
});
|
||||
// Same reason: the model answers with the fields it read out of the document.
|
||||
// Parsed, not raw — this prints the nodes the client already read out of the
|
||||
// response, so its single quotes and bare keys are util.inspect's, not the
|
||||
@@ -289,7 +310,40 @@ export class LlmParseService {
|
||||
kiItems.push(normalized as unknown as KiReservation);
|
||||
}
|
||||
|
||||
return { kiItems, warnings };
|
||||
// The whole document, not the window the model read: a total sits at its end.
|
||||
const priced = collapseRepeatedBookingTotal(kiItems as unknown as Record<string, unknown>[], fullText);
|
||||
return { kiItems: priced as unknown as KiReservation[], warnings };
|
||||
}
|
||||
|
||||
/**
|
||||
* One extraction call, and one more with a shorter text when the server says
|
||||
* the prompt does not fit its context. Every cloud provider gets the full
|
||||
* window first, OpenAI-compatible gateways included (#2652); a self-hosted
|
||||
* server with a small context then gets the compact window instead of failing.
|
||||
*/
|
||||
private async extractFitting(
|
||||
input: LlmExtractionInput,
|
||||
config: ResolvedLlmConfig,
|
||||
fullText: string | undefined,
|
||||
onShortened: () => void,
|
||||
): Promise<Record<string, unknown>[]> {
|
||||
const client = createLlmClient(config);
|
||||
try {
|
||||
return await client.extract(input);
|
||||
} catch (err) {
|
||||
// A PDF read natively has no text window to shorten; a second try would only send more.
|
||||
if (
|
||||
!(err instanceof LlmContextOverflowError) ||
|
||||
input.file ||
|
||||
!fullText ||
|
||||
fullText.length <= COMPACT_TEXT_WINDOW
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
input.text = fitTextWindow(fullText, COMPACT_TEXT_WINDOW).text;
|
||||
onShortened();
|
||||
return await client.extract(input);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ export function buildSystemPrompt(): string {
|
||||
'When present, also include at the reservation ROOT: "seat", "class" (fare/cabin class), "platform" (trains/buses), and the total "price" (a number) with "priceCurrency" (ISO 4217 code, e.g. EUR).',
|
||||
'Extract EVERY flight/segment in the document, including return legs — a round trip has TWO OR MORE flights, and each row of a flight table is a separate reservation. Do NOT stop after the first.',
|
||||
"Each flight shares the booking's reservationNumber. Use the date shown for that specific flight as its departureTime; if a flight lists only one date (no separate arrival time), leave arrivalTime null — never reuse another flight's date.",
|
||||
'A total for the whole booking belongs on exactly one reservation, the first leg or segment to depart; never repeat it on each leg. Keep prices that the document states separately for each leg.',
|
||||
'If the document contains no recognizable reservation, return { "reservations": [] }.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* How much extracted text a cloud model is sent. A confirmation keeps its total
|
||||
* and payment block near the end, and 4,000 characters cut both off a long email
|
||||
* without a word (#2652). 24,000 characters are about 6,000 input tokens, far
|
||||
* under the context of any current cloud model, and cost roughly one or two cents
|
||||
* per import on a mid-tier model, a fraction of a cent on a small one.
|
||||
*/
|
||||
export const CLOUD_TEXT_WINDOW = 24_000;
|
||||
|
||||
/**
|
||||
* A local model reads on the CPU, where every character costs prompt-eval time. A
|
||||
* flight itinerary lists its legs throughout a long document, so it keeps a
|
||||
* generous window; a single booking has the essentials up top (an 11-page rental
|
||||
* voucher took ~200s at 16k, its booking data sat in the first ~2k).
|
||||
*/
|
||||
export const LOCAL_FLIGHT_TEXT_WINDOW = 16_000;
|
||||
export const LOCAL_TEXT_WINDOW = 6_000;
|
||||
|
||||
/**
|
||||
* The second try for a server that refused the cloud window for its context
|
||||
* length: a self-hosted vLLM or llama.cpp with 8k tokens takes this together
|
||||
* with the prompt, the schema and the answer (#2652).
|
||||
*/
|
||||
export const COMPACT_TEXT_WINDOW = 8_000;
|
||||
|
||||
/** Share of a window kept from the start; the rest comes from the end. */
|
||||
const HEAD_SHARE = 0.75;
|
||||
|
||||
/** How far a cut may move to land on a line break instead of inside a line. */
|
||||
const LINE_SNAP = 200;
|
||||
|
||||
/**
|
||||
* The text cut to `window` characters, keeping the start and the end. The legs
|
||||
* and names sit up top, the total and the payment summary at the bottom, so the
|
||||
* middle is what goes. The cuts land on line breaks where one is close, so an
|
||||
* amount is never split in half, and a marker says how much is missing.
|
||||
* `omitted` is 0 when the text fit.
|
||||
*/
|
||||
export function fitTextWindow(text: string, window: number): { text: string; omitted: number } {
|
||||
if (text.length <= window) return { text, omitted: 0 };
|
||||
let headEnd = Math.floor(window * HEAD_SHARE);
|
||||
let tailStart = text.length - (window - headEnd);
|
||||
const lineBefore = text.lastIndexOf('\n', headEnd);
|
||||
if (lineBefore > 0 && lineBefore > headEnd - LINE_SNAP) headEnd = lineBefore;
|
||||
const lineAfter = text.indexOf('\n', tailStart - 1);
|
||||
if (lineAfter !== -1 && lineAfter < tailStart + LINE_SNAP) tailStart = lineAfter + 1;
|
||||
const omitted = tailStart - headEnd;
|
||||
return {
|
||||
text: `${text.slice(0, headEnd)}\n[... ${omitted} characters omitted ...]\n${text.slice(tailStart)}`,
|
||||
omitted,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { collapseRepeatedBookingTotal } from '../../../../src/nest/llm-parse/booking-total';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const leg = (
|
||||
flightNumber: string,
|
||||
departureTime: string,
|
||||
price?: unknown,
|
||||
{ currency = 'USD', ref = 'DEMO77', type = 'FlightReservation' } = {},
|
||||
): Record<string, unknown> => ({
|
||||
'@type': type,
|
||||
reservationNumber: ref,
|
||||
...(price === undefined ? {} : { price, priceCurrency: currency }),
|
||||
reservationFor: { flightNumber, departureTime },
|
||||
});
|
||||
|
||||
const prices = (nodes: Record<string, unknown>[]) => nodes.map((n) => n.price ?? null);
|
||||
const TOTAL = 'Payment summary\nTotal paid: USD 200.00';
|
||||
|
||||
describe('collapseRepeatedBookingTotal (#2651)', () => {
|
||||
it('keeps a total the flights repeat on the first to depart, whatever order the model gave', () => {
|
||||
const nodes = [leg('EX101', '2031-02-08T07:00', 200), leg('EX100', '2031-02-05T12:00', 200)];
|
||||
const out = collapseRepeatedBookingTotal(nodes, TOTAL);
|
||||
expect(prices(out)).toEqual([null, 200]);
|
||||
expect(out[0]).not.toHaveProperty('priceCurrency');
|
||||
expect(out[1]).toHaveProperty('priceCurrency', 'USD');
|
||||
});
|
||||
|
||||
it('leaves fares the document states per leg alone, even when they are equal', () => {
|
||||
const text = 'Fare EX100 USD 200.00\nFare EX101 USD 200.00\nTotal paid: USD 400.00';
|
||||
const nodes = [leg('EX100', '2031-02-05T12:00', 200), leg('EX101', '2031-02-08T07:00', 200)];
|
||||
expect(prices(collapseRepeatedBookingTotal(nodes, text))).toEqual([200, 200]);
|
||||
});
|
||||
|
||||
it('leaves a fare printed on every leg alone, even on a line labelled Amount', () => {
|
||||
const text = 'Flight LH1 Amount 89.00 EUR\nFlight LH2 Amount 89.00 EUR\nTotal 178.00 EUR';
|
||||
const nodes = [
|
||||
leg('LH1', '2031-02-05T12:00', 89, { currency: 'EUR' }),
|
||||
leg('LH2', '2031-02-08T07:00', 89, { currency: 'EUR' }),
|
||||
];
|
||||
expect(collapseRepeatedBookingTotal(nodes, text)).toBe(nodes);
|
||||
});
|
||||
|
||||
it('leaves different amounts, references, currencies and mixed booking types alone', () => {
|
||||
const cases = [
|
||||
[leg('EX100', '2031-02-05T12:00', 120), leg('EX101', '2031-02-08T07:00', 80)],
|
||||
[leg('EX100', '2031-02-05T12:00', 200, { ref: 'A1' }), leg('EX101', '2031-02-08T07:00', 200, { ref: 'B2' })],
|
||||
[leg('EX100', '2031-02-05T12:00', 200), leg('EX101', '2031-02-08T07:00', 200, { currency: 'EUR' })],
|
||||
[leg('EX100', '2031-02-05T12:00', 200), leg('Stay', '2031-02-05T15:00', 200, { type: 'LodgingReservation' })],
|
||||
[leg('EX100', '2031-02-05T12:00', 200, { ref: '' }), leg('EX101', '2031-02-08T07:00', 200, { ref: '' })],
|
||||
];
|
||||
for (const nodes of cases) expect(collapseRepeatedBookingTotal(nodes, TOTAL)).toBe(nodes);
|
||||
});
|
||||
|
||||
it('does nothing without document text to check, or when the text names no such total', () => {
|
||||
const nodes = [leg('EX100', '2031-02-05T12:00', 200), leg('EX101', '2031-02-08T07:00', 200)];
|
||||
expect(collapseRepeatedBookingTotal(nodes, undefined)).toBe(nodes);
|
||||
expect(collapseRepeatedBookingTotal(nodes, 'Fare conditions apply.')).toBe(nodes);
|
||||
});
|
||||
|
||||
it('compares in cents and reads a printed amount as one number', () => {
|
||||
const nodes = [leg('EX100', '2031-02-05T12:00', '200.00'), leg('EX101', '2031-02-08T07:00', 200)];
|
||||
// The date on the same line must not run into the amount.
|
||||
expect(prices(collapseRepeatedBookingTotal(nodes, '2031-02-05 Total paid USD 200.00'))).toEqual(['200.00', null]);
|
||||
const euro = [
|
||||
leg('LH1', '2031-02-05T12:00', 1234.5, { currency: 'EUR' }),
|
||||
leg('LH2', '2031-02-08T07:00', 1234.5, { currency: 'EUR' }),
|
||||
];
|
||||
expect(prices(collapseRepeatedBookingTotal(euro, 'Gesamtbetrag: 1.234,50 EUR'))).toEqual([1234.5, null]);
|
||||
});
|
||||
|
||||
it('leaves an unpriced leg as it is and does not change the nodes it was given', () => {
|
||||
const nodes = [
|
||||
leg('EX100', '2031-02-05T12:00', 200),
|
||||
leg('EX101', '2031-02-08T07:00', 200),
|
||||
leg('EX102', '2031-02-09T07:00'),
|
||||
];
|
||||
const snapshot = JSON.parse(JSON.stringify(nodes));
|
||||
const out = collapseRepeatedBookingTotal(nodes, TOTAL);
|
||||
expect(prices(out)).toEqual([200, null, null]);
|
||||
expect(out[2]).toBe(nodes[2]);
|
||||
expect(nodes).toEqual(snapshot);
|
||||
});
|
||||
|
||||
it('needs two priced flights to call it a repeat', () => {
|
||||
const nodes = [leg('EX100', '2031-02-05T12:00', 200), leg('EX101', '2031-02-08T07:00')];
|
||||
expect(collapseRepeatedBookingTotal(nodes, TOTAL)).toBe(nodes);
|
||||
expect(collapseRepeatedBookingTotal([nodes[0]], TOTAL)).toEqual([nodes[0]]);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { readEnv } from '../../../../src/app-config';
|
||||
import { AnthropicClient } from '../../../../src/nest/llm-parse/clients/anthropic.client';
|
||||
import { LlmContextOverflowError } from '../../../../src/nest/llm-parse/clients/context-overflow';
|
||||
import { OpenAiCompatibleClient } from '../../../../src/nest/llm-parse/clients/openai-compatible.client';
|
||||
import type { LlmExtractionInput } from '../../../../src/nest/llm-parse/llm-provider.interface';
|
||||
import {
|
||||
@@ -143,6 +144,23 @@ describe('OpenAiCompatibleClient', () => {
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(1); // no retry on non-400
|
||||
});
|
||||
|
||||
it('stops at a context-length 400: no response_format makes the prompt shorter (#2652)', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse(
|
||||
{
|
||||
error: {
|
||||
message:
|
||||
"This model's maximum context length is 8192 tokens. However, your messages resulted in 9500 tokens.",
|
||||
},
|
||||
},
|
||||
false,
|
||||
400,
|
||||
),
|
||||
);
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toBeInstanceOf(LlmContextOverflowError);
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('retries once with json_object when the server rejects json_schema (400)', async () => {
|
||||
safeFetchLlmMock
|
||||
.mockResolvedValueOnce(jsonResponse({ error: { message: 'response_format type is unavailable' } }, false, 400))
|
||||
@@ -523,6 +541,34 @@ describe('OpenAiCompatibleClient — NuExtract path', () => {
|
||||
});
|
||||
|
||||
describe('AnthropicClient', () => {
|
||||
it('talks to a Messages gateway set as the base URL, and to Anthropic without one (#2655)', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({
|
||||
stop_reason: 'tool_use',
|
||||
content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }],
|
||||
}),
|
||||
);
|
||||
await new AnthropicClient().extract({ ...baseInput, apiKey: 'k', baseUrl: 'http://gateway.lan:4000/' });
|
||||
await new AnthropicClient().extract({ ...baseInput, apiKey: 'k' });
|
||||
expect(fetchFn.mock.calls[0][0]).toBe('http://gateway.lan:4000/v1/messages');
|
||||
expect(fetchFn.mock.calls[1][0]).toBe('https://api.anthropic.com/v1/messages');
|
||||
const body = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string);
|
||||
expect(body.tool_choice).toMatchObject({ type: 'tool', name: 'emit_reservations' });
|
||||
});
|
||||
|
||||
it('tells a prompt too long for a gateway apart from other refusals', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse({ error: { message: 'prompt is too long: 210000 tokens > 200000 maximum' } }, false, 400),
|
||||
);
|
||||
await expect(new AnthropicClient().extract({ ...baseInput, apiKey: 'k' })).rejects.toBeInstanceOf(
|
||||
LlmContextOverflowError,
|
||||
);
|
||||
mockFetch(() => jsonResponse({ error: { message: 'invalid x-api-key' } }, false, 401));
|
||||
const refused = new AnthropicClient().extract({ ...baseInput, apiKey: 'k' });
|
||||
await expect(refused).rejects.toThrow(/401/);
|
||||
await expect(refused).rejects.not.toBeInstanceOf(LlmContextOverflowError);
|
||||
});
|
||||
|
||||
it('sends the pages of a scanned PDF as image blocks after the first', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
routeExtraction,
|
||||
routeImageExtraction,
|
||||
} from '../../../../src/nest/llm-parse/router/extraction-router';
|
||||
import { fitTextWindow } from '../../../../src/nest/llm-parse/text-window';
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
@@ -176,6 +177,26 @@ describe('routeExtraction', () => {
|
||||
expect(flat.currency).toBe('EUR');
|
||||
});
|
||||
|
||||
it('reads a long document cut to its start and end like the whole one, the marker changes nothing (#2652)', async () => {
|
||||
const legs = 'Example Airways\nEX 101 LIS-OPO 2026-11-03 08:10\nEX 102 OPO-LIS 2026-11-07 19:40\nPNR: EX4Y2Z';
|
||||
const text = `${legs}\n${'Fare conditions apply.\n'.repeat(1000)}Total amount: 200.00 USD\n`;
|
||||
const fitted = fitTextWindow(text, 6000);
|
||||
expect(fitted.text).toMatch(/\[\.\.\. \d+ characters omitted \.\.\.\]/);
|
||||
expect(detectFlightNumbers(fitted.text)).toEqual(['EX101', 'EX102']);
|
||||
extractEnforced.mockResolvedValue({
|
||||
flights: [
|
||||
{ vehicle_number: 'EX101', from_code: 'LIS', to_code: 'OPO', departure_time: '2026-11-03T08:10' },
|
||||
{ vehicle_number: 'EX102', from_code: 'OPO', to_code: 'LIS', departure_time: '2026-11-07T19:40' },
|
||||
],
|
||||
});
|
||||
await routeExtraction(fitted.text, CTX);
|
||||
const flats = mapToKi.mock.calls[0][0];
|
||||
expect(flats).toHaveLength(2);
|
||||
expect(flats[0].booking_reference).toBe('EX4Y2Z');
|
||||
expect(flats[0].price).toBe('200.00');
|
||||
expect(flats[0].currency).toBe('USD');
|
||||
});
|
||||
|
||||
it("lets the document's currency override the model but keeps a price the model already found", async () => {
|
||||
extractEnforced.mockResolvedValue({
|
||||
name: 'B&B Hotel',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import { LlmContextOverflowError } from '../../../../src/nest/llm-parse/clients/context-overflow';
|
||||
import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver';
|
||||
import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
|
||||
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
@@ -116,7 +117,8 @@ describe('LlmParseService', () => {
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.file).toEqual({ mimeType: 'application/pdf', data: expect.any(Buffer) });
|
||||
expect(input.text).toBeUndefined();
|
||||
expect(extractText).not.toHaveBeenCalled();
|
||||
// Read only to check a repeated booking total against (#2651); the model never sees it.
|
||||
expect(extractText).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('warns when a pdf yields no readable text (e.g. a scan)', async () => {
|
||||
@@ -204,8 +206,13 @@ describe('LlmParseService', () => {
|
||||
extractText.mockResolvedValue(long);
|
||||
detectFlightNumbers.mockReturnValue([]);
|
||||
routeExtraction.mockResolvedValue({ kiItems: [], warnings: [] });
|
||||
await svc().parse(file('hotel.txt'), 1);
|
||||
expect(routeExtraction.mock.calls[0][0]).toHaveLength(6000); // single booking → tighter cap
|
||||
const res = await svc().parse(file('hotel.txt'), 1);
|
||||
// single booking → tighter cap, keeping the start and the end
|
||||
const sent = routeExtraction.mock.calls[0][0] as string;
|
||||
expect(sent).toBe(`${'x'.repeat(4500)}
|
||||
[... 1000 characters omitted ...]
|
||||
${'x'.repeat(1500)}`);
|
||||
expect(res.warnings).toEqual(['hotel.txt: long document, the middle part was not read']);
|
||||
});
|
||||
|
||||
it('degrades to a warning when the local router throws', async () => {
|
||||
@@ -332,7 +339,8 @@ describe('LlmParseService: cleaning up an AI answer (#2477)', () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic' }));
|
||||
extract.mockResolvedValue([stay({ priceCurrency: 'ZZZ' })]);
|
||||
const res = await svc().parse(file('b.pdf', '%PDF'), 1);
|
||||
expect(extractText).not.toHaveBeenCalled();
|
||||
// The text read for the total check is not a document the currency is asked from.
|
||||
expect(extract.mock.calls[0][0].text).toBeUndefined();
|
||||
expect(res.kiItems[0]).not.toHaveProperty('priceCurrency');
|
||||
});
|
||||
|
||||
@@ -553,3 +561,135 @@ describe('LlmParseService: a scanned PDF', () => {
|
||||
expect(renderPdfPages).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A long confirmation keeps its total near the end (#2652): two Example Airways
|
||||
* legs up top, then itinerary and footer text, then the payment block. Synthetic.
|
||||
*/
|
||||
describe('LlmParseService: a long document', () => {
|
||||
const legs = [
|
||||
'Example Airways booking confirmation',
|
||||
'Booking reference: EX4Y2Z',
|
||||
'EX 101 Lisbon (LIS) to Porto (OPO) 2026-11-03 08:10 - 09:05',
|
||||
'EX 102 Porto (OPO) to Lisbon (LIS) 2026-11-07 19:40 - 20:35',
|
||||
].join('\n');
|
||||
const filler = (chars: number) =>
|
||||
'Baggage, check-in and fare conditions apply as published. '.repeat(Math.ceil(chars / 58)).slice(0, chars);
|
||||
const confirmation = (fillerChars: number) =>
|
||||
`${legs}\n${filler(fillerChars)}\nPayment summary\nTotal paid: USD 200.00\nThank you for flying with us.`;
|
||||
|
||||
it('sends an early total and leaves a short document untouched, without a warning', async () => {
|
||||
const doc = `${legs}\nTotal paid: USD 200.00\n${filler(1500)}`;
|
||||
extractText.mockResolvedValue(doc);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
expect(extract.mock.calls[0][0].text).toBe(doc);
|
||||
expect(res.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it('sends a total that sits past the first 4,000 characters to a cloud model', async () => {
|
||||
const doc = confirmation(6000);
|
||||
expect(doc.indexOf('Total paid')).toBeGreaterThan(4000);
|
||||
extractText.mockResolvedValue(doc);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
const sent = extract.mock.calls[0][0].text as string;
|
||||
expect(sent).toBe(doc);
|
||||
expect(sent).toContain('Total paid: USD 200.00');
|
||||
expect(res.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps the start and the end of a document longer than the window, and says so', async () => {
|
||||
const doc = confirmation(60_000);
|
||||
extractText.mockResolvedValue(doc);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
const sent = extract.mock.calls[0][0].text as string;
|
||||
expect(sent.length).toBeLessThan(doc.length);
|
||||
expect(sent.startsWith(legs)).toBe(true);
|
||||
expect(sent).toContain('Total paid: USD 200.00');
|
||||
expect(sent.endsWith('Thank you for flying with us.')).toBe(true);
|
||||
expect(sent).toMatch(/\[\.\.\. \d+ characters omitted \.\.\.\]/);
|
||||
expect(res.warnings).toEqual(['confirmation.txt: long document, the middle part was not read']);
|
||||
});
|
||||
|
||||
it('keeps the start and the end for a local model too, inside its own window', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local' }));
|
||||
routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'FlightReservation' }], warnings: ['note'] });
|
||||
const doc = confirmation(9000);
|
||||
extractText.mockResolvedValue(doc);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
const sent = routeExtraction.mock.calls[0][0] as string;
|
||||
expect(sent.startsWith(legs)).toBe(true);
|
||||
expect(sent).toContain('Total paid: USD 200.00');
|
||||
expect(sent).toMatch(/\[\.\.\. \d+ characters omitted \.\.\.\]/);
|
||||
expect(res.warnings).toEqual(['confirmation.txt: long document, the middle part was not read', 'note']);
|
||||
});
|
||||
});
|
||||
|
||||
/** Too long for a small self-hosted server, and a total shared by two flights (#2651, #2652). */
|
||||
describe('LlmParseService: a server that runs out of context, and a shared booking total', () => {
|
||||
const legs =
|
||||
'Example Airways\nBooking reference: DEMO77\nEX100 LIS-OPO 2031-02-05 12:00\nEX101 OPO-LIS 2031-02-08 07:00';
|
||||
const doc = `${legs}\n${'Fare conditions apply.\n'.repeat(1500)}Payment summary\nTotal paid: USD 200.00\n`;
|
||||
const flight = (n: string, dep: string) => ({
|
||||
'@type': 'FlightReservation',
|
||||
reservationNumber: 'DEMO77',
|
||||
price: 200,
|
||||
priceCurrency: 'USD',
|
||||
reservationFor: { '@type': 'Flight', flightNumber: n, departureTime: dep },
|
||||
});
|
||||
|
||||
it('asks once more with the compact window when the server says the prompt does not fit', async () => {
|
||||
extractText.mockResolvedValue(doc);
|
||||
extract
|
||||
.mockRejectedValueOnce(
|
||||
new LlmContextOverflowError("LLM request failed (400): This model's maximum context length is 8192 tokens"),
|
||||
)
|
||||
.mockResolvedValueOnce([{ '@type': 'FlightReservation' }]);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
expect(extract).toHaveBeenCalledTimes(2);
|
||||
const second = extract.mock.calls[1][0].text as string;
|
||||
expect(second.length).toBeLessThan(8_100);
|
||||
expect(second.startsWith(legs)).toBe(true);
|
||||
expect(second).toContain('Total paid: USD 200.00');
|
||||
expect(res.kiItems).toHaveLength(1);
|
||||
expect(res.warnings).toEqual(['confirmation.txt: long document, the middle part was not read']);
|
||||
});
|
||||
|
||||
it('does not retry a PDF read natively, which has no text window to shorten', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic' }));
|
||||
extractText.mockResolvedValue(doc);
|
||||
extract.mockRejectedValueOnce(new LlmContextOverflowError('Anthropic request failed (400): prompt is too long'));
|
||||
const res = await svc().parse(file('ticket.pdf', '%PDF'), 1);
|
||||
expect(extract).toHaveBeenCalledTimes(1);
|
||||
expect(res.warnings).toHaveLength(1);
|
||||
expect(res.warnings[0]).toMatch(/AI parsing failed/);
|
||||
});
|
||||
|
||||
it('does not retry any other failure', async () => {
|
||||
extractText.mockResolvedValue(doc);
|
||||
extract.mockRejectedValueOnce(new Error('LLM request failed (401): bad key'));
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
expect(extract).toHaveBeenCalledTimes(1);
|
||||
expect(res.warnings[0]).toMatch(/AI parsing failed/);
|
||||
});
|
||||
|
||||
it('keeps a total both flights repeat on the first to depart only', async () => {
|
||||
extractText.mockResolvedValue(doc);
|
||||
extract.mockResolvedValue([flight('EX101', '2031-02-08T07:00:00'), flight('EX100', '2031-02-05T12:00:00')]);
|
||||
const res = await svc().parse(file('confirmation.txt'), 1);
|
||||
const [late, early] = res.kiItems as unknown as Record<string, unknown>[];
|
||||
expect(early).toMatchObject({ price: 200, priceCurrency: 'USD' });
|
||||
expect(late).not.toHaveProperty('price');
|
||||
expect(late).not.toHaveProperty('priceCurrency');
|
||||
});
|
||||
|
||||
it('checks a native PDF against its text too, without sending that text', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic' }));
|
||||
extractText.mockResolvedValue(doc);
|
||||
extract.mockResolvedValue([flight('EX100', '2031-02-05T12:00:00'), flight('EX101', '2031-02-08T07:00:00')]);
|
||||
const res = await svc().parse(file('ticket.pdf', '%PDF'), 1);
|
||||
expect(extract.mock.calls[0][0].text).toBeUndefined();
|
||||
const [first, second] = res.kiItems as unknown as Record<string, unknown>[];
|
||||
expect(first).toHaveProperty('price', 200);
|
||||
expect(second).not.toHaveProperty('price');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,6 +21,12 @@ describe('llm-prompt', () => {
|
||||
for (const t of KI_RESERVATION_TYPES) expect(prompt).toContain(t);
|
||||
});
|
||||
|
||||
it('puts a total for the whole booking on one leg, not on each (#2651)', () => {
|
||||
const prompt = buildSystemPrompt();
|
||||
expect(prompt).toContain('A total for the whole booking belongs on exactly one reservation');
|
||||
expect(prompt).toContain('never repeat it on each leg');
|
||||
});
|
||||
|
||||
it('instructs JSON-only output wrapped in reservations', () => {
|
||||
const prompt = buildSystemPrompt();
|
||||
expect(prompt).toMatch(/"reservations"/);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import {
|
||||
CLOUD_TEXT_WINDOW,
|
||||
fitTextWindow,
|
||||
LOCAL_FLIGHT_TEXT_WINDOW,
|
||||
LOCAL_TEXT_WINDOW,
|
||||
} from '../../../../src/nest/llm-parse/text-window';
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
describe('fitTextWindow', () => {
|
||||
it('leaves a text that fits as it is', () => {
|
||||
expect(fitTextWindow('short', 10)).toEqual({ text: 'short', omitted: 0 });
|
||||
expect(fitTextWindow('x'.repeat(10), 10)).toEqual({ text: 'x'.repeat(10), omitted: 0 });
|
||||
});
|
||||
|
||||
it('keeps three quarters from the start and a quarter from the end around a marker', () => {
|
||||
const text = 'a'.repeat(5000) + 'b'.repeat(5000);
|
||||
const { text: out, omitted } = fitTextWindow(text, 4000);
|
||||
expect(omitted).toBe(6000);
|
||||
expect(out).toBe(`${'a'.repeat(3000)}\n[... 6000 characters omitted ...]\n${'b'.repeat(1000)}`);
|
||||
});
|
||||
|
||||
it('cuts on a nearby line break, so a line at either cut stays whole', () => {
|
||||
const lines = Array.from({ length: 400 }, (_, i) => `Line ${String(i).padStart(3, '0')}: Total paid USD 200.00`);
|
||||
const text = lines.join('\n');
|
||||
const { text: out, omitted } = fitTextWindow(text, 4000);
|
||||
const [head, tail] = out.split(/\n\[\.\.\. \d+ characters omitted \.\.\.\]\n/);
|
||||
for (const part of [head, tail]) for (const line of part.split('\n')) expect(lines).toContain(line);
|
||||
expect(head.length + tail.length).toBeLessThanOrEqual(4000);
|
||||
expect(head.length + tail.length + omitted).toBe(text.length);
|
||||
expect(tail.endsWith(lines[lines.length - 1])).toBe(true);
|
||||
});
|
||||
|
||||
it('cuts inside a line when no line break is close', () => {
|
||||
const text = `${'a'.repeat(2000)}\n${'b'.repeat(8000)}`;
|
||||
const { text: out } = fitTextWindow(text, 4000);
|
||||
expect(out.startsWith(`${'a'.repeat(2000)}\n${'b'.repeat(999)}\n[...`)).toBe(true);
|
||||
});
|
||||
|
||||
it('gives a cloud model a far wider window than a local one, and keeps the local ones', () => {
|
||||
expect(CLOUD_TEXT_WINDOW).toBe(24_000);
|
||||
expect(LOCAL_FLIGHT_TEXT_WINDOW).toBe(16_000);
|
||||
expect(LOCAL_TEXT_WINDOW).toBe(6_000);
|
||||
});
|
||||
});
|
||||
@@ -26,10 +26,14 @@ The addon supports three providers:
|
||||
|----------|-----------|-------|
|
||||
| **Local (Ollama)** | Your own hardware | No booking data leaves your network. Recommended for privacy; runs on a CPU. |
|
||||
| **OpenAI** | OpenAI's API, or any **OpenAI-compatible** endpoint through a custom base URL | Needs an API key. |
|
||||
| **Anthropic** | Anthropic's API | Needs an API key. **Reads PDFs natively, scans included.** |
|
||||
| **Anthropic** | Anthropic's API, or a gateway that speaks its Messages API through a custom base URL | Needs an API key. **Reads PDFs natively, scans included.** |
|
||||
|
||||
> **Scanned PDFs:** local and OpenAI-compatible models get the document's *extracted text*. A scanned PDF has no text layer. When the model reads images, the first two pages are drawn as images and sent like a photo; otherwise those providers return nothing for it. **Anthropic** receives the PDF itself and reads scans either way.
|
||||
>
|
||||
> **Long documents:** extracted text is sent up to 24,000 characters to a cloud model (OpenAI, Anthropic for anything that is not a PDF, OpenAI-compatible), and up to 6,000 to a local model (16,000 for a flight itinerary), since a CPU reads every character. A longer document keeps its start and its end, where the total and the payment summary sit, and loses the middle. The import then warns *long document, the middle part was not read*, so check the booking in the review. A self-hosted server whose context is too small for 24,000 characters (vLLM or llama.cpp with 8k tokens, say) refuses the request; TREK then asks once more with 8,000 characters, start and end kept, and warns the same way. Ollama's OpenAI-compatible endpoint cuts a long prompt without saying so, so use the **Local** provider for Ollama.
|
||||
>
|
||||
> **A total for the whole booking:** a round trip that states one total for all flights gets that price once, on the first flight, so saving every leg does not book the purchase twice. Fares the document lists per leg stay on their legs.
|
||||
>
|
||||
> **Photos:** a photo (JPG, PNG, WEBP) goes to the model as an image, with every provider, as long as the model reads images. See [Photos](#photos).
|
||||
|
||||
## Admin: instance-wide configuration
|
||||
@@ -39,7 +43,7 @@ Once the addon is on, a configuration panel appears right under it in [Admin-Add
|
||||
> *Set instance-wide config (applies to all users). Leave blank to let each user configure their own provider.*
|
||||
|
||||
- **Provider**: Local · OpenAI-compatible, OpenAI, or Anthropic.
|
||||
- **Base URL**: for every provider except Anthropic. It defaults to `http://localhost:11434/v1` for a local Ollama server and `https://api.openai.com/v1` for OpenAI. Any OpenAI-compatible endpoint goes here.
|
||||
- **Base URL**: empty means the provider's own address: `http://localhost:11434/v1` for a local Ollama server, `https://api.openai.com/v1` for OpenAI and `https://api.anthropic.com` for Anthropic. Any OpenAI-compatible endpoint goes here. For Anthropic it can name a gateway that speaks the Anthropic Messages API; give it without `/v1`, since TREK adds `/v1/messages`, and the gateway has to accept PDF document blocks and forced tool calls. Picking another provider empties the field, so an address meant for one never receives another provider's key.
|
||||
- **API key**: optional for a local server (the field says *(often not required)*), required for the cloud providers. It is stored **encrypted** and shown masked (`••••••••`) once saved; leaving it unchanged keeps the stored key.
|
||||
- **Model**: the model id, for example `qwen3.5:4b`, `gpt-4o` or `claude-opus-4-8`.
|
||||
- **Model reads images**: **Automatic**, **Yes** or **No**. With the Local provider, **Automatic** asks the Ollama server what the model can do (`/api/show`) and follows its answer. With OpenAI or Anthropic, **Automatic** means no. **Yes** and **No** apply as set, whatever the server or the model id says. Photos are only accepted when this comes out as yes.
|
||||
|
||||
Reference in New Issue
Block a user