feat(llm-parse): a typed outcome for every AI reading instead of warning text

LlmParseService.parse() now says how a reading ended (ok, empty, no_text,
quota_exhausted, ai_disabled, user_cap, provider_error, too_large). The clients throw
LlmRequestError with status and body code, so the TREK Cloud gate's 429 QUOTA_SPENT
and HARD_CAP read as a spent allowance. The preview reports the outcome per file and
the assistant reads its empty reason from it rather than matching warning strings.
This commit is contained in:
Maurice
2026-11-13 14:59:53 +01:00
parent 5dc1de0e5b
commit d557f08e28
13 changed files with 256 additions and 34 deletions
@@ -38,20 +38,17 @@ interface BookingRead {
reason: BookingReadReason | null;
}
/** The warnings of the AI fallback that say it could not read the file (llm-parse.service.ts words them). */
const AI_FAILED = [/AI parsing failed/, /could not read file/, /does not read images/];
const NO_TEXT = /no readable text found/;
/**
* Why a preview of one file came back empty, from its warnings and its file
* report. The import dialog shows those warnings as text; the chat names the
* reason in its answer instead, so it reads them here once.
* Why a preview of one file came back empty, from the outcome its file report
* carries where the AI read it. The import dialog shows the warnings as text;
* the chat names the reason in its answer instead. Every way the AI reading can
* fail is one reason to the chat: the AI reader could not read it.
*/
export function emptyReason(preview: BookingImportPreviewResponse, aiAvailable: boolean): BookingReadReason {
if (!aiAvailable) return 'ai_not_configured';
if (preview.warnings.some((warning) => NO_TEXT.test(warning))) return 'no_text';
if (preview.warnings.some((warning) => AI_FAILED.some((pattern) => pattern.test(warning)))) return 'ai_failed';
return 'nothing_found';
const outcome = preview.files?.[0]?.outcome;
if (outcome === undefined || outcome === 'ok' || outcome === 'empty') return 'nothing_found';
return outcome === 'no_text' ? 'no_text' : 'ai_failed';
}
/**
@@ -32,6 +32,7 @@ import type {
BookingImportFailureReason,
BookingImportMode,
BookingImportFileReport,
LlmParseOutcome,
Reservation,
} from '@trek/shared';
@@ -209,6 +210,7 @@ export class BookingImportService {
for (const file of files) {
let kiItems: KiReservation[] = [];
let aiUsed = false;
let outcome: LlmParseOutcome | undefined;
// Stage 1: kitinerary (skipped entirely when forcing AI, and for a photo,
// which only a model can read).
@@ -229,10 +231,11 @@ export class BookingImportService {
aiUsed = true;
const llm = await this.llmParse.parse({ buffer: file.buffer, originalName: file.originalname }, userId);
kiItems = llm.kiItems;
outcome = llm.outcome;
allWarnings.push(...llm.warnings);
}
fileReports.push({ fileName: file.originalname, aiAvailable, aiUsed });
fileReports.push({ fileName: file.originalname, aiAvailable, aiUsed, ...(outcome && { outcome }) });
if (kiItems.length === 0) {
allWarnings.push(`${file.originalname}: no reservations found`);
@@ -3,6 +3,7 @@ import { safeFetchLlm } from '../../../utils/ssrfGuard';
import { toReservationList } from '../lenient-json';
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
import { llmRequestError } from './llm-request-error';
import { UnreadableLlmResponse } from './openai-compatible.client';
/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
@@ -86,10 +87,10 @@ export class AnthropicClient implements LlmExtractionClient {
if (!res.ok) {
const detail = await res.text().catch(() => '');
const message = `Anthropic request failed (${res.status}): ${detail.slice(0, 300)}`;
const refused = llmRequestError('Anthropic request failed', res.status, detail);
// A gateway in front of a smaller model can run out of context (#2652, #2655).
if (res.status === 400 && isContextOverflow(detail)) throw new LlmContextOverflowError(message);
throw new Error(message);
if (res.status === 400 && isContextOverflow(detail)) throw new LlmContextOverflowError(refused.message);
throw refused;
}
const data = (await res.json()) as {
@@ -0,0 +1,48 @@
/**
* A provider answered a parsing request with an error status. The message
* stays the one the import has always logged and shown; `status` and `code`
* let the caller tell a spent allowance from a broken request without
* reading that message.
*/
export class LlmRequestError extends Error {
constructor(
message: string,
readonly status: number,
/** The error code the body named (`QUOTA_SPENT` from the TREK Cloud gate), when it named one. */
readonly code: string | undefined,
) {
super(message);
this.name = 'LlmRequestError';
}
}
/** How much of an error body is read for its code: the gate's and the providers' are a few hundred bytes. */
const MAX_CODE_BODY = 4096;
/**
* The `code` of a JSON error body, at the top (the TREK Cloud gate's
* `{ error, code }`) or inside `error` (OpenAI's and Anthropic's shape).
* Undefined for a body that is not JSON or names none.
*/
export function errorBodyCode(detail: string): string | undefined {
if (!detail || detail.length > MAX_CODE_BODY) return undefined;
let body: unknown;
try {
body = JSON.parse(detail);
} catch {
return undefined;
}
if (!body || typeof body !== 'object') return undefined;
const record = body as { code?: unknown; error?: unknown };
if (typeof record.code === 'string') return record.code;
const inner = record.error;
if (inner && typeof inner === 'object' && typeof (inner as { code?: unknown }).code === 'string') {
return (inner as { code: string }).code;
}
return undefined;
}
/** The error a client throws for a refused request, its message as the import has always worded it. */
export function llmRequestError(label: string, status: number, detail: string): LlmRequestError {
return new LlmRequestError(`${label} (${status}): ${detail.slice(0, 300)}`, status, errorBodyCode(detail));
}
@@ -3,6 +3,7 @@ import { safeFetchLlm } from '../../../utils/ssrfGuard';
import { parseLenientJson, toReservationList } from '../lenient-json';
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
import { isContextOverflow, LlmContextOverflowError } from './context-overflow';
import { llmRequestError } from './llm-request-error';
import { isNuExtractModel, buildNuExtractUserText, nuExtractToKiReservations } from './nuextract';
/** The answer cap when LLM_MAX_OUTPUT_TOKENS is unset. */
@@ -187,7 +188,7 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
}
if (!res.ok) {
throw new Error(`LLM request failed (${res.status}): ${detail.slice(0, 300)}`);
throw llmRequestError('LLM request failed', res.status, detail);
}
// Which rung answered. A report could not tell whether the schema reached the
// model at all or the ladder had fallen back to the prompt alone (#2477).
+42 -6
View File
@@ -2,8 +2,9 @@ import { RuntimeEnvService } from '../app-config/runtime-env.service';
import type { KiReservation } from '../booking-import/kitinerary.types';
import { collapseRepeatedBookingTotal } from './booking-total';
import { LlmContextOverflowError } from './clients/context-overflow';
import { LlmRequestError } from './clients/llm-request-error';
import { toIsoCurrency } from './currency-code';
import { capImage, imageMimeType, renderPdfPages } from './image-input';
import { capImage, ImageTooLargeError, imageMimeType, renderPdfPages } from './image-input';
import { createLlmClient } from './llm-client.factory';
import type { ResolvedLlmConfig } from './llm-config';
import { LlmConfigResolver } from './llm-config.resolver';
@@ -34,7 +35,7 @@ import {
LOCAL_TEXT_WINDOW,
} from './text-window';
import { Injectable } from '@nestjs/common';
import { kiReservationSchema, type ReceiptScanResult } from '@trek/shared';
import { kiReservationSchema, type LlmParseOutcome, type ReceiptScanResult } from '@trek/shared';
const MIME_BY_EXT: Record<string, string> = {
'.pdf': 'application/pdf',
@@ -42,7 +43,30 @@ const MIME_BY_EXT: Record<string, string> = {
export interface LlmParseResult {
kiItems: KiReservation[];
/** For the person to read; `outcome` is what a caller decides on. */
warnings: string[];
outcome: LlmParseOutcome;
}
/** The codes the TREK Cloud gate answers a spent allowance with, on a 429. */
const QUOTA_CODES = new Set(['QUOTA_SPENT', 'HARD_CAP']);
/**
* How a failed reading ended, from the error alone: a spent Cloud allowance,
* a document too large for the model or the gate, or a provider that failed.
*/
export function failureOutcome(err: unknown): LlmParseOutcome {
if (err instanceof LlmRequestError) {
if (err.status === 429 && err.code !== undefined && QUOTA_CODES.has(err.code)) return 'quota_exhausted';
if (err.status === 413) return 'too_large';
}
if (err instanceof LlmContextOverflowError || err instanceof ImageTooLargeError) return 'too_large';
return 'provider_error';
}
/** A reading that found bookings or did not. */
function foundOutcome(items: readonly unknown[]): LlmParseOutcome {
return items.length > 0 ? 'ok' : 'empty';
}
/**
@@ -181,7 +205,7 @@ export class LlmParseService {
async parse(file: { buffer: Buffer; originalName: string }, userId: number): Promise<LlmParseResult> {
const config = await this.llmConfig.resolve(userId);
if (!config) return { kiItems: [], warnings: ['AI parsing is not configured'] };
if (!config) return { kiItems: [], warnings: ['AI parsing is not configured'], outcome: 'ai_disabled' };
const warnings: string[] = [];
const input: LlmExtractionInput = {
@@ -197,7 +221,11 @@ export class LlmParseService {
const imageType = imageMimeType(file.originalName);
if (imageType && !(await this.visionFor(config))) {
return { kiItems: [], warnings: [`${file.originalName}: the configured model does not read images`] };
return {
kiItems: [],
warnings: [`${file.originalName}: the configured model does not read images`],
outcome: 'provider_error',
};
}
// A photo goes as an image to every provider. Native PDF only for Anthropic
@@ -249,6 +277,7 @@ export class LlmParseService {
warnings: [
`${file.originalName}: no readable text found (a scanned PDF needs a model that reads images)`,
],
outcome: 'no_text',
};
}
input.text = undefined;
@@ -264,6 +293,7 @@ export class LlmParseService {
return {
kiItems: [],
warnings: [`${file.originalName}: could not read file — ${err instanceof Error ? err.message : String(err)}`],
outcome: err instanceof ImageTooLargeError ? 'too_large' : 'provider_error',
};
}
@@ -284,7 +314,11 @@ export class LlmParseService {
ctx,
)
: await routeExtraction(input.text ?? '', ctx);
return { kiItems: routed.kiItems, warnings: [...warnings, ...routed.warnings] };
return {
kiItems: routed.kiItems,
warnings: [...warnings, ...routed.warnings],
outcome: foundOutcome(routed.kiItems),
};
} catch (err) {
console.error(
`[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`,
@@ -293,6 +327,7 @@ export class LlmParseService {
return {
kiItems: [],
warnings: [`${file.originalName}: AI parsing failed — ${err instanceof Error ? err.message : String(err)}`],
outcome: failureOutcome(err),
};
}
}
@@ -316,6 +351,7 @@ export class LlmParseService {
return {
kiItems: [],
warnings: [`${file.originalName}: AI parsing failed — ${err instanceof Error ? err.message : String(err)}`],
outcome: failureOutcome(err),
};
}
@@ -339,7 +375,7 @@ export class LlmParseService {
// The whole document, not the window the model read: a total sits at its end.
const priced = collapseRepeatedBookingTotal(kiItems as unknown as Record<string, unknown>[], fullText);
return { kiItems: priced as unknown as KiReservation[], warnings };
return { kiItems: priced as unknown as KiReservation[], warnings, outcome: foundOutcome(priced) };
}
/**
+3 -1
View File
@@ -217,7 +217,9 @@ describe('Booking import e2e (#2477): a schema-bound provider on the upload rout
expect(res.body.warnings[0]).toMatch(
/^Incomplete LodgingReservation in Bestätigung_1234567890\.PDF\[0\] \(no reservationFor\)/,
);
expect(res.body.files).toEqual([{ fileName: 'Bestätigung_1234567890.PDF', aiAvailable: true, aiUsed: true }]);
expect(res.body.files).toEqual([
{ fileName: 'Bestätigung_1234567890.PDF', aiAvailable: true, aiUsed: true, outcome: 'ok' },
]);
});
it('once Gemini fills the venue: one hotel in EUR, attached to the uploaded file', async () => {
+1 -1
View File
@@ -184,7 +184,7 @@ describe('Photos through AI Parsing e2e', () => {
expect(res.status).toBe(201);
expect(res.body.items).toHaveLength(1);
expect(res.body.files).toEqual([{ fileName: 'ticket.png', aiAvailable: true, aiUsed: true }]);
expect(res.body.files).toEqual([{ fileName: 'ticket.png', aiAvailable: true, aiUsed: true, outcome: 'ok' }]);
const image = sentUserContent().find((p: { type: string }) => p.type === 'image_url');
expect(image.image_url.url).toMatch(/^data:image\/png;base64,/);
});
@@ -15,7 +15,7 @@ import { DomainError } from '../../../../src/nest/common/domain-error';
import { McpToolGuardsService } from '../../../../src/nest/mcp-shared/mcp-tool-guards.service';
import { TripAccessService } from '../../../../src/nest/trip-membership/trip-access.service';
import { Test } from '@nestjs/testing';
import type { BookingImportPreviewItem } from '@trek/shared';
import type { BookingImportPreviewItem, LlmParseOutcome } from '@trek/shared';
import { describe, expect, it, vi } from 'vitest';
@@ -115,15 +115,21 @@ describe('AssistantBookingImportService', () => {
expect(read.found).toEqual({ bookings: expect.any(Array) });
});
it('ASTBKI-006: reads the reason from the warnings of the preview, for each way it comes back empty', () => {
const preview = (...warnings: string[]) => ({ items: [], warnings });
expect(emptyReason(preview('a.pdf: no reservations found'), false)).toBe('ai_not_configured');
const scan = preview('a.pdf: no readable text found (a scanned PDF needs a model that reads images)');
expect(emptyReason(scan, true)).toBe('no_text');
expect(emptyReason(preview('a.pdf: AI parsing failed — 401'), true)).toBe('ai_failed');
expect(emptyReason(preview('a.pdf: could not read file — bad xref'), true)).toBe('ai_failed');
expect(emptyReason(preview('a.jpg: the configured model does not read images'), true)).toBe('ai_failed');
expect(emptyReason(preview('a.pdf: no reservations found'), true)).toBe('nothing_found');
it('ASTBKI-006: reads the reason from the outcome of the AI reading, for each way it comes back empty', () => {
const preview = (outcome?: LlmParseOutcome) => ({
items: [],
warnings: ['a.pdf: some text the reason is no longer read from'],
files: [{ fileName: 'a.pdf', aiAvailable: true, aiUsed: outcome !== undefined, ...(outcome && { outcome }) }],
});
expect(emptyReason(preview('provider_error'), false)).toBe('ai_not_configured');
expect(emptyReason(preview('no_text'), true)).toBe('no_text');
for (const failed of ['provider_error', 'quota_exhausted', 'too_large', 'ai_disabled', 'user_cap'] as const) {
expect(emptyReason(preview(failed), true)).toBe('ai_failed');
}
expect(emptyReason(preview('empty'), true)).toBe('nothing_found');
// The extractor read it alone: no AI outcome, nothing found.
expect(emptyReason(preview(), true)).toBe('nothing_found');
expect(emptyReason({ items: [], warnings: [] }, true)).toBe('nothing_found');
});
it('ASTBKI-003: an applied booking is created through the import, on the trip, without a socket', async () => {
@@ -117,6 +117,16 @@ describe('BookingImportService.preview', () => {
expect(res.files![0]).toEqual({ fileName: 'a.pdf', aiAvailable: true, aiUsed: true });
});
it('reports how the AI reading of a file ended, where it ran', async () => {
const { svc } = make({
kit: true,
ai: true,
parse: async () => ({ kiItems: [], warnings: ['a.pdf: AI parsing failed — 429'], outcome: 'quota_exhausted' }),
});
const res = await svc.preview([file()], 'fallback-on-empty', 1);
expect(res.files).toEqual([{ fileName: 'a.pdf', aiAvailable: true, aiUsed: true, outcome: 'quota_exhausted' }]);
});
it('fallback-on-empty: skips the LLM when kitinerary already found items', async () => {
const { svc, llmParse } = make({ kit: true, ai: true, extract: async () => [HOTEL_KI] });
const res = await svc.preview([file()], 'fallback-on-empty', 1);
@@ -1,8 +1,10 @@
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
import { LlmContextOverflowError } from '../../../../src/nest/llm-parse/clients/context-overflow';
import { LlmRequestError } from '../../../../src/nest/llm-parse/clients/llm-request-error';
import { ImageTooLargeError } from '../../../../src/nest/llm-parse/image-input';
import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver';
import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
import { failureOutcome, LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
import { describe, it, expect, vi, beforeEach } from 'vitest';
@@ -699,3 +701,53 @@ describe('LlmParseService: a server that runs out of context, and a shared booki
expect(second).not.toHaveProperty('price');
});
});
describe('LlmParseService.parse outcome', () => {
it('LLM-OUTCOME-001: names a reading that found bookings, and one that found none', async () => {
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('ok');
extract.mockResolvedValueOnce([]);
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty');
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local' }));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('ok');
routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: [] });
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty');
});
it('LLM-OUTCOME-002: no AI set up, a scan nothing can read, and a photo the model cannot see', async () => {
resolveLlmConfig.mockReturnValueOnce(null);
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('ai_disabled');
extractText.mockResolvedValueOnce('');
expect((await svc().parse(file('scan.pdf', '%PDF'), 1)).outcome).toBe('no_text');
expect((await svc().parse(file('ticket.jpg'), 1)).outcome).toBe('provider_error');
});
it('LLM-OUTCOME-003: a spent Cloud allowance, a refusal for size and any other failure', async () => {
extract.mockRejectedValueOnce(new LlmRequestError('Anthropic request failed (429): {}', 429, 'QUOTA_SPENT'));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('quota_exhausted');
extract.mockRejectedValueOnce(new LlmRequestError('Anthropic request failed (413): {}', 413, undefined));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('too_large');
extract.mockRejectedValueOnce(new Error('LLM request failed (401): bad key'));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('provider_error');
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local' }));
routeExtraction.mockRejectedValueOnce(new Error('ollama down'));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('provider_error');
});
it('LLM-OUTCOME-004: a photo too large to decode is too large, a file that cannot be read a provider error', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
const huge = await svc().parse({ buffer: pngClaiming(20000, 20000), originalName: 'big.png' }, 1);
expect(huge.outcome).toBe('too_large');
extractText.mockRejectedValueOnce(new Error('bad xref'));
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('provider_error');
});
it('LLM-OUTCOME-005: failureOutcome reads the status and the code, not the message', () => {
expect(failureOutcome(new LlmRequestError('x', 429, 'HARD_CAP'))).toBe('quota_exhausted');
// A provider's own rate limit is not the Cloud allowance.
expect(failureOutcome(new LlmRequestError('x', 429, 'rate_limit_error'))).toBe('provider_error');
expect(failureOutcome(new LlmRequestError('x', 429, undefined))).toBe('provider_error');
expect(failureOutcome(new LlmContextOverflowError('prompt is too long'))).toBe('too_large');
expect(failureOutcome(new ImageTooLargeError('too many pixels'))).toBe('too_large');
expect(failureOutcome('boom')).toBe('provider_error');
});
});
@@ -0,0 +1,41 @@
import {
errorBodyCode,
llmRequestError,
LlmRequestError,
} from '../../../../src/nest/llm-parse/clients/llm-request-error';
import { describe, expect, it } from 'vitest';
describe('llm-request-error', () => {
it('LLM-REQERR-001: reads the code at the top, as the TREK Cloud gate sends it', () => {
const body = JSON.stringify({ error: 'The document analyses for this period are used up.', code: 'QUOTA_SPENT' });
expect(errorBodyCode(body)).toBe('QUOTA_SPENT');
});
it('LLM-REQERR-002: reads the code inside error, as OpenAI and Anthropic send it', () => {
expect(errorBodyCode(JSON.stringify({ error: { type: 'rate_limit_error', code: 'rate_limited' } }))).toBe(
'rate_limited',
);
expect(errorBodyCode(JSON.stringify({ error: { message: 'no code here' } }))).toBeUndefined();
});
it('LLM-REQERR-003: a body that is not JSON, not an object, empty or oversized names no code', () => {
expect(errorBodyCode('upstream down')).toBeUndefined();
expect(errorBodyCode('"QUOTA_SPENT"')).toBeUndefined();
expect(errorBodyCode('null')).toBeUndefined();
expect(errorBodyCode('')).toBeUndefined();
expect(errorBodyCode(JSON.stringify({ code: 'QUOTA_SPENT', pad: 'x'.repeat(5000) }))).toBeUndefined();
});
it('LLM-REQERR-004: the error keeps the message the import has always shown, cut at 300 characters', () => {
const detail = JSON.stringify({ error: 'over the ceiling', code: 'HARD_CAP' });
const err = llmRequestError('Anthropic request failed', 429, detail);
expect(err).toBeInstanceOf(LlmRequestError);
expect(err).toBeInstanceOf(Error);
expect(err.message).toBe(`Anthropic request failed (429): ${detail}`);
expect(err).toMatchObject({ status: 429, code: 'HARD_CAP', name: 'LlmRequestError' });
expect(llmRequestError('LLM request failed', 500, 'y'.repeat(400)).message).toHaveLength(
'LLM request failed (500): '.length + 300,
);
});
});
@@ -381,11 +381,36 @@ export type BookingImportPreviewItem = z.infer<typeof bookingImportPreviewItemSc
export const bookingImportModeSchema = z.enum(['no-ai', 'fallback-on-empty', 'force-ai']);
export type BookingImportMode = z.infer<typeof bookingImportModeSchema>;
/**
* How the AI reading of one document ended:
* - `ok` it found at least one booking
* - `empty` it read the document and found none
* - `no_text` nothing to read: a scan, and no model that reads images
* - `quota_exhausted` the TREK Cloud allowance for the period is used up (`QUOTA_SPENT`, `HARD_CAP`)
* - `ai_disabled` no AI reading is set up for this person
* - `user_cap` the person's own daily limit is reached
* - `provider_error` the provider refused or failed, or could not take this input
* - `too_large` the document is more than the model or the gate takes
*/
export const llmParseOutcomeSchema = z.enum([
'ok',
'empty',
'no_text',
'quota_exhausted',
'ai_disabled',
'user_cap',
'provider_error',
'too_large',
]);
export type LlmParseOutcome = z.infer<typeof llmParseOutcomeSchema>;
/** Per-file AI report so the preview UI can offer "Try AI parsing" only where it applies. */
export const bookingImportFileReportSchema = z.object({
fileName: z.string(),
aiAvailable: z.boolean(),
aiUsed: z.boolean(),
/** How the AI reading ended; only where it ran. */
outcome: llmParseOutcomeSchema.optional(),
});
export type BookingImportFileReport = z.infer<typeof bookingImportFileReportSchema>;