From 53ebc9d632061be3a41abe0aa54f7430fc1b67be Mon Sep 17 00:00:00 2001 From: Maurice Date: Fri, 13 Nov 2026 15:11:53 +0100 Subject: [PATCH] fix(llm-parse): the local router reports how its reading ended The Ollama router catches a failed model call and answered an empty list, which the outcome read as a document without bookings. It now returns the outcome with the list; failureOutcome and foundOutcome move to llm-outcome.ts so the router and the service share them without importing each other. --- server/src/nest/llm-parse/llm-outcome.ts | 25 +++++++++++++++ .../src/nest/llm-parse/llm-parse.service.ts | 32 +++---------------- .../llm-parse/router/extraction-router.ts | 30 +++++++++++++---- .../nest/llm-parse/extraction-router.test.ts | 3 +- .../nest/llm-parse/llm-parse.service.test.ts | 12 ++++--- 5 files changed, 62 insertions(+), 40 deletions(-) create mode 100644 server/src/nest/llm-parse/llm-outcome.ts diff --git a/server/src/nest/llm-parse/llm-outcome.ts b/server/src/nest/llm-parse/llm-outcome.ts new file mode 100644 index 000000000..aaa003a76 --- /dev/null +++ b/server/src/nest/llm-parse/llm-outcome.ts @@ -0,0 +1,25 @@ +import { LlmContextOverflowError } from './clients/context-overflow'; +import { LlmRequestError } from './clients/llm-request-error'; +import { ImageTooLargeError } from './image-input'; +import type { LlmParseOutcome } from '@trek/shared'; + +/** The codes the TREK Cloud gate answers a spent allowance with, on a 429. */ +const QUOTA_CODES = new Set(['QUOTA_SPENT', 'HARD_CAP']); + +/** + * How a failed reading ended, from the error alone: a spent Cloud allowance, + * a document too large for the model or the gate, or a provider that failed. + */ +export function failureOutcome(err: unknown): LlmParseOutcome { + if (err instanceof LlmRequestError) { + if (err.status === 429 && err.code !== undefined && QUOTA_CODES.has(err.code)) return 'quota_exhausted'; + if (err.status === 413) return 'too_large'; + } + if (err instanceof LlmContextOverflowError || err instanceof ImageTooLargeError) return 'too_large'; + return 'provider_error'; +} + +/** A reading that found bookings or did not. */ +export function foundOutcome(items: readonly unknown[]): LlmParseOutcome { + return items.length > 0 ? 'ok' : 'empty'; +} diff --git a/server/src/nest/llm-parse/llm-parse.service.ts b/server/src/nest/llm-parse/llm-parse.service.ts index e0df9f349..005088e9d 100644 --- a/server/src/nest/llm-parse/llm-parse.service.ts +++ b/server/src/nest/llm-parse/llm-parse.service.ts @@ -2,13 +2,13 @@ import { RuntimeEnvService } from '../app-config/runtime-env.service'; import type { KiReservation } from '../booking-import/kitinerary.types'; import { collapseRepeatedBookingTotal } from './booking-total'; import { LlmContextOverflowError } from './clients/context-overflow'; -import { LlmRequestError } from './clients/llm-request-error'; import { toIsoCurrency } from './currency-code'; -import { capImage, ImageTooLargeError, imageMimeType, renderPdfPages } from './image-input'; +import { capImage, imageMimeType, renderPdfPages } from './image-input'; import { createLlmClient } from './llm-client.factory'; import type { ResolvedLlmConfig } from './llm-config'; import { LlmConfigResolver } from './llm-config.resolver'; import { LlmLocalService } from './llm-local.service'; +import { failureOutcome, foundOutcome } from './llm-outcome'; import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt'; import type { LlmExtractionFile, LlmExtractionInput } from './llm-provider.interface'; import { @@ -48,26 +48,6 @@ export interface LlmParseResult { outcome: LlmParseOutcome; } -/** The codes the TREK Cloud gate answers a spent allowance with, on a 429. */ -const QUOTA_CODES = new Set(['QUOTA_SPENT', 'HARD_CAP']); - -/** - * How a failed reading ended, from the error alone: a spent Cloud allowance, - * a document too large for the model or the gate, or a provider that failed. - */ -export function failureOutcome(err: unknown): LlmParseOutcome { - if (err instanceof LlmRequestError) { - if (err.status === 429 && err.code !== undefined && QUOTA_CODES.has(err.code)) return 'quota_exhausted'; - if (err.status === 413) return 'too_large'; - } - if (err instanceof LlmContextOverflowError || err instanceof ImageTooLargeError) return 'too_large'; - return 'provider_error'; -} - -/** A reading that found bookings or did not. */ -function foundOutcome(items: readonly unknown[]): LlmParseOutcome { - return items.length > 0 ? 'ok' : 'empty'; -} /** * Orchestrates the LLM fallback: resolve config → pick client → build input @@ -293,7 +273,7 @@ export class LlmParseService { return { kiItems: [], warnings: [`${file.originalName}: could not read file — ${err instanceof Error ? err.message : String(err)}`], - outcome: err instanceof ImageTooLargeError ? 'too_large' : 'provider_error', + outcome: failureOutcome(err), }; } @@ -314,11 +294,7 @@ export class LlmParseService { ctx, ) : await routeExtraction(input.text ?? '', ctx); - return { - kiItems: routed.kiItems, - warnings: [...warnings, ...routed.warnings], - outcome: foundOutcome(routed.kiItems), - }; + return { kiItems: routed.kiItems, warnings: [...warnings, ...routed.warnings], outcome: routed.outcome }; } catch (err) { console.error( `[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`, diff --git a/server/src/nest/llm-parse/router/extraction-router.ts b/server/src/nest/llm-parse/router/extraction-router.ts index f8f65f401..f4bffee3c 100644 --- a/server/src/nest/llm-parse/router/extraction-router.ts +++ b/server/src/nest/llm-parse/router/extraction-router.ts @@ -20,6 +20,7 @@ import type { KiReservation } from '../../booking-import/kitinerary.types'; import { nuExtractToKiReservations } from '../clients/nuextract'; import { parseLenientJson } from '../lenient-json'; +import { failureOutcome, foundOutcome } from '../llm-outcome'; import { FLAT_SCHEMA_BY_TYPE, FLAT_TYPES, @@ -29,7 +30,7 @@ import { type FlatLike, } from './flat-schemas'; import { extractEnforced } from './ollama-format.client'; -import { normalizeLocalDateTime, parseMeridiemClock } from '@trek/shared'; +import { normalizeLocalDateTime, parseMeridiemClock, type LlmParseOutcome } from '@trek/shared'; export interface RouterContext { baseUrl: string; @@ -319,6 +320,13 @@ function fillBookingWideFields(flats: Record[], text: string): }); } +/** What the router read, and how the reading ended (a failed model call is `[]` with its outcome). */ +interface RoutedExtraction { + kiItems: KiReservation[]; + warnings: string[]; + outcome: LlmParseOutcome; +} + /** * Run the router on extracted document text and return schema.org KiReservation nodes. * Returns `[]` (never throws for content reasons) so the caller degrades gracefully. @@ -326,7 +334,7 @@ function fillBookingWideFields(flats: Record[], text: string): export async function routeExtraction( text: string, ctx: RouterContext, -): Promise<{ kiItems: KiReservation[]; warnings: string[] }> { +): Promise { const warnings: string[] = []; // Schicht 1 — exactly one model call. @@ -341,7 +349,11 @@ export async function routeExtraction( const looksLikeFlight = detectFlightNumbers(text).length > 0 && !(named && TRANSPORT_FLAT_TYPES.has(named)); flats = looksLikeFlight ? await extractFlights(text, ctx) : [await extractSingle(text, ctx)]; } catch (err) { - return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] }; + return { + kiItems: [], + warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`], + outcome: failureOutcome(err), + }; } // Schicht 2 — deterministic booking-wide fields the per-call schema doesn't carry. @@ -350,7 +362,7 @@ export async function routeExtraction( const kiItems = nuExtractToKiReservations( flats as unknown as Record[], ) as unknown as KiReservation[]; - return { kiItems, warnings }; + return { kiItems, warnings, outcome: foundOutcome(kiItems) }; } /** @@ -362,18 +374,22 @@ export async function routeExtraction( export async function routeImageExtraction( images: Buffer[], ctx: RouterContext, -): Promise<{ kiItems: KiReservation[]; warnings: string[] }> { +): Promise { let flat: FlatLike; try { flat = unwrapPrice(await extractSingle('', ctx, images)); } catch (err) { - return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] }; + return { + kiItems: [], + warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`], + outcome: failureOutcome(err), + }; } const kiItems = nuExtractToKiReservations([flat] as unknown as Record< string, unknown >[]) as unknown as KiReservation[]; - return { kiItems, warnings: [] }; + return { kiItems, warnings: [], outcome: foundOutcome(kiItems) }; } /** diff --git a/server/tests/unit/nest/llm-parse/extraction-router.test.ts b/server/tests/unit/nest/llm-parse/extraction-router.test.ts index 6cb3ec773..d0fd625c9 100644 --- a/server/tests/unit/nest/llm-parse/extraction-router.test.ts +++ b/server/tests/unit/nest/llm-parse/extraction-router.test.ts @@ -217,6 +217,7 @@ describe('routeExtraction', () => { expect(res.kiItems).toEqual([]); expect(res.warnings[0]).toContain('AI parsing failed'); expect(res.warnings[0]).toContain('connection refused'); + expect(res.outcome).toBe('provider_error'); }); // #2076 — the lodging keywords used to be tested second and included @@ -330,7 +331,7 @@ describe('routeImageExtraction', () => { currency: 'EUR', }); const out = await routeImageExtraction([Buffer.from('photo')], CTX); - expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [] }); + expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [], outcome: 'ok' }); expect(extractEnforced).toHaveBeenCalledTimes(1); const call = extractEnforced.mock.calls[0][0]; expect(call.images).toEqual([Buffer.from('photo').toString('base64')]); diff --git a/server/tests/unit/nest/llm-parse/llm-parse.service.test.ts b/server/tests/unit/nest/llm-parse/llm-parse.service.test.ts index ef90b09f1..9756fb350 100644 --- a/server/tests/unit/nest/llm-parse/llm-parse.service.test.ts +++ b/server/tests/unit/nest/llm-parse/llm-parse.service.test.ts @@ -4,7 +4,8 @@ import { LlmRequestError } from '../../../../src/nest/llm-parse/clients/llm-requ import { ImageTooLargeError } from '../../../../src/nest/llm-parse/image-input'; import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver'; import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service'; -import { failureOutcome, LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service'; +import { failureOutcome } from '../../../../src/nest/llm-parse/llm-outcome'; +import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service'; import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -77,8 +78,8 @@ beforeEach(() => { extract.mockResolvedValue([{ '@type': 'FlightReservation' }]); extractText.mockResolvedValue('Flight AB123'); detectFlightNumbers.mockReturnValue([]); - routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [] }); - routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [] }); + routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [], outcome: 'ok' }); + routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [], outcome: 'ok' }); modelCapabilities.mockResolvedValue(null); renderPdfPages.mockResolvedValue([]); }); @@ -709,8 +710,11 @@ describe('LlmParseService.parse outcome', () => { expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty'); resolveLlmConfig.mockReturnValue(cfg({ provider: 'local' })); expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('ok'); - routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: [] }); + // The router's own outcome is passed on, a failed model call inside it included. + routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: [], outcome: 'empty' }); expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty'); + routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: ['failed'], outcome: 'provider_error' }); + expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('provider_error'); }); it('LLM-OUTCOME-002: no AI set up, a scan nothing can read, and a photo the model cannot see', async () => {