mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 19:40:34 +00:00
fix(llm-parse): the local router reports how its reading ended
The Ollama router catches a failed model call and answered an empty list, which the outcome read as a document without bookings. It now returns the outcome with the list; failureOutcome and foundOutcome move to llm-outcome.ts so the router and the service share them without importing each other.
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
import { LlmContextOverflowError } from './clients/context-overflow';
|
||||
import { LlmRequestError } from './clients/llm-request-error';
|
||||
import { ImageTooLargeError } from './image-input';
|
||||
import type { LlmParseOutcome } from '@trek/shared';
|
||||
|
||||
/** The codes the TREK Cloud gate answers a spent allowance with, on a 429. */
|
||||
const QUOTA_CODES = new Set(['QUOTA_SPENT', 'HARD_CAP']);
|
||||
|
||||
/**
|
||||
* How a failed reading ended, from the error alone: a spent Cloud allowance,
|
||||
* a document too large for the model or the gate, or a provider that failed.
|
||||
*/
|
||||
export function failureOutcome(err: unknown): LlmParseOutcome {
|
||||
if (err instanceof LlmRequestError) {
|
||||
if (err.status === 429 && err.code !== undefined && QUOTA_CODES.has(err.code)) return 'quota_exhausted';
|
||||
if (err.status === 413) return 'too_large';
|
||||
}
|
||||
if (err instanceof LlmContextOverflowError || err instanceof ImageTooLargeError) return 'too_large';
|
||||
return 'provider_error';
|
||||
}
|
||||
|
||||
/** A reading that found bookings or did not. */
|
||||
export function foundOutcome(items: readonly unknown[]): LlmParseOutcome {
|
||||
return items.length > 0 ? 'ok' : 'empty';
|
||||
}
|
||||
@@ -2,13 +2,13 @@ import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import type { KiReservation } from '../booking-import/kitinerary.types';
|
||||
import { collapseRepeatedBookingTotal } from './booking-total';
|
||||
import { LlmContextOverflowError } from './clients/context-overflow';
|
||||
import { LlmRequestError } from './clients/llm-request-error';
|
||||
import { toIsoCurrency } from './currency-code';
|
||||
import { capImage, ImageTooLargeError, imageMimeType, renderPdfPages } from './image-input';
|
||||
import { capImage, imageMimeType, renderPdfPages } from './image-input';
|
||||
import { createLlmClient } from './llm-client.factory';
|
||||
import type { ResolvedLlmConfig } from './llm-config';
|
||||
import { LlmConfigResolver } from './llm-config.resolver';
|
||||
import { LlmLocalService } from './llm-local.service';
|
||||
import { failureOutcome, foundOutcome } from './llm-outcome';
|
||||
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt';
|
||||
import type { LlmExtractionFile, LlmExtractionInput } from './llm-provider.interface';
|
||||
import {
|
||||
@@ -48,26 +48,6 @@ export interface LlmParseResult {
|
||||
outcome: LlmParseOutcome;
|
||||
}
|
||||
|
||||
/** The codes the TREK Cloud gate answers a spent allowance with, on a 429. */
|
||||
const QUOTA_CODES = new Set(['QUOTA_SPENT', 'HARD_CAP']);
|
||||
|
||||
/**
|
||||
* How a failed reading ended, from the error alone: a spent Cloud allowance,
|
||||
* a document too large for the model or the gate, or a provider that failed.
|
||||
*/
|
||||
export function failureOutcome(err: unknown): LlmParseOutcome {
|
||||
if (err instanceof LlmRequestError) {
|
||||
if (err.status === 429 && err.code !== undefined && QUOTA_CODES.has(err.code)) return 'quota_exhausted';
|
||||
if (err.status === 413) return 'too_large';
|
||||
}
|
||||
if (err instanceof LlmContextOverflowError || err instanceof ImageTooLargeError) return 'too_large';
|
||||
return 'provider_error';
|
||||
}
|
||||
|
||||
/** A reading that found bookings or did not. */
|
||||
function foundOutcome(items: readonly unknown[]): LlmParseOutcome {
|
||||
return items.length > 0 ? 'ok' : 'empty';
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrates the LLM fallback: resolve config → pick client → build input
|
||||
@@ -293,7 +273,7 @@ export class LlmParseService {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: could not read file — ${err instanceof Error ? err.message : String(err)}`],
|
||||
outcome: err instanceof ImageTooLargeError ? 'too_large' : 'provider_error',
|
||||
outcome: failureOutcome(err),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -314,11 +294,7 @@ export class LlmParseService {
|
||||
ctx,
|
||||
)
|
||||
: await routeExtraction(input.text ?? '', ctx);
|
||||
return {
|
||||
kiItems: routed.kiItems,
|
||||
warnings: [...warnings, ...routed.warnings],
|
||||
outcome: foundOutcome(routed.kiItems),
|
||||
};
|
||||
return { kiItems: routed.kiItems, warnings: [...warnings, ...routed.warnings], outcome: routed.outcome };
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`,
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
import type { KiReservation } from '../../booking-import/kitinerary.types';
|
||||
import { nuExtractToKiReservations } from '../clients/nuextract';
|
||||
import { parseLenientJson } from '../lenient-json';
|
||||
import { failureOutcome, foundOutcome } from '../llm-outcome';
|
||||
import {
|
||||
FLAT_SCHEMA_BY_TYPE,
|
||||
FLAT_TYPES,
|
||||
@@ -29,7 +30,7 @@ import {
|
||||
type FlatLike,
|
||||
} from './flat-schemas';
|
||||
import { extractEnforced } from './ollama-format.client';
|
||||
import { normalizeLocalDateTime, parseMeridiemClock } from '@trek/shared';
|
||||
import { normalizeLocalDateTime, parseMeridiemClock, type LlmParseOutcome } from '@trek/shared';
|
||||
|
||||
export interface RouterContext {
|
||||
baseUrl: string;
|
||||
@@ -319,6 +320,13 @@ function fillBookingWideFields(flats: Record<string, unknown>[], text: string):
|
||||
});
|
||||
}
|
||||
|
||||
/** What the router read, and how the reading ended (a failed model call is `[]` with its outcome). */
|
||||
interface RoutedExtraction {
|
||||
kiItems: KiReservation[];
|
||||
warnings: string[];
|
||||
outcome: LlmParseOutcome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the router on extracted document text and return schema.org KiReservation nodes.
|
||||
* Returns `[]` (never throws for content reasons) so the caller degrades gracefully.
|
||||
@@ -326,7 +334,7 @@ function fillBookingWideFields(flats: Record<string, unknown>[], text: string):
|
||||
export async function routeExtraction(
|
||||
text: string,
|
||||
ctx: RouterContext,
|
||||
): Promise<{ kiItems: KiReservation[]; warnings: string[] }> {
|
||||
): Promise<RoutedExtraction> {
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Schicht 1 — exactly one model call.
|
||||
@@ -341,7 +349,11 @@ export async function routeExtraction(
|
||||
const looksLikeFlight = detectFlightNumbers(text).length > 0 && !(named && TRANSPORT_FLAT_TYPES.has(named));
|
||||
flats = looksLikeFlight ? await extractFlights(text, ctx) : [await extractSingle(text, ctx)];
|
||||
} catch (err) {
|
||||
return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`],
|
||||
outcome: failureOutcome(err),
|
||||
};
|
||||
}
|
||||
|
||||
// Schicht 2 — deterministic booking-wide fields the per-call schema doesn't carry.
|
||||
@@ -350,7 +362,7 @@ export async function routeExtraction(
|
||||
const kiItems = nuExtractToKiReservations(
|
||||
flats as unknown as Record<string, unknown>[],
|
||||
) as unknown as KiReservation[];
|
||||
return { kiItems, warnings };
|
||||
return { kiItems, warnings, outcome: foundOutcome(kiItems) };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -362,18 +374,22 @@ export async function routeExtraction(
|
||||
export async function routeImageExtraction(
|
||||
images: Buffer[],
|
||||
ctx: RouterContext,
|
||||
): Promise<{ kiItems: KiReservation[]; warnings: string[] }> {
|
||||
): Promise<RoutedExtraction> {
|
||||
let flat: FlatLike;
|
||||
try {
|
||||
flat = unwrapPrice(await extractSingle('', ctx, images));
|
||||
} catch (err) {
|
||||
return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`],
|
||||
outcome: failureOutcome(err),
|
||||
};
|
||||
}
|
||||
const kiItems = nuExtractToKiReservations([flat] as unknown as Record<
|
||||
string,
|
||||
unknown
|
||||
>[]) as unknown as KiReservation[];
|
||||
return { kiItems, warnings: [] };
|
||||
return { kiItems, warnings: [], outcome: foundOutcome(kiItems) };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -217,6 +217,7 @@ describe('routeExtraction', () => {
|
||||
expect(res.kiItems).toEqual([]);
|
||||
expect(res.warnings[0]).toContain('AI parsing failed');
|
||||
expect(res.warnings[0]).toContain('connection refused');
|
||||
expect(res.outcome).toBe('provider_error');
|
||||
});
|
||||
|
||||
// #2076 — the lodging keywords used to be tested second and included
|
||||
@@ -330,7 +331,7 @@ describe('routeImageExtraction', () => {
|
||||
currency: 'EUR',
|
||||
});
|
||||
const out = await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [] });
|
||||
expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [], outcome: 'ok' });
|
||||
expect(extractEnforced).toHaveBeenCalledTimes(1);
|
||||
const call = extractEnforced.mock.calls[0][0];
|
||||
expect(call.images).toEqual([Buffer.from('photo').toString('base64')]);
|
||||
|
||||
@@ -4,7 +4,8 @@ import { LlmRequestError } from '../../../../src/nest/llm-parse/clients/llm-requ
|
||||
import { ImageTooLargeError } from '../../../../src/nest/llm-parse/image-input';
|
||||
import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver';
|
||||
import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
|
||||
import { failureOutcome, LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
import { failureOutcome } from '../../../../src/nest/llm-parse/llm-outcome';
|
||||
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
@@ -77,8 +78,8 @@ beforeEach(() => {
|
||||
extract.mockResolvedValue([{ '@type': 'FlightReservation' }]);
|
||||
extractText.mockResolvedValue('Flight AB123');
|
||||
detectFlightNumbers.mockReturnValue([]);
|
||||
routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [] });
|
||||
routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [] });
|
||||
routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [], outcome: 'ok' });
|
||||
routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [], outcome: 'ok' });
|
||||
modelCapabilities.mockResolvedValue(null);
|
||||
renderPdfPages.mockResolvedValue([]);
|
||||
});
|
||||
@@ -709,8 +710,11 @@ describe('LlmParseService.parse outcome', () => {
|
||||
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty');
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local' }));
|
||||
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('ok');
|
||||
routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: [] });
|
||||
// The router's own outcome is passed on, a failed model call inside it included.
|
||||
routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: [], outcome: 'empty' });
|
||||
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('empty');
|
||||
routeExtraction.mockResolvedValueOnce({ kiItems: [], warnings: ['failed'], outcome: 'provider_error' });
|
||||
expect((await svc().parse(file('a.txt'), 1)).outcome).toBe('provider_error');
|
||||
});
|
||||
|
||||
it('LLM-OUTCOME-002: no AI set up, a scan nothing can read, and a photo the model cannot see', async () => {
|
||||
|
||||
Reference in New Issue
Block a user