mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
fix(import): a booking the AI reads without its venue is asked for one, and a failed import does not look like a success
Gemini behind the OpenAI-compatible provider decodes against the JSON schema TREK sends and fills declared properties only. reservationFor was an open object without any, so the model left it out: a Booking.com stay came back with its dates and price but no hotel name, and the mapper had to drop it (#2477). It is the same cause as #1638, whose fix only made the JSON readable. The schema now declares every field the prompt names inside reservationFor, for venues, flights, trains, buses, boats, events and rental cars, at most two levels deep, and requires it; the car desks at the root get the same venue fields. Every object stays open to keys the prompt does not name. On the cloud AI path an answer is cleaned up before it is mapped. Exact duplicate nodes are dropped, so one stay is one booking. priceCurrency is read as an ISO 4217 code ("EURials" is EUR, a lone symbol maps to its code); when the model names none, a priced node takes the currency of the document's total, and otherwise the field goes instead of saving a code nobody can convert. The kitinerary path is untouched. The import upload decodes file names as UTF-8 like every other upload route. "Bestätigung.pdf" reached the warnings as mojibake, and the review could not attach the file to the booking it created, because it matches on the name. The OpenAI-compatible client logs which response_format the answering attempt carried, metadata only, so the next report shows whether the schema reached the model at all. The background import card showed a green tick beside "no reservations could be extracted". A parse that found nothing now wears the warning mark instead; it is the same card on the desktop and on the phone.
This commit is contained in:
@@ -44,6 +44,12 @@ function rejectsTemperature(detail: string): boolean {
|
||||
&& /unsupported|not supported|does not support|only the default/i.test(detail);
|
||||
}
|
||||
|
||||
/** The `response_format` the given attempt carried, by its type. */
|
||||
function sentFormat(shape: RequestShape, nuextract: boolean): string {
|
||||
if (nuextract || shape.noResponseFormat) return 'none';
|
||||
return shape.jsonObject ? 'json_object' : 'json_schema';
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenAI-compatible chat-completions client. Covers both the "openai" cloud
|
||||
* provider and the "local" provider (Ollama / vLLM / llama.cpp / LM Studio),
|
||||
@@ -170,6 +176,10 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
if (!res.ok) {
|
||||
throw new Error(`LLM request failed (${res.status}): ${detail.slice(0, 300)}`);
|
||||
}
|
||||
// Which rung answered. A report could not tell whether the schema reached the
|
||||
// model at all or the ladder had fallen back to the prompt alone (#2477).
|
||||
// Metadata only, so it is the same on a managed install.
|
||||
console.debug(`[DEBUG] LLM answered with response_format=${sentFormat(shape, nuextract)}`);
|
||||
|
||||
const data = (await res.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* The ISO 4217 codes this runtime knows. Node ships full ICU, so this is the
|
||||
* complete list; a build without `Intl.supportedValuesOf` leaves it empty and
|
||||
* `toIsoCurrency` then settles for the shape of a code.
|
||||
*/
|
||||
const ISO_CURRENCIES: ReadonlySet<string> = new Set(
|
||||
typeof Intl.supportedValuesOf === 'function' ? Intl.supportedValuesOf('currency') : [],
|
||||
);
|
||||
|
||||
/** A symbol standing on its own. `$` alone is read as USD, as the extractor does. */
|
||||
const SYMBOLS: ReadonlyMap<string, string> = new Map([
|
||||
['€', 'EUR'],
|
||||
['£', 'GBP'],
|
||||
['¥', 'JPY'],
|
||||
['$', 'USD'],
|
||||
]);
|
||||
|
||||
/**
|
||||
* The currency an AI answer named, as an ISO 4217 code, or undefined when it
|
||||
* names none.
|
||||
*
|
||||
* A model's `priceCurrency` is whatever it made of the document: `EUR`, `eur`,
|
||||
* `€`, `Euros`, and once `EURials` (#2477). That value used to reach the cost
|
||||
* form untouched. It is read as an exact code in any case first, then as the
|
||||
* code it starts with when those three letters are a real code (`EURials` and
|
||||
* `Euros` are EUR, `LEK` is nothing, since the lek is ALL), then as a lone
|
||||
* currency symbol. Anything else is undefined, so the caller can fall back to
|
||||
* the document or drop the field instead of passing on a currency nobody can
|
||||
* convert.
|
||||
*/
|
||||
export function toIsoCurrency(raw: unknown, known: ReadonlySet<string> = ISO_CURRENCIES): string | undefined {
|
||||
if (typeof raw !== 'string') return undefined;
|
||||
const value = raw.trim().toUpperCase();
|
||||
if (!value) return undefined;
|
||||
const isCode = (code: string) => (known.size > 0 ? known.has(code) : /^[A-Z]{3}$/.test(code));
|
||||
if (isCode(value)) return value;
|
||||
const prefix = /^[A-Z]{3}/.exec(value)?.[0];
|
||||
if (prefix && isCode(prefix)) return prefix;
|
||||
return SYMBOLS.get(value);
|
||||
}
|
||||
@@ -4,7 +4,8 @@ import { LlmConfigResolver } from './llm-config.resolver';
|
||||
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt';
|
||||
import type { LlmExtractionInput } from './llm-provider.interface';
|
||||
import { isPdf, extractText } from './text-extract';
|
||||
import { routeExtraction, detectFlightNumbers } from './router/extraction-router';
|
||||
import { routeExtraction, detectFlightNumbers, extractTotalPrice } from './router/extraction-router';
|
||||
import { toIsoCurrency } from './currency-code';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { kiReservationSchema } from '@trek/shared';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
@@ -130,10 +131,21 @@ export class LlmParseService {
|
||||
}
|
||||
|
||||
const kiItems: KiReservation[] = [];
|
||||
// A model can list one stay twice, and did (#1638, #2477): two identical
|
||||
// nodes are one booking, not two hotels. Only exact copies, and only on this
|
||||
// path; the kitinerary extractor reports what the document holds.
|
||||
const seen = new Set<string>();
|
||||
for (const node of raw) {
|
||||
const result = kiReservationSchema.safeParse(node);
|
||||
if (result.success) kiItems.push(normalizeNode(result.data) as unknown as KiReservation);
|
||||
else warnings.push(`${file.originalName}: skipped an unrecognized AI result`);
|
||||
if (!result.success) {
|
||||
warnings.push(`${file.originalName}: skipped an unrecognized AI result`);
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeCurrency(normalizeNode(result.data), input.text);
|
||||
const key = stableKey(normalized);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
kiItems.push(normalized as unknown as KiReservation);
|
||||
}
|
||||
|
||||
return { kiItems, warnings };
|
||||
@@ -182,3 +194,31 @@ function normalizeNode(node: Record<string, unknown>): Record<string, unknown> {
|
||||
out.reservationFor = reservationFor;
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* The node with its `priceCurrency` as an ISO 4217 code, or without one.
|
||||
*
|
||||
* What the model wrote wins when it reads as a code. Otherwise, and only when
|
||||
* the node carries a price, the currency comes from the document's own total
|
||||
* (a Booking.com print shows `€ 89,35`). When neither yields a code the field
|
||||
* is dropped: an empty currency lets the cost form pick one, a made up one
|
||||
* (`EURials`, #2477) is saved and cannot be converted.
|
||||
*/
|
||||
function normalizeCurrency(node: Record<string, unknown>, documentText: string | undefined): Record<string, unknown> {
|
||||
const hasPrice = node.price != null && node.price !== '';
|
||||
if (node.priceCurrency == null && !hasPrice) return node;
|
||||
const code =
|
||||
toIsoCurrency(node.priceCurrency) ??
|
||||
(hasPrice && documentText ? toIsoCurrency(extractTotalPrice(documentText)?.currency) : undefined);
|
||||
const { priceCurrency: _dropped, ...rest } = node;
|
||||
return code ? { ...rest, priceCurrency: code } : rest;
|
||||
}
|
||||
|
||||
/** JSON with every object's keys sorted, so two equal nodes give one string. */
|
||||
function stableKey(value: unknown): string {
|
||||
return JSON.stringify(value, (_key, v: unknown) =>
|
||||
v && typeof v === 'object' && !Array.isArray(v)
|
||||
? Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b)))
|
||||
: v,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -35,6 +35,11 @@ const MAX_FILES = 5;
|
||||
const UPLOAD = {
|
||||
storage: memoryStorage(),
|
||||
limits: { fileSize: MAX_FILE_BYTES, files: MAX_FILES },
|
||||
// Like every other upload route (storage-upload.factory.ts). Multer reads the
|
||||
// name as latin1 otherwise, so "Bestätigung.pdf" came back as "Bestätigung.pdf"
|
||||
// in the warnings, and the review could not find the file to attach it to the
|
||||
// booking, because the client matches on the name it sent (#2477).
|
||||
defParamCharset: 'utf8' as const,
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user