mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
feat(extract): extract data using LLM
This commit is contained in:
+3
-15
@@ -46,23 +46,11 @@ COPY package.json package-lock.json ./
|
||||
COPY shared/package.json ./shared/
|
||||
COPY server/package.json ./server/
|
||||
|
||||
# better-sqlite3 native addon requires build tools (purged after compile).
|
||||
# kitinerary-extractor for booking-confirmation import:
|
||||
# amd64 — static binary from KDE CDN (glibc 2.17+; wget stays for healthcheck)
|
||||
# arm64 — apt package (KDE publishes no arm64 static binary)
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends tzdata dumb-init wget ca-certificates python3 build-essential && \
|
||||
apt-get install -y --no-install-recommends tzdata dumb-init wget ca-certificates python3 build-essential \
|
||||
libkitinerary-bin && \
|
||||
npm ci --workspace=server --omit=dev && \
|
||||
ARCH=$(dpkg --print-architecture) && \
|
||||
if [ "$ARCH" = "amd64" ]; then \
|
||||
wget -qO /tmp/ki.tgz https://cdn.kde.org/ci-builds/pim/kitinerary/release-26.04/linux/kitinerary-extractor-x86_64-26.04.2.tgz && \
|
||||
echo "ba5cfb4a2353157c8f54cbeaea0097c5bf2c3a810e0342f63d6e524826176628 /tmp/ki.tgz" | sha256sum -c && \
|
||||
tar -xz -C /usr/local -f /tmp/ki.tgz bin/kitinerary-extractor share/locale && \
|
||||
rm /tmp/ki.tgz; \
|
||||
else \
|
||||
apt-get install -y --no-install-recommends libkitinerary-bin && \
|
||||
ln -sf "$(find /usr/lib -name kitinerary-extractor -type f | head -1)" /usr/local/bin/kitinerary-extractor; \
|
||||
fi && \
|
||||
ln -sf "$(find /usr/lib -name kitinerary-extractor -type f | head -1)" /usr/local/bin/kitinerary-extractor; \
|
||||
apt-get purge -y python3 build-essential && \
|
||||
apt-get autoremove -y && \
|
||||
rm -rf /var/lib/apt/lists/* /usr/local/lib/node_modules/npm /usr/local/bin/npm /usr/local/bin/npx
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
type BookingImportPreviewItem,
|
||||
type BookingImportPreviewResponse,
|
||||
type BookingImportConfirmResponse,
|
||||
type BookingImportMode,
|
||||
} from '@trek/shared'
|
||||
import { getSocketId } from './websocket'
|
||||
import { isReachable, probeNow } from '../sync/connectivity'
|
||||
@@ -442,6 +443,41 @@ export const adminApi = {
|
||||
updateOidc: (data: Record<string, unknown>) => apiClient.put('/admin/oidc', data).then(r => r.data),
|
||||
addons: () => apiClient.get('/admin/addons').then(r => r.data),
|
||||
updateAddon: (id: number | string, data: Record<string, unknown>) => apiClient.put(`/admin/addons/${id}`, data).then(r => r.data),
|
||||
// Local LLM (Ollama) management for the AI-parsing addon.
|
||||
llmLocalModels: (baseUrl: string): Promise<{ models: { name: string; size: number }[] }> =>
|
||||
apiClient.get('/admin/llm/local/models', { params: { baseUrl } }).then(r => r.data),
|
||||
/** Pull a model, streaming Ollama's NDJSON progress to `onProgress`. */
|
||||
llmLocalPull: async (
|
||||
baseUrl: string,
|
||||
model: string,
|
||||
onProgress: (p: { status?: string; total?: number; completed?: number; error?: string }) => void,
|
||||
): Promise<void> => {
|
||||
const res = await fetch('/api/admin/llm/local/pull', {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ baseUrl, model }),
|
||||
})
|
||||
if (!res.ok || !res.body) {
|
||||
let msg = `Pull failed (${res.status})`
|
||||
try { msg = (await res.json())?.error ?? msg } catch { /* non-json */ }
|
||||
throw new Error(msg)
|
||||
}
|
||||
const reader = res.body.getReader()
|
||||
const dec = new TextDecoder()
|
||||
let buf = ''
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) break
|
||||
buf += dec.decode(value, { stream: true })
|
||||
const lines = buf.split('\n')
|
||||
buf = lines.pop() ?? ''
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
try { onProgress(JSON.parse(line)) } catch { /* skip partial */ }
|
||||
}
|
||||
}
|
||||
},
|
||||
checkVersion: () => apiClient.get('/admin/version-check').then(r => r.data),
|
||||
getBagTracking: () => apiClient.get('/admin/bag-tracking').then(r => r.data),
|
||||
updateBagTracking: (enabled: boolean) => apiClient.put('/admin/bag-tracking', { enabled }).then(r => r.data),
|
||||
@@ -625,9 +661,10 @@ export const reservationsApi = {
|
||||
update: (tripId: number | string, id: number, data: ReservationUpdateRequest) => apiClient.put(`/trips/${tripId}/reservations/${id}`, data).then(r => r.data),
|
||||
delete: (tripId: number | string, id: number) => apiClient.delete(`/trips/${tripId}/reservations/${id}`).then(r => r.data),
|
||||
updatePositions: (tripId: number | string, positions: { id: number; day_plan_position: number }[], dayId?: number) => apiClient.put(`/trips/${tripId}/reservations/positions`, { positions, day_id: dayId }).then(r => r.data),
|
||||
importBookingPreview: (tripId: number | string, files: File[]): Promise<BookingImportPreviewResponse> => {
|
||||
importBookingPreview: (tripId: number | string, files: File[], mode: BookingImportMode = 'no-ai'): Promise<BookingImportPreviewResponse> => {
|
||||
const fd = new FormData()
|
||||
for (const f of files) fd.append('files', f)
|
||||
fd.append('mode', mode)
|
||||
return apiClient.post(`/trips/${tripId}/reservations/import/booking`, fd, { headers: { 'Content-Type': 'multipart/form-data' } }).then(r => r.data)
|
||||
},
|
||||
importBookingConfirm: (tripId: number | string, items: BookingImportPreviewItem[]): Promise<BookingImportConfirmResponse> =>
|
||||
@@ -635,7 +672,7 @@ export const reservationsApi = {
|
||||
}
|
||||
|
||||
export const healthApi = {
|
||||
features: (): Promise<{ bookingImport: boolean }> => apiClient.get('/health/features').then(r => r.data),
|
||||
features: (): Promise<{ bookingImport: boolean; aiParsing: boolean }> => apiClient.get('/health/features').then(r => r.data),
|
||||
}
|
||||
|
||||
export const weatherApi = {
|
||||
|
||||
@@ -298,7 +298,12 @@ export default function AddonManager({ bagTrackingEnabled, onToggleBagTracking,
|
||||
</span>
|
||||
</div>
|
||||
{integrationAddons.map(addon => (
|
||||
<AddonRow key={addon.id} addon={addon} onToggle={handleToggle} t={t} />
|
||||
<div key={addon.id}>
|
||||
<AddonRow addon={addon} onToggle={handleToggle} t={t} />
|
||||
{addon.id === 'llm_parsing' && addon.enabled && (
|
||||
<LlmParsingConfig addon={addon} />
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
@@ -309,6 +314,200 @@ export default function AddonManager({ bagTrackingEnabled, onToggleBagTracking,
|
||||
)
|
||||
}
|
||||
|
||||
const MASKED = '••••••••'
|
||||
const DEFAULT_OLLAMA_URL = 'http://localhost:11434/v1'
|
||||
|
||||
/** Curated NuExtract models, pullable via Ollama (HF GGUF for 2.0; library for 1.5). */
|
||||
const NUEXTRACT_MODELS: { id: string; label: string; note: string; recommended: boolean; vision: boolean }[] = [
|
||||
{ id: 'hf.co/numind/NuExtract-2.0-2B-GGUF', label: 'NuExtract 2.0 — 2B', note: 'Vision · lightest · commercial license', recommended: true, vision: true },
|
||||
{ id: 'hf.co/numind/NuExtract-2.0-4B-GGUF', label: 'NuExtract 2.0 — 4B', note: 'Vision · best balance', recommended: true, vision: true },
|
||||
{ id: 'hf.co/numind/NuExtract-2.0-8B-GGUF', label: 'NuExtract 2.0 — 8B', note: 'Vision · highest quality', recommended: false, vision: true },
|
||||
{ id: 'nuextract', label: 'NuExtract 1.5 — 3.8B', note: 'Text-only', recommended: false, vision: false },
|
||||
]
|
||||
|
||||
/**
|
||||
* Instance-wide AI-parsing config. When set, applies to the whole instance and
|
||||
* overrides per-user config (see server llmConfig.ts). The API key is masked on
|
||||
* read; an unchanged mask is treated as a no-op by the server. For the local
|
||||
* provider, it also lists installed Ollama models and can pull NuExtract models.
|
||||
*/
|
||||
function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
const toast = useToast()
|
||||
const cfg = (addon.config ?? {}) as Record<string, unknown>
|
||||
const [provider, setProvider] = useState<string>((cfg.provider as string) ?? 'local')
|
||||
const [model, setModel] = useState<string>((cfg.model as string) ?? '')
|
||||
const [baseUrl, setBaseUrl] = useState<string>((cfg.baseUrl as string) ?? '')
|
||||
const [apiKey, setApiKey] = useState<string>((cfg.apiKey as string) ?? '')
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
// Local-provider model management.
|
||||
const [installed, setInstalled] = useState<string[]>([])
|
||||
const [modelsErr, setModelsErr] = useState('')
|
||||
const [loadingModels, setLoadingModels] = useState(false)
|
||||
const [pulling, setPulling] = useState<string | null>(null)
|
||||
const [pullPct, setPullPct] = useState(0)
|
||||
const [pullStatus, setPullStatus] = useState('')
|
||||
|
||||
const effectiveUrl = baseUrl.trim() || DEFAULT_OLLAMA_URL
|
||||
const isInstalled = (id: string) => installed.some(n => n === id || n.startsWith(id + ':') || n.startsWith(id))
|
||||
|
||||
const loadModels = async () => {
|
||||
if (provider !== 'local') return
|
||||
setLoadingModels(true)
|
||||
setModelsErr('')
|
||||
try {
|
||||
const res = await adminApi.llmLocalModels(effectiveUrl)
|
||||
setInstalled(res.models.map(m => m.name))
|
||||
} catch (e: unknown) {
|
||||
setModelsErr(e instanceof Error ? e.message : 'Could not reach the local LLM server')
|
||||
setInstalled([])
|
||||
} finally {
|
||||
setLoadingModels(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Load installed models when the local provider is active.
|
||||
useEffect(() => {
|
||||
if (provider === 'local') loadModels()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [provider])
|
||||
|
||||
const pull = async (id: string) => {
|
||||
if (pulling) return
|
||||
setPulling(id)
|
||||
setPullPct(0)
|
||||
setPullStatus('starting…')
|
||||
try {
|
||||
await adminApi.llmLocalPull(effectiveUrl, id, (p) => {
|
||||
if (p.error) throw new Error(p.error)
|
||||
if (p.status) setPullStatus(p.status)
|
||||
if (p.total && p.completed != null) setPullPct(Math.round((p.completed / p.total) * 100))
|
||||
})
|
||||
toast.success('Model pulled')
|
||||
setModel(id)
|
||||
await loadModels()
|
||||
} catch (e: unknown) {
|
||||
toast.error(e instanceof Error ? e.message : 'Pull failed')
|
||||
} finally {
|
||||
setPulling(null)
|
||||
setPullPct(0)
|
||||
setPullStatus('')
|
||||
}
|
||||
}
|
||||
|
||||
const save = async () => {
|
||||
setSaving(true)
|
||||
try {
|
||||
// Send the masked sentinel unchanged so the server keeps the stored key.
|
||||
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: baseUrl.trim(), apiKey, multimodal: cfg.multimodal === true } })
|
||||
toast.success('Saved')
|
||||
} catch {
|
||||
toast.error('Failed to save')
|
||||
} finally {
|
||||
setSaving(false)
|
||||
}
|
||||
}
|
||||
|
||||
const inputCls = 'w-full rounded-md border border-edge-secondary bg-surface px-2 py-1.5 text-sm text-content'
|
||||
return (
|
||||
<div className="px-6 py-4 border-b border-edge-secondary bg-surface-secondary space-y-3" style={{ paddingLeft: 70 }}>
|
||||
<p className="text-xs text-content-faint">
|
||||
Set instance-wide config (applies to all users). Leave blank to let each user configure their own provider.
|
||||
</p>
|
||||
<label className="block text-xs font-medium text-content-secondary">Provider
|
||||
<select className={inputCls} value={provider} onChange={e => setProvider(e.target.value)}>
|
||||
<option value="local">Local (OpenAI-compatible, e.g. Ollama)</option>
|
||||
<option value="openai">OpenAI</option>
|
||||
<option value="anthropic">Anthropic</option>
|
||||
</select>
|
||||
</label>
|
||||
<label className="block text-xs font-medium text-content-secondary">Model
|
||||
<input className={inputCls} value={model} onChange={e => setModel(e.target.value)} placeholder={provider === 'anthropic' ? 'claude-opus-4-8' : provider === 'openai' ? 'gpt-4o' : 'select or pull below'} />
|
||||
</label>
|
||||
{provider !== 'anthropic' && (
|
||||
<label className="block text-xs font-medium text-content-secondary">Base URL
|
||||
<input className={inputCls} value={baseUrl} onChange={e => setBaseUrl(e.target.value)} onBlur={loadModels} placeholder={provider === 'local' ? 'http://localhost:11434/v1' : 'https://api.openai.com/v1'} />
|
||||
</label>
|
||||
)}
|
||||
|
||||
{/* Local model management (Ollama) */}
|
||||
{provider === 'local' && (
|
||||
<div className="rounded-lg border border-edge-secondary p-3 space-y-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-semibold text-content-secondary">Installed models</span>
|
||||
<button onClick={loadModels} disabled={loadingModels} className="text-xs text-content-muted underline disabled:opacity-60">
|
||||
{loadingModels ? 'Loading…' : 'Refresh'}
|
||||
</button>
|
||||
</div>
|
||||
{modelsErr && <p className="text-xs text-[#b91c1c]">{modelsErr}</p>}
|
||||
{!modelsErr && installed.length === 0 && !loadingModels && (
|
||||
<p className="text-xs text-content-faint">No models installed yet — pull one below.</p>
|
||||
)}
|
||||
{installed.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{installed.map(name => (
|
||||
<button
|
||||
key={name}
|
||||
onClick={() => setModel(name)}
|
||||
className={`rounded-full px-2.5 py-1 text-xs border ${model === name ? 'bg-accent text-accent-text border-transparent' : 'border-edge-secondary text-content-secondary'}`}
|
||||
>
|
||||
{name}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="text-xs font-semibold text-content-secondary pt-1">Pull a NuExtract model</div>
|
||||
<div className="space-y-2">
|
||||
{NUEXTRACT_MODELS.map(m => {
|
||||
const installedHere = isInstalled(m.id)
|
||||
const isPulling = pulling === m.id
|
||||
return (
|
||||
<div key={m.id} className="flex items-center gap-3">
|
||||
<div style={{ flex: 1, minWidth: 0 }}>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-sm text-content">{m.label}</span>
|
||||
{m.recommended && (
|
||||
<span className="bg-[rgba(16,185,129,0.15)] text-[#047857]" style={{ fontSize: 10, fontWeight: 600, padding: '1px 6px', borderRadius: 6 }}>Recommended</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-xs text-content-faint">{m.note}</div>
|
||||
{isPulling && (
|
||||
<div className="mt-1">
|
||||
<div className="h-1.5 w-full rounded-full bg-surface-tertiary overflow-hidden">
|
||||
<div className="h-full bg-accent" style={{ width: `${pullPct}%`, transition: 'width 0.2s' }} />
|
||||
</div>
|
||||
<div className="text-[10px] text-content-faint mt-0.5">{pullStatus} {pullPct ? `· ${pullPct}%` : ''}</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{installedHere ? (
|
||||
<button onClick={() => setModel(m.id)} className="shrink-0 rounded-md border border-edge-secondary px-3 py-1.5 text-xs text-content-secondary">Use</button>
|
||||
) : (
|
||||
<button onClick={() => pull(m.id)} disabled={!!pulling} className="shrink-0 rounded-md bg-accent text-accent-text px-3 py-1.5 text-xs font-medium disabled:opacity-60">
|
||||
{isPulling ? 'Pulling…' : 'Pull'}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<label className="block text-xs font-medium text-content-secondary">API key
|
||||
<input type="password" className={inputCls} value={apiKey} onChange={e => setApiKey(e.target.value)} placeholder={apiKey === MASKED ? MASKED : provider === 'local' ? '(often not required)' : 'sk-…'} />
|
||||
</label>
|
||||
{provider === 'anthropic' && (
|
||||
<p className="text-xs text-content-faint">Anthropic reads PDFs (including scans) natively. Local/OpenAI models receive extracted text — scanned PDFs need Anthropic.</p>
|
||||
)}
|
||||
<button onClick={save} disabled={saving} className="bg-accent text-accent-text rounded-md px-3 py-1.5 text-sm font-medium disabled:opacity-60">
|
||||
{saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
interface AddonRowProps {
|
||||
addon: Addon
|
||||
onToggle: (addon: Addon) => void
|
||||
|
||||
@@ -2,11 +2,12 @@ import React from 'react'
|
||||
import ReactDOM from 'react-dom'
|
||||
import { useState, useRef, useEffect } from 'react'
|
||||
import { Upload, Plane, Train, Hotel, UtensilsCrossed, Car, Anchor, Calendar, ArrowLeft, X } from 'lucide-react'
|
||||
import type { BookingImportPreviewItem } from '@trek/shared'
|
||||
import type { BookingImportPreviewItem, BookingImportFileReport } from '@trek/shared'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import { useToast } from '../shared/Toast'
|
||||
import { reservationsApi } from '../../api/client'
|
||||
import { reservationsApi, healthApi } from '../../api/client'
|
||||
import { useTripStore } from '../../store/tripStore'
|
||||
import { useSettingsStore } from '../../store/settingsStore'
|
||||
|
||||
interface BookingImportModalProps {
|
||||
isOpen: boolean
|
||||
@@ -54,6 +55,7 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
const { t } = useTranslation()
|
||||
const toast = useToast()
|
||||
const loadTrip = useTripStore((s) => s.loadTrip)
|
||||
const alwaysRetryAi = useSettingsStore((s) => s.settings.llm_always_retry)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const mouseDownTarget = useRef<EventTarget | null>(null)
|
||||
|
||||
@@ -66,6 +68,10 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
const [previewItems, setPreviewItems] = useState<BookingImportPreviewItem[]>([])
|
||||
const [warnings, setWarnings] = useState<string[]>([])
|
||||
const [excluded, setExcluded] = useState<Set<number>>(() => new Set())
|
||||
// AI fallback: addon-level availability + per-file report + in-flight retries.
|
||||
const [aiParsing, setAiParsing] = useState(false)
|
||||
const [fileReports, setFileReports] = useState<BookingImportFileReport[]>([])
|
||||
const [retrying, setRetrying] = useState<Set<string>>(() => new Set())
|
||||
|
||||
const reset = () => {
|
||||
setPhase('upload')
|
||||
@@ -76,6 +82,8 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
setPreviewItems([])
|
||||
setWarnings([])
|
||||
setExcluded(new Set())
|
||||
setFileReports([])
|
||||
setRetrying(new Set())
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
@@ -84,6 +92,11 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [isOpen])
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
healthApi.features().then(f => setAiParsing(!!f.aiParsing)).catch(() => setAiParsing(false))
|
||||
}, [isOpen])
|
||||
|
||||
const handleClose = () => { reset(); onClose() }
|
||||
|
||||
const validateFile = (f: File): string | null => {
|
||||
@@ -126,9 +139,13 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
setLoading(true)
|
||||
setError('')
|
||||
try {
|
||||
const result = await reservationsApi.importBookingPreview(tripId, files)
|
||||
// When the user opted into "always retry with AI", rescue files kitinerary
|
||||
// can't read automatically; otherwise offer a per-file retry in the preview.
|
||||
const mode = aiParsing && alwaysRetryAi ? 'fallback-on-empty' : 'no-ai'
|
||||
const result = await reservationsApi.importBookingPreview(tripId, files, mode)
|
||||
setPreviewItems(result.items ?? [])
|
||||
setWarnings(result.warnings ?? [])
|
||||
setFileReports(result.files ?? [])
|
||||
setExcluded(new Set())
|
||||
setPhase('preview')
|
||||
} catch (err: any) {
|
||||
@@ -139,6 +156,24 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
}
|
||||
}
|
||||
|
||||
// Re-run a single file through the LLM (force-ai) and merge any new items in.
|
||||
const handleRetryAi = async (fileName: string) => {
|
||||
const file = files.find(f => f.name === fileName)
|
||||
if (!file || retrying.has(fileName)) return
|
||||
setRetrying(prev => new Set(prev).add(fileName))
|
||||
try {
|
||||
const result = await reservationsApi.importBookingPreview(tripId, [file], 'force-ai')
|
||||
setPreviewItems(prev => [...prev, ...(result.items ?? [])])
|
||||
setWarnings(prev => [...prev, ...(result.warnings ?? [])])
|
||||
setFileReports(prev => prev.map(r => r.fileName === fileName ? { ...r, aiUsed: true } : r))
|
||||
} catch (err: any) {
|
||||
const msg = err?.response?.data?.error ?? t('reservations.import.error')
|
||||
setError(msg)
|
||||
} finally {
|
||||
setRetrying(prev => { const next = new Set(prev); next.delete(fileName); return next })
|
||||
}
|
||||
}
|
||||
|
||||
const handleConfirm = async () => {
|
||||
const toImport = previewItems.filter((_, i) => !excluded.has(i))
|
||||
if (toImport.length === 0) return
|
||||
@@ -290,8 +325,15 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
<Icon size={15} color={typeColor(item.type)} />
|
||||
</div>
|
||||
<div style={{ flex: 1, minWidth: 0 }}>
|
||||
<div style={{ fontSize: 13, fontWeight: 600, color: 'var(--text-primary)', marginBottom: 2, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{item.title}
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 6, marginBottom: 2 }}>
|
||||
<span style={{ fontSize: 13, fontWeight: 600, color: 'var(--text-primary)', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{item.title}
|
||||
</span>
|
||||
{item.needs_review && (
|
||||
<span className="bg-[rgba(245,158,11,0.15)] text-[#92400e]" style={{ flexShrink: 0, fontSize: 10, fontWeight: 600, padding: '1px 6px', borderRadius: 6 }}>
|
||||
{t('reservations.import.needsReview')}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{fromEp && toEp && (
|
||||
<div style={{ fontSize: 11, color: 'var(--text-muted)', marginBottom: 2 }}>
|
||||
@@ -326,6 +368,23 @@ export default function BookingImportModal({ isOpen, onClose, tripId, pushUndo }
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
|
||||
{/* Per-file AI fallback: offer a retry for files kitinerary couldn't read. */}
|
||||
{phase === 'preview' && fileReports.filter(r => r.aiAvailable && !r.aiUsed).map(r => (
|
||||
<div key={`ai-${r.fileName}`} className="bg-surface-secondary" style={{ borderRadius: 10, padding: '8px 12px', marginBottom: 8, border: '1px dashed var(--border-primary)', display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<span style={{ flex: 1, minWidth: 0, fontSize: 12, color: 'var(--text-muted)', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{r.fileName}
|
||||
</span>
|
||||
<button
|
||||
onClick={() => handleRetryAi(r.fileName)}
|
||||
disabled={retrying.has(r.fileName)}
|
||||
className="bg-accent text-accent-text"
|
||||
style={{ flexShrink: 0, border: 'none', borderRadius: 8, padding: '5px 10px', fontSize: 12, fontWeight: 500, cursor: retrying.has(r.fileName) ? 'default' : 'pointer', fontFamily: 'inherit', opacity: retrying.has(r.fileName) ? 0.6 : 1 }}
|
||||
>
|
||||
{retrying.has(r.fileName) ? t('reservations.import.aiParsing') : t('reservations.import.tryAi')}
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</>
|
||||
)}
|
||||
|
||||
|
||||
@@ -360,6 +360,37 @@ export default function DisplaySettingsTab(): React.ReactElement {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Always retry booking imports with AI */}
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-2 text-content-secondary">{t('settings.aiAlwaysRetry')}</label>
|
||||
<div className="flex gap-3">
|
||||
{[
|
||||
{ value: true, label: t('settings.on') || 'On' },
|
||||
{ value: false, label: t('settings.off') || 'Off' },
|
||||
].map(opt => (
|
||||
<button
|
||||
key={String(opt.value)}
|
||||
onClick={async () => {
|
||||
try { await updateSetting('llm_always_retry', opt.value) }
|
||||
catch (e: unknown) { toast.error(e instanceof Error ? e.message : t('common.error')) }
|
||||
}}
|
||||
style={{
|
||||
display: 'flex', alignItems: 'center', gap: 8,
|
||||
padding: '10px 20px', borderRadius: 10, cursor: 'pointer',
|
||||
fontFamily: 'inherit', fontSize: 14, fontWeight: 500,
|
||||
border: (!!settings.llm_always_retry) === opt.value ? '2px solid var(--text-primary)' : '2px solid var(--border-primary)',
|
||||
background: (!!settings.llm_always_retry) === opt.value ? 'var(--bg-hover)' : 'var(--bg-card)',
|
||||
color: 'var(--text-primary)',
|
||||
transition: 'all 0.15s',
|
||||
}}
|
||||
>
|
||||
{opt.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-xs mt-1 text-content-faint">{t('settings.aiAlwaysRetryHint')}</p>
|
||||
</div>
|
||||
|
||||
{/* Optimize route from accommodation */}
|
||||
<div>
|
||||
<label className="block text-sm font-medium mb-2 text-content-secondary">{t('settings.optimizeFromAccommodation')}</label>
|
||||
|
||||
@@ -128,6 +128,14 @@ export interface Settings {
|
||||
dashboard_fx_from?: string
|
||||
dashboard_fx_to?: string
|
||||
dashboard_timezones?: string[]
|
||||
// AI booking-import fallback (per-user config; used when the admin has not set
|
||||
// instance-wide config on the llm_parsing addon). llm_api_key is masked on read.
|
||||
llm_always_retry?: boolean
|
||||
llm_provider?: 'local' | 'openai' | 'anthropic'
|
||||
llm_model?: string
|
||||
llm_base_url?: string
|
||||
llm_multimodal?: boolean
|
||||
llm_api_key?: string
|
||||
}
|
||||
|
||||
export interface AssignmentsMap {
|
||||
|
||||
Generated
+217
@@ -4481,6 +4481,190 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz",
|
||||
"integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==",
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "0.1.80",
|
||||
"@napi-rs/canvas-darwin-arm64": "0.1.80",
|
||||
"@napi-rs/canvas-darwin-x64": "0.1.80",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "0.1.80",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "0.1.80",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.80",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "0.1.80",
|
||||
"@napi-rs/canvas-linux-x64-musl": "0.1.80",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "0.1.80"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz",
|
||||
"integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz",
|
||||
"integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz",
|
||||
"integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz",
|
||||
"integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz",
|
||||
"integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz",
|
||||
"integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz",
|
||||
"integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz",
|
||||
"integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz",
|
||||
"integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "0.1.80",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz",
|
||||
"integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.5.tgz",
|
||||
@@ -15436,6 +15620,38 @@
|
||||
"pbf": "bin/pbf"
|
||||
}
|
||||
},
|
||||
"node_modules/pdf-parse": {
|
||||
"version": "2.4.5",
|
||||
"resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz",
|
||||
"integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@napi-rs/canvas": "0.1.80",
|
||||
"pdfjs-dist": "5.4.296"
|
||||
},
|
||||
"bin": {
|
||||
"pdf-parse": "bin/cli.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.16.0 <21 || >=22.3.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/mehmet-kozan"
|
||||
}
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.4.296",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz",
|
||||
"integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=20.16.0 || >=22.3.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^0.1.80"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
@@ -20907,6 +21123,7 @@
|
||||
"node-cron": "^4.2.1",
|
||||
"nodemailer": "^9.0.1",
|
||||
"otplib": "^12.0.1",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"qrcode": "^1.5.4",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.2",
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
"node-cron": "^4.2.1",
|
||||
"nodemailer": "^9.0.1",
|
||||
"otplib": "^12.0.1",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"qrcode": "^1.5.4",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.2",
|
||||
|
||||
@@ -8,6 +8,7 @@ export const ADDON_IDS = {
|
||||
COLLAB: 'collab',
|
||||
JOURNEY: 'journey',
|
||||
AIRTRAIL: 'airtrail',
|
||||
LLM_PARSING: 'llm_parsing',
|
||||
} as const;
|
||||
|
||||
export type AddonId = typeof ADDON_IDS[keyof typeof ADDON_IDS];
|
||||
|
||||
@@ -120,6 +120,7 @@ function seedAddons(db: Database.Database): void {
|
||||
{ id: 'collab', name: 'Collab', description: 'Notes, polls, and live chat for trip collaboration', type: 'trip', icon: 'Users', enabled: 1, sort_order: 6 },
|
||||
{ id: 'journey', name: 'Journey', description: 'Trip tracking & travel journal — check-ins, photos, daily stories', type: 'global', icon: 'Compass', enabled: 0, sort_order: 35 },
|
||||
{ id: 'airtrail', name: 'AirTrail', description: 'Sync flights from your self-hosted AirTrail instance', type: 'integration', icon: 'Plane', enabled: 0, sort_order: 14 },
|
||||
{ id: 'llm_parsing', name: 'AI Parsing', description: 'LLM fallback for booking imports kitinerary cannot read', type: 'integration', icon: 'Sparkles', enabled: 0, sort_order: 15 },
|
||||
];
|
||||
const insertAddon = db.prepare('INSERT OR IGNORE INTO addons (id, name, description, type, icon, enabled, sort_order) VALUES (?, ?, ?, ?, ?, ?, ?)');
|
||||
for (const a of defaultAddons) insertAddon.run(a.id, a.name, a.description, a.type, a.icon, a.enabled, a.sort_order);
|
||||
|
||||
@@ -15,7 +15,8 @@ import type { User } from '../../types';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { BookingImportService } from './booking-import.service';
|
||||
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse } from '@trek/shared';
|
||||
import { bookingImportModeSchema } from '@trek/shared';
|
||||
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse, BookingImportMode } from '@trek/shared';
|
||||
|
||||
const ACCEPTED_EXTS = new Set(['.eml', '.pdf', '.pkpass', '.html', '.htm', '.txt']);
|
||||
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
@@ -54,11 +55,23 @@ export class BookingImportController {
|
||||
@CurrentUser() user: User,
|
||||
@Param('tripId') tripId: string,
|
||||
@UploadedFiles() files: Express.Multer.File[] | undefined,
|
||||
@Body('mode') rawMode?: string,
|
||||
) {
|
||||
const trip = this.requireTrip(tripId, user);
|
||||
this.requireEdit(trip, user);
|
||||
|
||||
if (!this.bookingImport.isAvailable()) {
|
||||
const modeResult = bookingImportModeSchema.safeParse(rawMode ?? 'no-ai');
|
||||
if (!modeResult.success) {
|
||||
throw new HttpException({ error: 'Invalid mode' }, 400);
|
||||
}
|
||||
const mode: BookingImportMode = modeResult.data;
|
||||
|
||||
// Forcing AI requires it to be configured; otherwise surface a clear 4xx.
|
||||
if (mode === 'force-ai' && !this.bookingImport.aiAvailable(user.id)) {
|
||||
throw new HttpException({ error: 'AI parsing is not configured' }, 409);
|
||||
}
|
||||
// For the kitinerary-only path, keep the existing 503 contract.
|
||||
if (mode === 'no-ai' && !this.bookingImport.isAvailable()) {
|
||||
throw new HttpException({ error: 'KItinerary extractor is not available on this server' }, 503);
|
||||
}
|
||||
|
||||
@@ -74,7 +87,7 @@ export class BookingImportController {
|
||||
}
|
||||
}
|
||||
|
||||
const result: BookingImportPreviewResponse = await this.bookingImport.preview(files);
|
||||
const result: BookingImportPreviewResponse = await this.bookingImport.preview(files, mode, user.id);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@ import { BookingImportController } from './booking-import.controller';
|
||||
import { BookingImportService } from './booking-import.service';
|
||||
import { KitineraryExtractorService } from './kitinerary-extractor.service';
|
||||
import { FeaturesController } from './features.controller';
|
||||
import { LlmParseModule } from '../llm-parse/llm-parse.module';
|
||||
|
||||
@Module({
|
||||
imports: [LlmParseModule],
|
||||
controllers: [BookingImportController, FeaturesController],
|
||||
providers: [BookingImportService, KitineraryExtractorService],
|
||||
})
|
||||
|
||||
@@ -8,9 +8,10 @@ import { searchNominatim } from '../../services/mapsService';
|
||||
import { db } from '../../db/database';
|
||||
import type { User } from '../../types';
|
||||
import { KitineraryExtractorService } from './kitinerary-extractor.service';
|
||||
import { LlmParseService } from '../llm-parse/llm-parse.service';
|
||||
import { mapReservations } from './kitinerary-mapper';
|
||||
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse, Reservation } from '@trek/shared';
|
||||
import type { ParsedBookingItem } from './kitinerary.types';
|
||||
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse, BookingImportMode, BookingImportFileReport, Reservation } from '@trek/shared';
|
||||
import type { ParsedBookingItem, KiReservation } from './kitinerary.types';
|
||||
|
||||
function resolveDayId(tripId: string, iso: string | null | undefined): number | null {
|
||||
if (!iso) return null;
|
||||
@@ -22,12 +23,20 @@ function resolveDayId(tripId: string, iso: string | null | undefined): number |
|
||||
|
||||
@Injectable()
|
||||
export class BookingImportService {
|
||||
constructor(private readonly extractor: KitineraryExtractorService) {}
|
||||
constructor(
|
||||
private readonly extractor: KitineraryExtractorService,
|
||||
private readonly llmParse: LlmParseService,
|
||||
) {}
|
||||
|
||||
isAvailable(): boolean {
|
||||
return this.extractor.isAvailable();
|
||||
}
|
||||
|
||||
/** True when the LLM fallback is enabled and configured for this user. */
|
||||
aiAvailable(userId: number): boolean {
|
||||
return this.llmParse.isAvailable(userId);
|
||||
}
|
||||
|
||||
verifyTripAccess(tripId: string, userId: number) {
|
||||
return verifyTripAccess(tripId, userId);
|
||||
}
|
||||
@@ -37,37 +46,65 @@ export class BookingImportService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse uploaded files through kitinerary-extractor and return a preview list.
|
||||
* Does NOT persist anything.
|
||||
* Parse uploaded files and return a preview list. Does NOT persist anything.
|
||||
* Runs kitinerary first; depending on `mode`, falls back to the LLM:
|
||||
* - no-ai: kitinerary only
|
||||
* - fallback-on-empty: LLM for files kitinerary returns nothing for
|
||||
* - force-ai: LLM on every file (kitinerary skipped)
|
||||
* LLM-derived items are flagged needs_review. Per-file AI usage is reported.
|
||||
*/
|
||||
async preview(files: Express.Multer.File[]): Promise<BookingImportPreviewResponse> {
|
||||
if (!this.extractor.isAvailable()) {
|
||||
async preview(
|
||||
files: Express.Multer.File[],
|
||||
mode: BookingImportMode,
|
||||
userId: number,
|
||||
): Promise<BookingImportPreviewResponse> {
|
||||
const kitineraryAvailable = this.extractor.isAvailable();
|
||||
const aiAvailable = this.llmParse.isAvailable(userId);
|
||||
if (!kitineraryAvailable && !aiAvailable) {
|
||||
throw new HttpException({ error: 'KItinerary extractor is not available on this server' }, 503);
|
||||
}
|
||||
|
||||
const allItems: ParsedBookingItem[] = [];
|
||||
const allWarnings: string[] = [];
|
||||
const fileReports: BookingImportFileReport[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
let kiItems;
|
||||
try {
|
||||
kiItems = await this.extractor.extract(file.buffer, file.originalname);
|
||||
} catch (err) {
|
||||
allWarnings.push(`${file.originalname}: extraction failed — ${err instanceof Error ? err.message : String(err)}`);
|
||||
continue;
|
||||
let kiItems: KiReservation[] = [];
|
||||
let aiUsed = false;
|
||||
|
||||
// Stage 1: kitinerary (skipped entirely when forcing AI).
|
||||
if (mode !== 'force-ai' && kitineraryAvailable) {
|
||||
try {
|
||||
kiItems = await this.extractor.extract(file.buffer, file.originalname);
|
||||
} catch (err) {
|
||||
allWarnings.push(`${file.originalname}: extraction failed — ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 1b: LLM fallback.
|
||||
const runLlm = aiAvailable && (mode === 'force-ai' || (mode === 'fallback-on-empty' && kiItems.length === 0));
|
||||
if (runLlm) {
|
||||
aiUsed = true;
|
||||
const llm = await this.llmParse.parse({ buffer: file.buffer, originalName: file.originalname }, userId);
|
||||
kiItems = llm.kiItems;
|
||||
allWarnings.push(...llm.warnings);
|
||||
}
|
||||
|
||||
fileReports.push({ fileName: file.originalname, aiAvailable, aiUsed });
|
||||
|
||||
if (kiItems.length === 0) {
|
||||
allWarnings.push(`${file.originalname}: no reservations found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { items, warnings } = mapReservations(kiItems, file.originalname);
|
||||
// LLM extraction is less certain than kitinerary — always flag for review.
|
||||
if (aiUsed) for (const it of items) it.needs_review = true;
|
||||
allItems.push(...items);
|
||||
allWarnings.push(...warnings);
|
||||
}
|
||||
|
||||
return { items: allItems, warnings: allWarnings };
|
||||
return { items: allItems, warnings: allWarnings, files: fileReports };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { KitineraryExtractorService } from './kitinerary-extractor.service';
|
||||
import { isAddonEnabled } from '../../services/adminService';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
|
||||
/** Exposes server feature flags consumed by the frontend to show/hide optional UI. */
|
||||
@Controller('api/health')
|
||||
@@ -10,6 +12,9 @@ export class FeaturesController {
|
||||
features() {
|
||||
return {
|
||||
bookingImport: this.extractor.isAvailable(),
|
||||
// Addon-level flag (per-user config availability is reported per-file in
|
||||
// the preview response). Drives whether the client shows AI affordances.
|
||||
aiParsing: isAddonEnabled(ADDON_IDS.LLM_PARSING),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
|
||||
|
||||
const TIMEOUT_MS = 120_000;
|
||||
const MAX_TOKENS = 8192;
|
||||
const ANTHROPIC_VERSION = '2023-06-01';
|
||||
const TOOL_NAME = 'emit_reservations';
|
||||
|
||||
/**
|
||||
* Anthropic Messages API client. Structured output via forced tool-use: a single
|
||||
* `emit_reservations` tool whose `input_schema` is the reservations schema, with
|
||||
* `tool_choice` forcing it — the documented, reliable way to get structured JSON.
|
||||
* PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs).
|
||||
* Raw fetch (no SDK) to match the codebase's HTTP style.
|
||||
*/
|
||||
export class AnthropicClient implements LlmExtractionClient {
|
||||
async extract(input: LlmExtractionInput): Promise<Record<string, unknown>[]> {
|
||||
const base = (input.baseUrl ?? 'https://api.anthropic.com').replace(/\/+$/, '');
|
||||
const url = `${base}/v1/messages`;
|
||||
|
||||
const content: unknown[] = [];
|
||||
if (input.file) {
|
||||
content.push({
|
||||
type: 'document',
|
||||
source: { type: 'base64', media_type: input.file.mimeType, data: input.file.data.toString('base64') },
|
||||
});
|
||||
}
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT,
|
||||
});
|
||||
|
||||
const body = {
|
||||
model: input.model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
system: input.prompt,
|
||||
tools: [
|
||||
{
|
||||
name: TOOL_NAME,
|
||||
description: 'Return the travel reservations extracted from the document.',
|
||||
input_schema: input.jsonSchema,
|
||||
},
|
||||
],
|
||||
tool_choice: { type: 'tool', name: TOOL_NAME },
|
||||
messages: [{ role: 'user', content }],
|
||||
};
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-api-key': input.apiKey ?? '',
|
||||
'anthropic-version': ANTHROPIC_VERSION,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => '');
|
||||
throw new Error(`Anthropic request failed (${res.status}): ${detail.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
stop_reason?: string;
|
||||
content?: { type: string; name?: string; input?: { reservations?: unknown } }[];
|
||||
};
|
||||
|
||||
if (data.stop_reason === 'refusal') {
|
||||
throw new Error('Anthropic declined to process this document');
|
||||
}
|
||||
|
||||
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === TOOL_NAME);
|
||||
const reservations = toolUse?.input?.reservations;
|
||||
return Array.isArray(reservations) ? (reservations as Record<string, unknown>[]) : [];
|
||||
}
|
||||
}
|
||||
|
||||
const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
|
||||
@@ -0,0 +1,95 @@
|
||||
import type { LlmExtractionClient, LlmExtractionInput } from '../llm-provider.interface';
|
||||
|
||||
const TIMEOUT_MS = 60_000;
|
||||
const MAX_TOKENS = 4096;
|
||||
|
||||
/**
|
||||
* OpenAI-compatible chat-completions client. Covers both the "openai" cloud
|
||||
* provider and the "local" provider (Ollama / vLLM / llama.cpp / LM Studio),
|
||||
* which all expose `POST {baseUrl}/chat/completions`. Native binaries (PDF) are
|
||||
* sent as an OpenAI `file` content part; text goes as a text part. Uses the
|
||||
* global fetch (no SDK) to match the codebase's HTTP style.
|
||||
*/
|
||||
export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
async extract(input: LlmExtractionInput): Promise<Record<string, unknown>[]> {
|
||||
const base = (input.baseUrl ?? 'https://api.openai.com/v1').replace(/\/+$/, '');
|
||||
const url = `${base}/chat/completions`;
|
||||
|
||||
const userContent: unknown[] = [
|
||||
{ type: 'text', text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT },
|
||||
];
|
||||
// Only genuine images go natively (as image_url) — OpenAI-compatible servers
|
||||
// (notably Ollama) reject `file`/PDF content parts. PDFs reach this client as
|
||||
// pre-extracted text (see llm-parse.service.ts), never as bytes.
|
||||
if (input.file && input.file.mimeType.startsWith('image/')) {
|
||||
const b64 = input.file.data.toString('base64');
|
||||
userContent.push({
|
||||
type: 'image_url',
|
||||
image_url: { url: `data:${input.file.mimeType};base64,${b64}` },
|
||||
});
|
||||
}
|
||||
|
||||
const body = {
|
||||
model: input.model,
|
||||
max_tokens: MAX_TOKENS,
|
||||
// Extraction is a deterministic task — Ollama defaults to 0.7, which makes
|
||||
// small models (NuExtract) drop fields or return empty. Pin to 0.
|
||||
temperature: 0,
|
||||
messages: [
|
||||
{ role: 'system', content: input.prompt },
|
||||
{ role: 'user', content: userContent },
|
||||
],
|
||||
response_format: {
|
||||
type: 'json_schema',
|
||||
json_schema: { name: 'reservations', schema: input.jsonSchema, strict: false },
|
||||
},
|
||||
};
|
||||
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: 'POST',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
...(input.apiKey ? { authorization: `Bearer ${input.apiKey}` } : {}),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => '');
|
||||
throw new Error(`LLM request failed (${res.status}): ${detail.slice(0, 300)}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
return parseReservations(content);
|
||||
}
|
||||
}
|
||||
|
||||
const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
|
||||
|
||||
/** Tolerant parse: strip code fences, JSON.parse, pull `reservations`. `[]` on failure. */
|
||||
function parseReservations(content: string | undefined | null): Record<string, unknown>[] {
|
||||
if (!content) return [];
|
||||
const stripped = content.trim().replace(/^```(?:json)?/i, '').replace(/```$/, '').trim();
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(stripped);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(parsed)) return parsed as Record<string, unknown>[];
|
||||
if (parsed && typeof parsed === 'object' && Array.isArray((parsed as { reservations?: unknown }).reservations)) {
|
||||
return (parsed as { reservations: Record<string, unknown>[] }).reservations;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { LlmExtractionClient } from './llm-provider.interface';
|
||||
import type { ResolvedLlmConfig } from '../../services/llmConfig';
|
||||
import { OpenAiCompatibleClient } from './clients/openai-compatible.client';
|
||||
import { AnthropicClient } from './clients/anthropic.client';
|
||||
|
||||
/**
|
||||
* Pick the provider client for a resolved config.
|
||||
* - 'anthropic' → Anthropic Messages API client
|
||||
* - 'openai' | 'local' → OpenAI-compatible client (cloud or local base URL)
|
||||
*/
|
||||
export function createLlmClient(config: ResolvedLlmConfig): LlmExtractionClient {
|
||||
switch (config.provider) {
|
||||
case 'anthropic':
|
||||
return new AnthropicClient();
|
||||
case 'openai':
|
||||
case 'local':
|
||||
return new OpenAiCompatibleClient();
|
||||
// TODO(nuextract): add a NuExtract template adapter here (local vision model
|
||||
// with its own template-fill API) once the OpenAI-compatible path proves
|
||||
// insufficient for small local models — see the design seam in the plan.
|
||||
default:
|
||||
return new OpenAiCompatibleClient();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { db } from '../../db/database';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { isAddonEnabled } from '../../services/adminService';
|
||||
import { getUserSettings, getDecryptedUserSetting } from '../../services/settingsService';
|
||||
import { decryptLlmApiKey, LLM_PROVIDERS, type LlmProvider, type ResolvedLlmConfig } from '../../services/llmConfig';
|
||||
|
||||
function asProvider(v: unknown): LlmProvider | null {
|
||||
return typeof v === 'string' && (LLM_PROVIDERS as string[]).includes(v) ? (v as LlmProvider) : null;
|
||||
}
|
||||
|
||||
function readInstanceConfig(): ResolvedLlmConfig | null {
|
||||
const row = db.prepare('SELECT config FROM addons WHERE id = ?').get(ADDON_IDS.LLM_PARSING) as { config?: string } | undefined;
|
||||
if (!row?.config) return null;
|
||||
let cfg: Record<string, unknown>;
|
||||
try {
|
||||
cfg = JSON.parse(row.config || '{}');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const provider = asProvider(cfg.provider);
|
||||
const model = typeof cfg.model === 'string' ? cfg.model.trim() : '';
|
||||
if (!provider || !model) return null;
|
||||
return {
|
||||
provider,
|
||||
model,
|
||||
baseUrl: typeof cfg.baseUrl === 'string' && cfg.baseUrl.trim() ? cfg.baseUrl.trim() : undefined,
|
||||
apiKey: decryptLlmApiKey(cfg.apiKey),
|
||||
multimodal: cfg.multimodal === true,
|
||||
};
|
||||
}
|
||||
|
||||
function readUserConfig(userId: number): ResolvedLlmConfig | null {
|
||||
const settings = getUserSettings(userId);
|
||||
const provider = asProvider(settings.llm_provider);
|
||||
const model = typeof settings.llm_model === 'string' ? settings.llm_model.trim() : '';
|
||||
if (!provider || !model) return null;
|
||||
const apiKey = getDecryptedUserSetting(userId, 'llm_api_key') ?? undefined;
|
||||
return {
|
||||
provider,
|
||||
model,
|
||||
baseUrl: typeof settings.llm_base_url === 'string' && settings.llm_base_url.trim() ? settings.llm_base_url.trim() : undefined,
|
||||
apiKey,
|
||||
multimodal: settings.llm_multimodal === true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the effective LLM config for a user, gated by the addon.
|
||||
* Order: addon disabled → null; admin instance config wins; else per-user config;
|
||||
* else null. This is the single place the API key is decrypted.
|
||||
*/
|
||||
export function resolveLlmConfig(userId: number): ResolvedLlmConfig | null {
|
||||
if (!isAddonEnabled(ADDON_IDS.LLM_PARSING)) return null;
|
||||
return readInstanceConfig() ?? readUserConfig(userId);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { Controller, Get, Post, Query, Body, Res, UseGuards } from '@nestjs/common';
|
||||
import type { Response } from 'express';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { AdminGuard } from '../auth/admin.guard';
|
||||
import { LlmLocalService } from './llm-local.service';
|
||||
|
||||
/**
|
||||
* Admin-only management of a local LLM server (Ollama): list installed models and
|
||||
* pull new ones (e.g. NuExtract). Used by the AI-parsing addon config UI.
|
||||
*/
|
||||
@Controller('api/admin/llm/local')
|
||||
@UseGuards(JwtAuthGuard, AdminGuard)
|
||||
export class LlmLocalController {
|
||||
constructor(private readonly local: LlmLocalService) {}
|
||||
|
||||
@Get('models')
|
||||
models(@Query('baseUrl') baseUrl?: string) {
|
||||
return this.local.listModels(baseUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a model pull. Proxies Ollama's NDJSON progress lines
|
||||
* ({ status, total?, completed? }) straight to the client, which reads the
|
||||
* response body to render a progress bar. Uses @Res() to stream manually.
|
||||
*/
|
||||
@Post('pull')
|
||||
async pull(@Body() body: { baseUrl?: string; model?: string }, @Res() res: Response): Promise<void> {
|
||||
const stream = await this.local.pull(body?.baseUrl, body?.model ?? '');
|
||||
res.status(200);
|
||||
res.setHeader('Content-Type', 'application/x-ndjson');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
const reader = stream.getReader();
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
res.write(Buffer.from(value));
|
||||
}
|
||||
} catch {
|
||||
// Upstream dropped mid-pull — close the response; the client surfaces it.
|
||||
} finally {
|
||||
res.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import { Injectable, HttpException } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Admin helpers for managing a local OpenAI-compatible LLM server (Ollama).
|
||||
* Talks to Ollama's *management* API (`/api/tags`, `/api/pull`), which lives at
|
||||
* the server root — not the `/v1` OpenAI-compatible path the extraction client
|
||||
* uses. Admin-only (guarded at the controller); the base URL is admin-supplied
|
||||
* and typically points at a localhost Ollama, so SSRF guarding is intentionally
|
||||
* not applied (it would block localhost) — we only validate the protocol.
|
||||
*/
|
||||
@Injectable()
|
||||
export class LlmLocalService {
|
||||
/** Derive the Ollama root from a configured base URL (strip a trailing /v1). */
|
||||
ollamaRoot(baseUrl: string | undefined): string {
|
||||
const raw = (baseUrl ?? 'http://localhost:11434').trim();
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(raw);
|
||||
} catch {
|
||||
throw new HttpException({ error: 'Invalid base URL' }, 400);
|
||||
}
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||
throw new HttpException({ error: 'Base URL must be http(s)' }, 400);
|
||||
}
|
||||
return raw.replace(/\/+$/, '').replace(/\/v1$/, '');
|
||||
}
|
||||
|
||||
/** List models already pulled on the local server. */
|
||||
async listModels(baseUrl: string | undefined): Promise<{ models: { name: string; size: number }[] }> {
|
||||
const root = this.ollamaRoot(baseUrl);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${root}/api/tags`, { signal: AbortSignal.timeout(10_000) });
|
||||
} catch {
|
||||
throw new HttpException({ error: `Could not reach local LLM server at ${root}` }, 502);
|
||||
}
|
||||
if (!res.ok) throw new HttpException({ error: `Local LLM server error (${res.status})` }, 502);
|
||||
const data = (await res.json()) as { models?: { name?: string; size?: number }[] };
|
||||
const models = (data.models ?? []).map(m => ({ name: m.name ?? '', size: m.size ?? 0 })).filter(m => m.name);
|
||||
return { models };
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a streamed pull. Returns the upstream NDJSON body so the controller can
|
||||
* pipe Ollama's progress lines straight to the client.
|
||||
*/
|
||||
async pull(baseUrl: string | undefined, model: string): Promise<ReadableStream<Uint8Array>> {
|
||||
if (!model?.trim()) throw new HttpException({ error: 'model is required' }, 400);
|
||||
const root = this.ollamaRoot(baseUrl);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${root}/api/pull`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: model.trim(), stream: true }),
|
||||
});
|
||||
} catch {
|
||||
throw new HttpException({ error: `Could not reach local LLM server at ${root}` }, 502);
|
||||
}
|
||||
if (!res.ok || !res.body) throw new HttpException({ error: `Pull failed (${res.status})` }, 502);
|
||||
return res.body;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { LlmParseService } from './llm-parse.service';
|
||||
import { LlmLocalService } from './llm-local.service';
|
||||
import { LlmLocalController } from './llm-local.controller';
|
||||
|
||||
/** Provides the LLM booking-import fallback; imported by BookingImportModule. */
|
||||
@Module({
|
||||
controllers: [LlmLocalController],
|
||||
providers: [LlmParseService, LlmLocalService],
|
||||
exports: [LlmParseService],
|
||||
})
|
||||
export class LlmParseModule {}
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { KiReservation } from '../booking-import/kitinerary.types';
|
||||
import { createLlmClient } from './llm-client.factory';
|
||||
import { resolveLlmConfig } from './llm-config.resolver';
|
||||
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt';
|
||||
import type { LlmExtractionInput } from './llm-provider.interface';
|
||||
import { isPdf, extractText } from './text-extract';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { kiReservationSchema } from '@trek/shared';
|
||||
|
||||
const MIME_BY_EXT: Record<string, string> = {
|
||||
'.pdf': 'application/pdf',
|
||||
};
|
||||
|
||||
export interface LlmParseResult {
|
||||
kiItems: KiReservation[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrates the LLM fallback: resolve config → pick client → build input
|
||||
* (native bytes vs extracted text by the `multimodal` flag) → call provider →
|
||||
* validate the response → return schema.org `KiReservation[]` for the shared
|
||||
* mapper. Never throws for content/provider reasons — degrades to `[]` + a
|
||||
* warning, mirroring the kitinerary extractor's tolerance.
|
||||
*/
|
||||
@Injectable()
|
||||
export class LlmParseService {
|
||||
/** True when the addon is enabled AND a usable config resolves for this user. */
|
||||
isAvailable(userId: number): boolean {
|
||||
return resolveLlmConfig(userId) !== null;
|
||||
}
|
||||
|
||||
async parse(file: { buffer: Buffer; originalName: string }, userId: number): Promise<LlmParseResult> {
|
||||
const config = resolveLlmConfig(userId);
|
||||
if (!config) return { kiItems: [], warnings: ['AI parsing is not configured'] };
|
||||
|
||||
const warnings: string[] = [];
|
||||
const input: LlmExtractionInput = {
|
||||
prompt: buildSystemPrompt(),
|
||||
jsonSchema: KI_RESERVATION_JSON_SCHEMA,
|
||||
model: config.model,
|
||||
baseUrl: config.baseUrl,
|
||||
apiKey: config.apiKey,
|
||||
};
|
||||
|
||||
// Native PDF only for Anthropic (its document block reads text AND scans).
|
||||
// OpenAI-compatible servers (incl. Ollama/NuExtract) can't ingest PDFs/`file`
|
||||
// parts, so every other provider gets extracted text.
|
||||
try {
|
||||
if (config.provider === 'anthropic' && isPdf(file.originalName)) {
|
||||
input.file = { mimeType: MIME_BY_EXT['.pdf'], data: file.buffer };
|
||||
console.debug(
|
||||
`[DEBUG] Extracted (native PDF, ${file.buffer.length} bytes) sent to ${config.provider}: ${file.originalName}`,
|
||||
);
|
||||
} else {
|
||||
input.text = await extractText(file.buffer, file.originalName);
|
||||
console.debug(`[DEBUG] Extracted text from ${file.originalName} (${input.text.length} chars):\n`, input.text);
|
||||
if (!input.text.trim()) {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: no readable text found (a scanned PDF needs a cloud/vision provider)`],
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: could not read file — ${err instanceof Error ? err.message : String(err)}`],
|
||||
};
|
||||
}
|
||||
|
||||
let raw: Record<string, unknown>[];
|
||||
try {
|
||||
raw = await createLlmClient(config).extract(input);
|
||||
console.debug('[DEBUG] Raw LLM Response: ', raw);
|
||||
} catch (err) {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: AI parsing failed — ${err instanceof Error ? err.message : String(err)}`],
|
||||
};
|
||||
}
|
||||
|
||||
const kiItems: KiReservation[] = [];
|
||||
for (const node of raw) {
|
||||
const result = kiReservationSchema.safeParse(node);
|
||||
if (result.success) kiItems.push(normalizeNode(result.data) as unknown as KiReservation);
|
||||
else warnings.push(`${file.originalName}: skipped an unrecognized AI result`);
|
||||
}
|
||||
|
||||
return { kiItems, warnings };
|
||||
}
|
||||
}
|
||||
|
||||
/** Root-level keys in the schema.org reservation shape; everything else is trip-specific. */
|
||||
const ROOT_KEYS = new Set([
|
||||
'@type',
|
||||
'reservationNumber',
|
||||
'checkinTime',
|
||||
'checkoutTime',
|
||||
'pickupTime',
|
||||
'dropoffTime',
|
||||
'startTime',
|
||||
'endTime',
|
||||
'pickupLocation',
|
||||
'reservationFor',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Small models often flatten the type-specific fields (flightNumber, airline,
|
||||
* departureAirport, …) onto the reservation root instead of nesting them under
|
||||
* `reservationFor`, which is where the kitinerary mapper reads them. When
|
||||
* `reservationFor` is missing/empty, fold the non-root keys into it so the
|
||||
* existing mappers work unchanged.
|
||||
*/
|
||||
function normalizeNode(node: Record<string, unknown>): Record<string, unknown> {
|
||||
const rf = node.reservationFor;
|
||||
if (rf && typeof rf === 'object' && Object.keys(rf as object).length > 0) return node;
|
||||
|
||||
const out: Record<string, unknown> = {};
|
||||
const reservationFor: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(node)) {
|
||||
if (ROOT_KEYS.has(k)) out[k] = v;
|
||||
else reservationFor[k] = v;
|
||||
}
|
||||
// Nothing to fold (no flattened type fields) — leave the node as-is.
|
||||
if (Object.keys(reservationFor).length === 0) return node;
|
||||
out.reservationFor = reservationFor;
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { KI_RESERVATION_JSON_SCHEMA, KI_RESERVATION_TYPES } from '@trek/shared';
|
||||
|
||||
export { KI_RESERVATION_JSON_SCHEMA };
|
||||
|
||||
/**
|
||||
* System instructions telling the model to emit schema.org reservation JSON-LD
|
||||
* in exactly the shape the kitinerary binary produces — so the result feeds the
|
||||
* same `mapReservations()` mapper. Pure (no I/O) so it's unit-testable.
|
||||
*/
|
||||
export function buildSystemPrompt(): string {
|
||||
return [
|
||||
'You extract travel reservations from a document (a booking confirmation, ticket, or itinerary).',
|
||||
'Return ONLY a JSON object of the form { "reservations": [ ... ] } — no prose, no markdown.',
|
||||
'Each reservation is a schema.org JSON-LD object whose "@type" is one of:',
|
||||
KI_RESERVATION_TYPES.map((t) => ` - ${t}`).join('\n'),
|
||||
'Put the booking/confirmation code in "reservationNumber" on each reservation.',
|
||||
'All dates/times are plain ISO 8601 local strings, e.g. "2026-06-11T10:00:00" (no timezone wrapper objects).',
|
||||
'IMPORTANT: nest the type-specific fields INSIDE a "reservationFor" object — do NOT place them at the top level of the reservation.',
|
||||
'Populate "reservationFor" with the type-specific fields:',
|
||||
' FlightReservation: { flightNumber, airline:{name,iataCode}, departureAirport:{iataCode,name,geo:{latitude,longitude}}, arrivalAirport:{...}, departureTime, arrivalTime }',
|
||||
' TrainReservation: { trainNumber, trainName, departureStation:{name,geo}, arrivalStation:{name,geo}, departureTime, arrivalTime }',
|
||||
' BusReservation: { busNumber, busName, departureBusStop:{name,geo}, arrivalBusStop:{name,geo}, departureTime, arrivalTime }',
|
||||
' BoatReservation: { name, departureBoatTerminal:{name,geo}, arrivalBoatTerminal:{name,geo}, departureTime, arrivalTime }',
|
||||
' LodgingReservation: { name, address, geo:{latitude,longitude}, telephone, url } — put check-in/out in root "checkinTime"/"checkoutTime"',
|
||||
' FoodEstablishmentReservation: { name, address, geo, telephone, url } — put booking time in root "startTime"/"endTime"',
|
||||
' RentalCarReservation: { name, model, make, rentalCompany:{name} } — put pickup/dropoff in root "pickupTime"/"dropoffTime" and "pickupLocation":{name,address,geo}',
|
||||
' EventReservation / TouristAttractionVisit: { name, startDate, endDate, location:{name,address,geo} }',
|
||||
'Extract EVERY flight/segment in the document, including return legs — a round trip has TWO OR MORE flights, and each row of a flight table is a separate reservation. Do NOT stop after the first.',
|
||||
"Each flight shares the booking's reservationNumber. Use the date shown for that specific flight as its departureTime; if a flight lists only one date (no separate arrival time), leave arrivalTime null — never reuse another flight's date.",
|
||||
'If the document contains no recognizable reservation, return { "reservations": [] }.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Short user-turn instruction that accompanies the document content. */
|
||||
export const USER_INSTRUCTION = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
|
||||
@@ -0,0 +1,30 @@
|
||||
/** A single binary file (e.g. a PDF) sent natively to a multimodal provider. */
|
||||
export interface LlmExtractionFile {
|
||||
mimeType: string;
|
||||
data: Buffer;
|
||||
}
|
||||
|
||||
/** Everything a provider client needs to extract reservations from one document. */
|
||||
export interface LlmExtractionInput {
|
||||
/** System instructions enumerating the schema.org shape (see llm-prompt.ts). */
|
||||
prompt: string;
|
||||
/** JSON Schema describing `{ reservations: KiReservation[] }`. */
|
||||
jsonSchema: object;
|
||||
model: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
/** Pre-extracted text (text-like files, or text-only-model mode). */
|
||||
text?: string;
|
||||
/** Native binary (PDF) for multimodal providers. */
|
||||
file?: LlmExtractionFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* A provider client turns one document into raw schema.org reservation objects.
|
||||
* It returns the parsed `reservations` array (best-effort: `[]` on a malformed or
|
||||
* empty response, never throwing for content reasons). The caller validates and
|
||||
* maps via the shared kitinerary mapper.
|
||||
*/
|
||||
export interface LlmExtractionClient {
|
||||
extract(input: LlmExtractionInput): Promise<Record<string, unknown>[]>;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { extname } from 'node:path';
|
||||
import { PDFParse } from 'pdf-parse';
|
||||
|
||||
/** File extensions whose bytes are inherently text and can be decoded directly. */
|
||||
const TEXT_LIKE = new Set(['.txt', '.html', '.htm', '.eml']);
|
||||
|
||||
export function isTextLike(fileName: string): boolean {
|
||||
return TEXT_LIKE.has(extname(fileName).toLowerCase());
|
||||
}
|
||||
|
||||
export function isPdf(fileName: string): boolean {
|
||||
return extname(fileName).toLowerCase() === '.pdf';
|
||||
}
|
||||
|
||||
/** Strip HTML/XML tags and collapse whitespace for a cleaner LLM prompt. */
|
||||
function stripMarkup(s: string): string {
|
||||
return s
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Extract the embedded text layer from a PDF (empty for scanned/image-only PDFs). */
|
||||
async function extractPdfText(buffer: Buffer): Promise<string> {
|
||||
const parser = new PDFParse({ data: new Uint8Array(buffer) });
|
||||
try {
|
||||
// Space (not tab) between same-line items reads more naturally for the LLM.
|
||||
const res = await parser.getText({ cellSeparator: ' ' });
|
||||
return cleanPdfText(res.text ?? '');
|
||||
} finally {
|
||||
await parser.destroy?.();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up pdf-parse output for the LLM:
|
||||
* - strip `-- N of M --` page markers
|
||||
* - normalize whitespace/tabs
|
||||
* - collapse letter-spaced UPPERCASE runs ("A M S T E R D A M" → "AMSTERDAM"),
|
||||
* a common PDF kerning artifact that otherwise hides booking fields
|
||||
*/
|
||||
function cleanPdfText(text: string): string {
|
||||
return text
|
||||
.replace(/^\s*-+\s*\d+\s+of\s+\d+\s*-+\s*$/gim, '')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/\b(?:[A-Z] ){2,}[A-Z]\b/g, m => m.replace(/ /g, ''))
|
||||
.replace(/ *\n */g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract text from a booking file for the OpenAI-compatible/local LLM path
|
||||
* (Ollama can't ingest PDFs or `file` parts, so everything becomes text).
|
||||
* - txt/html/htm/eml → decoded (markup stripped)
|
||||
* - pdf → embedded text layer via pdf-parse
|
||||
* - anything else → best-effort UTF-8 decode
|
||||
* A scanned/image-only PDF yields empty text — that case needs a vision provider
|
||||
* (Anthropic reads PDFs natively).
|
||||
*/
|
||||
export async function extractText(buffer: Buffer, fileName: string): Promise<string> {
|
||||
const ext = extname(fileName).toLowerCase();
|
||||
if (isPdf(fileName)) return extractPdfText(buffer);
|
||||
const raw = buffer.toString('utf8');
|
||||
if (ext === '.html' || ext === '.htm' || ext === '.eml') return stripMarkup(raw);
|
||||
return raw.trim();
|
||||
}
|
||||
@@ -11,6 +11,8 @@ import { revokeUserSessions, revokeUserSessionsForClient } from '../mcp';
|
||||
import { deleteUserCompletely } from './userCleanupService';
|
||||
import { validatePassword } from './passwordPolicy';
|
||||
import { getPhotoProviderConfig } from './memories/helpersService';
|
||||
import { ADDON_IDS } from '../addons';
|
||||
import { prepareLlmAddonConfigForWrite, maskLlmAddonConfig } from './llmConfig';
|
||||
import { send as sendNotification } from './notificationService';
|
||||
import { resolveAuthToggles } from './authService';
|
||||
|
||||
@@ -670,7 +672,13 @@ export function listAddons() {
|
||||
}
|
||||
|
||||
return [
|
||||
...addons.map(a => ({ ...a, enabled: !!a.enabled, config: JSON.parse(a.config || '{}') })),
|
||||
...addons.map(a => ({
|
||||
...a,
|
||||
enabled: !!a.enabled,
|
||||
config: a.id === ADDON_IDS.LLM_PARSING
|
||||
? maskLlmAddonConfig(JSON.parse(a.config || '{}'))
|
||||
: JSON.parse(a.config || '{}'),
|
||||
})),
|
||||
...providers.map(p => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
@@ -702,7 +710,14 @@ export function updateAddon(id: string, data: { enabled?: boolean; config?: Reco
|
||||
|
||||
if (addon) {
|
||||
if (data.enabled !== undefined) db.prepare('UPDATE addons SET enabled = ? WHERE id = ?').run(data.enabled ? 1 : 0, id);
|
||||
if (data.config !== undefined) db.prepare('UPDATE addons SET config = ? WHERE id = ?').run(JSON.stringify(data.config), id);
|
||||
if (data.config !== undefined) {
|
||||
// The AI-parsing addon holds an API key — encrypt it at rest and preserve
|
||||
// the stored key when the client echoes the mask sentinel (see llmConfig.ts).
|
||||
const configToStore = id === ADDON_IDS.LLM_PARSING
|
||||
? prepareLlmAddonConfigForWrite(data.config, JSON.parse(addon.config || '{}'))
|
||||
: data.config;
|
||||
db.prepare('UPDATE addons SET config = ? WHERE id = ?').run(JSON.stringify(configToStore), id);
|
||||
}
|
||||
} else {
|
||||
if (data.enabled !== undefined) db.prepare('UPDATE photo_providers SET enabled = ? WHERE id = ?').run(data.enabled ? 1 : 0, id);
|
||||
}
|
||||
@@ -710,7 +725,13 @@ export function updateAddon(id: string, data: { enabled?: boolean; config?: Reco
|
||||
const updatedAddon = db.prepare('SELECT * FROM addons WHERE id = ?').get(id) as Addon | undefined;
|
||||
const updatedProvider = db.prepare('SELECT * FROM photo_providers WHERE id = ?').get(id) as { id: string; name: string; description?: string | null; icon: string; enabled: number; sort_order: number } | undefined;
|
||||
const updated = updatedAddon
|
||||
? { ...updatedAddon, enabled: !!updatedAddon.enabled, config: JSON.parse(updatedAddon.config || '{}') }
|
||||
? {
|
||||
...updatedAddon,
|
||||
enabled: !!updatedAddon.enabled,
|
||||
config: updatedAddon.id === ADDON_IDS.LLM_PARSING
|
||||
? maskLlmAddonConfig(JSON.parse(updatedAddon.config || '{}'))
|
||||
: JSON.parse(updatedAddon.config || '{}'),
|
||||
}
|
||||
: updatedProvider
|
||||
? {
|
||||
id: updatedProvider.id,
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { maybe_encrypt_api_key, decrypt_api_key } from './apiKeyCrypto';
|
||||
|
||||
/**
|
||||
* Shared types + helpers for the `llm_parsing` addon configuration.
|
||||
*
|
||||
* Config can live in two places (resolution happens in
|
||||
* server/src/nest/llm-parse/llm-config.resolver.ts):
|
||||
* - instance-wide: the `llm_parsing` addon's `config` JSON (admin-set, wins)
|
||||
* - per-user: the `llm_*` keys in the per-user settings table (fallback)
|
||||
*
|
||||
* The API key is encrypted at rest (reusing apiKeyCrypto) and never returned to
|
||||
* the client in plaintext — it is masked with MASKED_VALUE, matching the
|
||||
* per-user encrypted-settings pattern in settingsService.ts.
|
||||
*/
|
||||
|
||||
export type LlmProvider = 'local' | 'openai' | 'anthropic';
|
||||
|
||||
/** Fully-resolved config the clients consume. */
|
||||
export interface ResolvedLlmConfig {
|
||||
provider: LlmProvider;
|
||||
model: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
multimodal: boolean;
|
||||
}
|
||||
|
||||
/** Shape of the admin instance config stored in `addons.config` (apiKey encrypted). */
|
||||
export interface LlmAddonConfig {
|
||||
provider?: LlmProvider;
|
||||
model?: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
multimodal?: boolean;
|
||||
}
|
||||
|
||||
export const LLM_PROVIDERS: LlmProvider[] = ['local', 'openai', 'anthropic'];
|
||||
export const MASKED_VALUE = '••••••••';
|
||||
|
||||
/**
|
||||
* Prepare an admin config blob for persistence: encrypt a freshly-entered apiKey,
|
||||
* and preserve the previously-stored (already-encrypted) key when the client
|
||||
* echoes back the mask sentinel (i.e. the user didn't change it).
|
||||
*/
|
||||
export function prepareLlmAddonConfigForWrite(
|
||||
incoming: Record<string, unknown>,
|
||||
existingStored: Record<string, unknown> | undefined,
|
||||
): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = { ...incoming };
|
||||
const key = incoming.apiKey;
|
||||
if (key === undefined || key === null || key === '' || key === MASKED_VALUE) {
|
||||
// Keep the existing encrypted key untouched (mask echoed or no key supplied).
|
||||
if (existingStored && 'apiKey' in existingStored) out.apiKey = existingStored.apiKey;
|
||||
else delete out.apiKey;
|
||||
} else {
|
||||
out.apiKey = maybe_encrypt_api_key(String(key)) ?? String(key);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Mask the apiKey for any client-facing response (never leak plaintext). */
|
||||
export function maskLlmAddonConfig(config: Record<string, unknown>): Record<string, unknown> {
|
||||
if (config && config.apiKey) return { ...config, apiKey: MASKED_VALUE };
|
||||
return config;
|
||||
}
|
||||
|
||||
/** Decrypt the stored apiKey for server-side use (resolver only). */
|
||||
export function decryptLlmApiKey(stored: unknown): string | undefined {
|
||||
if (!stored) return undefined;
|
||||
return decrypt_api_key(stored) ?? undefined;
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
import { db } from '../db/database';
|
||||
import { decrypt_api_key, maybe_encrypt_api_key } from './apiKeyCrypto';
|
||||
|
||||
const ENCRYPTED_SETTING_KEYS = new Set(['webhook_url', 'ntfy_token', 'mapbox_access_token']);
|
||||
const ENCRYPTED_SETTING_KEYS = new Set(['webhook_url', 'ntfy_token', 'mapbox_access_token', 'llm_api_key']);
|
||||
// Encrypted keys that are masked (••••••••) when returned to the client.
|
||||
// Keys not in this set but in ENCRYPTED_SETTING_KEYS are decrypted and returned.
|
||||
const MASKED_SETTING_KEYS = new Set(['webhook_url', 'ntfy_token']);
|
||||
const MASKED_SETTING_KEYS = new Set(['webhook_url', 'ntfy_token', 'llm_api_key']);
|
||||
|
||||
export const DEFAULTABLE_USER_SETTING_KEYS = [
|
||||
'temperature_unit',
|
||||
@@ -24,6 +24,16 @@ export const DEFAULTABLE_USER_SETTING_KEYS = [
|
||||
'maplibre_style',
|
||||
'mapbox_3d_enabled',
|
||||
'mapbox_quality_mode',
|
||||
// Per-user LLM fallback config for booking import (used when the admin has not
|
||||
// set instance-wide config on the llm_parsing addon). See llmConfig.ts.
|
||||
'llm_provider',
|
||||
'llm_model',
|
||||
'llm_base_url',
|
||||
'llm_multimodal',
|
||||
'llm_api_key',
|
||||
// "Always retry with AI" toggle — when on, the preview auto-runs the LLM on
|
||||
// files kitinerary returns nothing for.
|
||||
'llm_always_retry',
|
||||
] as const;
|
||||
|
||||
type DefaultableKey = typeof DEFAULTABLE_USER_SETTING_KEYS[number];
|
||||
@@ -34,9 +44,10 @@ const VALID_VALUES: Partial<Record<DefaultableKey, unknown[]>> = {
|
||||
time_format: ['12h', '24h'],
|
||||
dark_mode: [true, false, 'light', 'dark', 'auto'],
|
||||
map_provider: ['leaflet', 'mapbox-gl', 'maplibre-gl'],
|
||||
llm_provider: ['local', 'openai', 'anthropic'],
|
||||
};
|
||||
|
||||
const BOOLEAN_KEYS = new Set<DefaultableKey>(['blur_booking_codes', 'mapbox_3d_enabled', 'mapbox_quality_mode']);
|
||||
const BOOLEAN_KEYS = new Set<DefaultableKey>(['blur_booking_codes', 'mapbox_3d_enabled', 'mapbox_quality_mode', 'llm_multimodal', 'llm_always_retry']);
|
||||
|
||||
function parseValue(raw: string): unknown {
|
||||
try { return JSON.parse(raw); } catch { return raw; }
|
||||
@@ -157,3 +168,21 @@ export function bulkUpsertSettings(userId: number, settings: Record<string, unkn
|
||||
}
|
||||
return Object.keys(settings).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a single per-user setting, decrypting it if it's an encrypted key.
|
||||
* Unlike getUserSettings (which MASKS encrypted keys for the client), this
|
||||
* returns the plaintext — for server-side use only (e.g. the LLM config
|
||||
* resolver needs the real API key). Returns null when unset.
|
||||
*/
|
||||
export function getDecryptedUserSetting(userId: number, key: string): string | null {
|
||||
const row = db.prepare('SELECT value FROM settings WHERE user_id = ? AND key = ?').get(userId, key) as { value: string } | undefined;
|
||||
if (!row || row.value === '' || row.value == null) return null;
|
||||
if (ENCRYPTED_SETTING_KEYS.has(key)) return decrypt_api_key(row.value);
|
||||
try {
|
||||
const parsed = JSON.parse(row.value);
|
||||
return typeof parsed === 'string' ? parsed : row.value;
|
||||
} catch {
|
||||
return row.value;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { HttpException } from '@nestjs/common';
|
||||
import { BookingImportController } from '../../../../src/nest/booking-import/booking-import.controller';
|
||||
import type { BookingImportService } from '../../../../src/nest/booking-import/booking-import.service';
|
||||
import type { User } from '../../../../src/types';
|
||||
|
||||
const user = { id: 1, role: 'user' } as User;
|
||||
const file = (name = 'a.pdf') => ({ originalname: name, buffer: Buffer.from('x') } as Express.Multer.File);
|
||||
|
||||
function make(over: Partial<BookingImportService> = {}) {
|
||||
const svc = {
|
||||
verifyTripAccess: vi.fn(() => ({ user_id: 1 })),
|
||||
canEdit: vi.fn(() => true),
|
||||
isAvailable: vi.fn(() => true),
|
||||
aiAvailable: vi.fn(() => true),
|
||||
preview: vi.fn(async () => ({ items: [], warnings: [], files: [] })),
|
||||
...over,
|
||||
} as unknown as BookingImportService;
|
||||
return { c: new BookingImportController(svc), svc };
|
||||
}
|
||||
|
||||
async function status(fn: () => Promise<unknown>): Promise<number> {
|
||||
try { await fn(); } catch (e) { expect(e).toBeInstanceOf(HttpException); return (e as HttpException).getStatus(); }
|
||||
throw new Error('expected throw');
|
||||
}
|
||||
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
describe('BookingImportController.preview', () => {
|
||||
it('rejects an invalid mode with 400', async () => {
|
||||
const { c } = make();
|
||||
expect(await status(() => c.preview(user, 't1', [file()], 'bogus'))).toBe(400);
|
||||
});
|
||||
|
||||
it('returns 409 for force-ai when AI is not configured', async () => {
|
||||
const { c } = make({ aiAvailable: vi.fn(() => false) as any });
|
||||
expect(await status(() => c.preview(user, 't1', [file()], 'force-ai'))).toBe(409);
|
||||
});
|
||||
|
||||
it('returns 503 for no-ai when the extractor is unavailable', async () => {
|
||||
const { c } = make({ isAvailable: vi.fn(() => false) as any });
|
||||
expect(await status(() => c.preview(user, 't1', [file()], 'no-ai'))).toBe(503);
|
||||
});
|
||||
|
||||
it('returns 400 when no files are uploaded', async () => {
|
||||
const { c } = make();
|
||||
expect(await status(() => c.preview(user, 't1', [], 'no-ai'))).toBe(400);
|
||||
});
|
||||
|
||||
it('passes the parsed mode and user id through to the service', async () => {
|
||||
const { c, svc } = make();
|
||||
await c.preview(user, 't1', [file()], 'fallback-on-empty');
|
||||
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'fallback-on-empty', 1);
|
||||
});
|
||||
|
||||
it('defaults the mode to no-ai when omitted', async () => {
|
||||
const { c, svc } = make();
|
||||
await c.preview(user, 't1', [file()], undefined);
|
||||
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'no-ai', 1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,79 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { HttpException } from '@nestjs/common';
|
||||
|
||||
// Mock the heavy side-effect imports so the service module loads cleanly; the
|
||||
// preview() path under test only touches the extractor + llmParse deps.
|
||||
vi.mock('../../../../src/db/database', () => ({ db: { prepare: vi.fn() }, closeDb: () => {}, reinitialize: () => {} }));
|
||||
vi.mock('../../../../src/websocket', () => ({ broadcast: vi.fn() }));
|
||||
vi.mock('../../../../src/services/permissions', () => ({ checkPermission: vi.fn(() => true) }));
|
||||
vi.mock('../../../../src/services/tripAccess', () => ({ verifyTripAccess: vi.fn() }));
|
||||
vi.mock('../../../../src/services/reservationService', () => ({ createReservation: vi.fn() }));
|
||||
vi.mock('../../../../src/services/placeService', () => ({ createPlace: vi.fn() }));
|
||||
vi.mock('../../../../src/services/mapsService', () => ({ searchNominatim: vi.fn() }));
|
||||
|
||||
import { BookingImportService } from '../../../../src/nest/booking-import/booking-import.service';
|
||||
|
||||
const HOTEL_KI = { '@type': 'LodgingReservation', reservationNumber: 'ABC', reservationFor: { name: 'Hotel X' }, checkinTime: '2026-06-11T15:00', checkoutTime: '2026-06-12T11:00' };
|
||||
const file = (name = 'a.pdf') => ({ buffer: Buffer.from('x'), originalname: name } as any);
|
||||
|
||||
function make(opts: { kit?: boolean; ai?: boolean; extract?: any; parse?: any }) {
|
||||
const extractor = { isAvailable: () => opts.kit ?? false, extract: vi.fn(opts.extract ?? (async () => [])) };
|
||||
const llmParse = { isAvailable: () => opts.ai ?? false, parse: vi.fn(opts.parse ?? (async () => ({ kiItems: [], warnings: [] }))) };
|
||||
return { svc: new BookingImportService(extractor as any, llmParse as any), extractor, llmParse };
|
||||
}
|
||||
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
describe('BookingImportService.preview', () => {
|
||||
it('no-ai: maps kitinerary items, does not force needs_review, reports aiUsed:false', async () => {
|
||||
const { svc, llmParse } = make({ kit: true, ai: false, extract: async () => [HOTEL_KI] });
|
||||
const res = await svc.preview([file()], 'no-ai', 1);
|
||||
expect(res.items).toHaveLength(1);
|
||||
expect(res.items[0].needs_review).toBeFalsy();
|
||||
expect(res.files).toEqual([{ fileName: 'a.pdf', aiAvailable: false, aiUsed: false }]);
|
||||
expect(llmParse.parse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('throws 503 when neither parser is available', async () => {
|
||||
const { svc } = make({ kit: false, ai: false });
|
||||
try {
|
||||
await svc.preview([file()], 'no-ai', 1);
|
||||
throw new Error('expected throw');
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(HttpException);
|
||||
expect((err as HttpException).getStatus()).toBe(503);
|
||||
}
|
||||
});
|
||||
|
||||
it('fallback-on-empty: runs the LLM when kitinerary finds nothing and flags needs_review', async () => {
|
||||
const { svc, extractor, llmParse } = make({
|
||||
kit: true, ai: true,
|
||||
extract: async () => [],
|
||||
parse: async () => ({ kiItems: [HOTEL_KI], warnings: [] }),
|
||||
});
|
||||
const res = await svc.preview([file()], 'fallback-on-empty', 1);
|
||||
expect(extractor.extract).toHaveBeenCalled();
|
||||
expect(llmParse.parse).toHaveBeenCalled();
|
||||
expect(res.items).toHaveLength(1);
|
||||
expect(res.items[0].needs_review).toBe(true);
|
||||
expect(res.files![0]).toEqual({ fileName: 'a.pdf', aiAvailable: true, aiUsed: true });
|
||||
});
|
||||
|
||||
it('fallback-on-empty: skips the LLM when kitinerary already found items', async () => {
|
||||
const { svc, llmParse } = make({ kit: true, ai: true, extract: async () => [HOTEL_KI] });
|
||||
const res = await svc.preview([file()], 'fallback-on-empty', 1);
|
||||
expect(llmParse.parse).not.toHaveBeenCalled();
|
||||
expect(res.files![0].aiUsed).toBe(false);
|
||||
});
|
||||
|
||||
it('force-ai: skips kitinerary entirely and uses the LLM', async () => {
|
||||
const { svc, extractor, llmParse } = make({
|
||||
kit: true, ai: true,
|
||||
parse: async () => ({ kiItems: [HOTEL_KI], warnings: [] }),
|
||||
});
|
||||
const res = await svc.preview([file()], 'force-ai', 1);
|
||||
expect(extractor.extract).not.toHaveBeenCalled();
|
||||
expect(llmParse.parse).toHaveBeenCalled();
|
||||
expect(res.items[0].needs_review).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { OpenAiCompatibleClient } from '../../../../src/nest/llm-parse/clients/openai-compatible.client';
|
||||
import { AnthropicClient } from '../../../../src/nest/llm-parse/clients/anthropic.client';
|
||||
import type { LlmExtractionInput } from '../../../../src/nest/llm-parse/llm-provider.interface';
|
||||
|
||||
const baseInput: LlmExtractionInput = {
|
||||
prompt: 'system',
|
||||
jsonSchema: { type: 'object' },
|
||||
model: 'm',
|
||||
text: 'Flight AB123',
|
||||
};
|
||||
|
||||
function mockFetch(impl: (url: string, init: RequestInit) => Promise<Response> | Response) {
|
||||
const fn = vi.fn(impl as any);
|
||||
vi.stubGlobal('fetch', fn);
|
||||
return fn;
|
||||
}
|
||||
|
||||
function jsonResponse(body: unknown, ok = true, status = 200): Response {
|
||||
return { ok, status, json: async () => body, text: async () => JSON.stringify(body) } as unknown as Response;
|
||||
}
|
||||
|
||||
beforeEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe('OpenAiCompatibleClient', () => {
|
||||
it('posts to {baseUrl}/chat/completions and returns the reservations array', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({ choices: [{ message: { content: JSON.stringify({ reservations: [{ '@type': 'FlightReservation' }] }) } }] }),
|
||||
);
|
||||
const out = await new OpenAiCompatibleClient().extract({ ...baseInput, baseUrl: 'http://localhost:11434/v1/' });
|
||||
expect(out).toEqual([{ '@type': 'FlightReservation' }]);
|
||||
expect(fetchFn.mock.calls[0][0]).toBe('http://localhost:11434/v1/chat/completions');
|
||||
});
|
||||
|
||||
it('tolerates code-fenced JSON', async () => {
|
||||
mockFetch(() =>
|
||||
jsonResponse({ choices: [{ message: { content: '```json\n{"reservations":[{"@type":"TrainReservation"}]}\n```' } }] }),
|
||||
);
|
||||
const out = await new OpenAiCompatibleClient().extract(baseInput);
|
||||
expect(out).toEqual([{ '@type': 'TrainReservation' }]);
|
||||
});
|
||||
|
||||
it('returns [] on malformed content', async () => {
|
||||
mockFetch(() => jsonResponse({ choices: [{ message: { content: 'not json' } }] }));
|
||||
expect(await new OpenAiCompatibleClient().extract(baseInput)).toEqual([]);
|
||||
});
|
||||
|
||||
it('throws on non-2xx', async () => {
|
||||
mockFetch(() => jsonResponse({ error: 'bad' }, false, 401));
|
||||
await expect(new OpenAiCompatibleClient().extract(baseInput)).rejects.toThrow(/401/);
|
||||
});
|
||||
|
||||
it('sends an image natively as image_url but never a file/pdf part', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
|
||||
await new OpenAiCompatibleClient().extract({ ...baseInput, file: { mimeType: 'image/png', data: Buffer.from('IMG') } });
|
||||
let parts = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string).messages[1].content;
|
||||
expect(parts.some((p: any) => p.type === 'image_url')).toBe(true);
|
||||
expect(parts.some((p: any) => p.type === 'file')).toBe(false);
|
||||
|
||||
// A PDF must NOT be sent as a content part (Ollama rejects it).
|
||||
await new OpenAiCompatibleClient().extract({ ...baseInput, file: { mimeType: 'application/pdf', data: Buffer.from('PDF') } });
|
||||
parts = JSON.parse((fetchFn.mock.calls[1][1] as RequestInit).body as string).messages[1].content;
|
||||
expect(parts.every((p: any) => p.type !== 'file' && p.type !== 'image_url')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AnthropicClient', () => {
|
||||
it('forces the emit_reservations tool and reads its input', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [{ '@type': 'LodgingReservation' }] } }] }),
|
||||
);
|
||||
const out = await new AnthropicClient().extract(baseInput);
|
||||
expect(out).toEqual([{ '@type': 'LodgingReservation' }]);
|
||||
const body = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string);
|
||||
expect(body.tool_choice).toEqual({ type: 'tool', name: 'emit_reservations' });
|
||||
expect(body.tools[0].name).toBe('emit_reservations');
|
||||
});
|
||||
|
||||
it('throws on a refusal stop_reason', async () => {
|
||||
mockFetch(() => jsonResponse({ stop_reason: 'refusal', content: [] }));
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/declined/i);
|
||||
});
|
||||
|
||||
it('throws on non-2xx', async () => {
|
||||
mockFetch(() => jsonResponse({ error: 'bad' }, false, 500));
|
||||
await expect(new AnthropicClient().extract(baseInput)).rejects.toThrow(/500/);
|
||||
});
|
||||
|
||||
it('sends a native pdf as a base64 document block', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }));
|
||||
await new AnthropicClient().extract({ ...baseInput, file: { mimeType: 'application/pdf', data: Buffer.from('PDF') } });
|
||||
const body = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string);
|
||||
const blocks = body.messages[0].content;
|
||||
expect(blocks.some((b: any) => b.type === 'document' && b.source.type === 'base64')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
const { dbMock } = vi.hoisted(() => {
|
||||
const stmt = { get: vi.fn() };
|
||||
return { dbMock: { prepare: vi.fn(() => stmt), _stmt: stmt } };
|
||||
});
|
||||
vi.mock('../../../../src/db/database', () => ({ db: dbMock, closeDb: () => {}, reinitialize: () => {} }));
|
||||
|
||||
const { isAddonEnabled } = vi.hoisted(() => ({ isAddonEnabled: vi.fn() }));
|
||||
vi.mock('../../../../src/services/adminService', () => ({ isAddonEnabled }));
|
||||
|
||||
const { getUserSettings, getDecryptedUserSetting } = vi.hoisted(() => ({
|
||||
getUserSettings: vi.fn(() => ({}) as Record<string, unknown>),
|
||||
getDecryptedUserSetting: vi.fn(() => null as string | null),
|
||||
}));
|
||||
vi.mock('../../../../src/services/settingsService', () => ({ getUserSettings, getDecryptedUserSetting }));
|
||||
|
||||
import { resolveLlmConfig } from '../../../../src/nest/llm-parse/llm-config.resolver';
|
||||
|
||||
function setInstanceConfig(config: unknown) {
|
||||
dbMock._stmt.get.mockReturnValue(config === undefined ? undefined : { config: JSON.stringify(config) });
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
isAddonEnabled.mockReturnValue(true);
|
||||
setInstanceConfig(undefined);
|
||||
getUserSettings.mockReturnValue({});
|
||||
getDecryptedUserSetting.mockReturnValue(null);
|
||||
});
|
||||
|
||||
describe('resolveLlmConfig', () => {
|
||||
it('returns null when the addon is disabled', () => {
|
||||
isAddonEnabled.mockReturnValue(false);
|
||||
expect(resolveLlmConfig(1)).toBeNull();
|
||||
});
|
||||
|
||||
it('uses instance config when present (and decrypts the key)', () => {
|
||||
setInstanceConfig({ provider: 'anthropic', model: 'claude-opus-4-8', apiKey: 'sk-plain', multimodal: true });
|
||||
expect(resolveLlmConfig(1)).toEqual({
|
||||
provider: 'anthropic',
|
||||
model: 'claude-opus-4-8',
|
||||
baseUrl: undefined,
|
||||
apiKey: 'sk-plain',
|
||||
multimodal: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to per-user config when instance config is incomplete', () => {
|
||||
setInstanceConfig({ provider: 'anthropic' }); // no model → not usable
|
||||
getUserSettings.mockReturnValue({ llm_provider: 'local', llm_model: 'nuextract', llm_base_url: 'http://x/v1', llm_multimodal: true });
|
||||
getDecryptedUserSetting.mockReturnValue('user-key');
|
||||
expect(resolveLlmConfig(7)).toEqual({
|
||||
provider: 'local',
|
||||
model: 'nuextract',
|
||||
baseUrl: 'http://x/v1',
|
||||
apiKey: 'user-key',
|
||||
multimodal: true,
|
||||
});
|
||||
expect(getDecryptedUserSetting).toHaveBeenCalledWith(7, 'llm_api_key');
|
||||
});
|
||||
|
||||
it('returns null when neither instance nor user config is usable', () => {
|
||||
getUserSettings.mockReturnValue({ llm_provider: 'openai' }); // no model
|
||||
expect(resolveLlmConfig(1)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { HttpException } from '@nestjs/common';
|
||||
import { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
|
||||
|
||||
const svc = () => new LlmLocalService();
|
||||
|
||||
function mockFetch(impl: any) {
|
||||
const fn = vi.fn(impl);
|
||||
vi.stubGlobal('fetch', fn);
|
||||
return fn;
|
||||
}
|
||||
|
||||
beforeEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe('LlmLocalService.ollamaRoot', () => {
|
||||
it('strips a trailing /v1 and slashes', () => {
|
||||
expect(svc().ollamaRoot('http://localhost:11434/v1')).toBe('http://localhost:11434');
|
||||
expect(svc().ollamaRoot('http://localhost:11434/v1/')).toBe('http://localhost:11434');
|
||||
expect(svc().ollamaRoot('http://host:1/')).toBe('http://host:1');
|
||||
});
|
||||
|
||||
it('defaults when no base URL is given', () => {
|
||||
expect(svc().ollamaRoot(undefined)).toBe('http://localhost:11434');
|
||||
});
|
||||
|
||||
it('rejects non-http(s) and invalid URLs', () => {
|
||||
expect(() => svc().ollamaRoot('ftp://x')).toThrow(HttpException);
|
||||
expect(() => svc().ollamaRoot('not a url')).toThrow(HttpException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmLocalService.listModels', () => {
|
||||
it('returns named models from /api/tags', async () => {
|
||||
const fetchFn = mockFetch(async () => ({ ok: true, json: async () => ({ models: [{ name: 'nuextract', size: 100 }, { name: '' }] }) }));
|
||||
const out = await svc().listModels('http://localhost:11434/v1');
|
||||
expect(out.models).toEqual([{ name: 'nuextract', size: 100 }]);
|
||||
expect(fetchFn.mock.calls[0][0]).toBe('http://localhost:11434/api/tags');
|
||||
});
|
||||
|
||||
it('502s when the server is unreachable', async () => {
|
||||
mockFetch(async () => { throw new Error('ECONNREFUSED'); });
|
||||
await expect(svc().listModels('http://localhost:11434')).rejects.toThrow(HttpException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmLocalService.pull', () => {
|
||||
it('requires a model', async () => {
|
||||
await expect(svc().pull('http://localhost:11434', '')).rejects.toThrow(HttpException);
|
||||
});
|
||||
|
||||
it('posts to /api/pull and returns the stream body', async () => {
|
||||
const body = {} as ReadableStream<Uint8Array>;
|
||||
const fetchFn = mockFetch(async () => ({ ok: true, body }));
|
||||
const out = await svc().pull('http://localhost:11434/v1', 'nuextract');
|
||||
expect(out).toBe(body);
|
||||
expect(fetchFn.mock.calls[0][0]).toBe('http://localhost:11434/api/pull');
|
||||
const init = fetchFn.mock.calls[0][1];
|
||||
expect(JSON.parse(init.body)).toEqual({ model: 'nuextract', stream: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
|
||||
const { resolveLlmConfig } = vi.hoisted(() => ({ resolveLlmConfig: vi.fn() }));
|
||||
vi.mock('../../../../src/nest/llm-parse/llm-config.resolver', () => ({ resolveLlmConfig }));
|
||||
|
||||
const { createLlmClient, extract } = vi.hoisted(() => {
|
||||
const extract = vi.fn();
|
||||
return { createLlmClient: vi.fn(() => ({ extract })), extract };
|
||||
});
|
||||
vi.mock('../../../../src/nest/llm-parse/llm-client.factory', () => ({ createLlmClient }));
|
||||
|
||||
const { extractText } = vi.hoisted(() => ({ extractText: vi.fn(async () => 'Flight AB123') }));
|
||||
vi.mock('../../../../src/nest/llm-parse/text-extract', async (orig) => {
|
||||
const actual = await orig() as Record<string, unknown>;
|
||||
return { ...actual, extractText };
|
||||
});
|
||||
|
||||
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
|
||||
const cfg = (over: Record<string, unknown> = {}) => ({ provider: 'openai', model: 'm', multimodal: false, ...over });
|
||||
const svc = () => new LlmParseService();
|
||||
const file = (name: string, body = 'Flight AB123') => ({ buffer: Buffer.from(body), originalName: name });
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
resolveLlmConfig.mockReturnValue(cfg());
|
||||
extract.mockResolvedValue([{ '@type': 'FlightReservation' }]);
|
||||
extractText.mockResolvedValue('Flight AB123');
|
||||
});
|
||||
|
||||
describe('LlmParseService', () => {
|
||||
it('isAvailable reflects whether a config resolves', () => {
|
||||
resolveLlmConfig.mockReturnValueOnce(null);
|
||||
expect(svc().isAvailable(1)).toBe(false);
|
||||
expect(svc().isAvailable(1)).toBe(true);
|
||||
});
|
||||
|
||||
it('returns a not-configured warning when no config resolves', async () => {
|
||||
resolveLlmConfig.mockReturnValue(null);
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
expect(res.kiItems).toEqual([]);
|
||||
expect(res.warnings[0]).toMatch(/not configured/i);
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('sends extracted text for a text-like file', async () => {
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
expect(res.kiItems).toEqual([{ '@type': 'FlightReservation' }]);
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.text).toBe('Flight AB123');
|
||||
expect(input.file).toBeUndefined();
|
||||
});
|
||||
|
||||
it('extracts text for a pdf on the OpenAI-compatible/local path (no native bytes)', async () => {
|
||||
extractText.mockResolvedValue('Hotel X');
|
||||
await svc().parse(file('a.pdf', '%PDF'), 1);
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.text).toBe('Hotel X');
|
||||
expect(input.file).toBeUndefined();
|
||||
});
|
||||
|
||||
it('sends a pdf as native bytes only for Anthropic', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic' }));
|
||||
await svc().parse(file('a.pdf', '%PDF'), 1);
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.file).toEqual({ mimeType: 'application/pdf', data: expect.any(Buffer) });
|
||||
expect(input.text).toBeUndefined();
|
||||
expect(extractText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('warns when a pdf yields no readable text (e.g. a scan)', async () => {
|
||||
extractText.mockResolvedValue(' ');
|
||||
const res = await svc().parse(file('a.pdf', '%PDF'), 1);
|
||||
expect(res.kiItems).toEqual([]);
|
||||
expect(res.warnings[0]).toMatch(/no readable text/i);
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('folds flattened type fields into reservationFor (small-model output)', async () => {
|
||||
extract.mockResolvedValue([{
|
||||
'@type': 'FlightReservation',
|
||||
reservationNumber: 'ABC',
|
||||
flightNumber: 'EZY1357',
|
||||
airline: { iataCode: 'EG' },
|
||||
departureAirport: { iataCode: 'GEG' },
|
||||
arrivalAirport: { iataCode: 'AMS' },
|
||||
departureTime: '2026-06-11T10:00:00',
|
||||
}]);
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
const item = res.kiItems[0] as any;
|
||||
expect(item.reservationNumber).toBe('ABC');
|
||||
expect(item.reservationFor).toMatchObject({ flightNumber: 'EZY1357', departureAirport: { iataCode: 'GEG' } });
|
||||
// root-level keys are not duplicated into reservationFor
|
||||
expect(item.reservationFor.reservationNumber).toBeUndefined();
|
||||
});
|
||||
|
||||
it('leaves already-nested reservationFor untouched', async () => {
|
||||
extract.mockResolvedValue([{ '@type': 'FlightReservation', reservationFor: { flightNumber: 'X1' } }]);
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
expect((res.kiItems[0] as any).reservationFor).toEqual({ flightNumber: 'X1' });
|
||||
});
|
||||
|
||||
it('drops nodes without a string @type and warns', async () => {
|
||||
extract.mockResolvedValue([{ '@type': 'FlightReservation' }, { foo: 'bar' }]);
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
expect(res.kiItems).toEqual([{ '@type': 'FlightReservation' }]);
|
||||
expect(res.warnings.some(w => /unrecognized/i.test(w))).toBe(true);
|
||||
});
|
||||
|
||||
it('degrades to a warning when the client throws', async () => {
|
||||
extract.mockRejectedValue(new Error('boom'));
|
||||
const res = await svc().parse(file('a.txt'), 1);
|
||||
expect(res.kiItems).toEqual([]);
|
||||
expect(res.warnings[0]).toMatch(/AI parsing failed/i);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from '../../../../src/nest/llm-parse/llm-prompt';
|
||||
import { KI_RESERVATION_TYPES } from '@trek/shared';
|
||||
|
||||
describe('llm-prompt', () => {
|
||||
it('names every recognized @type the mapper supports', () => {
|
||||
const prompt = buildSystemPrompt();
|
||||
for (const t of KI_RESERVATION_TYPES) expect(prompt).toContain(t);
|
||||
});
|
||||
|
||||
it('instructs JSON-only output wrapped in reservations', () => {
|
||||
const prompt = buildSystemPrompt();
|
||||
expect(prompt).toMatch(/"reservations"/);
|
||||
expect(prompt.toLowerCase()).toContain('iso 8601');
|
||||
});
|
||||
|
||||
it('exposes a strict-safe object-root JSON schema enumerating the types', () => {
|
||||
const schema = KI_RESERVATION_JSON_SCHEMA as any;
|
||||
expect(schema.type).toBe('object');
|
||||
expect(schema.additionalProperties).toBe(false);
|
||||
expect(schema.required).toContain('reservations');
|
||||
const item = schema.properties.reservations.items;
|
||||
expect(item.properties['@type'].enum).toEqual([...KI_RESERVATION_TYPES]);
|
||||
expect(item.required).toContain('@type');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
|
||||
const { getText } = vi.hoisted(() => ({ getText: vi.fn(async () => ({ text: 'Hotel X — confirmation ABC' })) }));
|
||||
vi.mock('pdf-parse', () => ({
|
||||
PDFParse: class {
|
||||
getText = getText;
|
||||
destroy = vi.fn(async () => {});
|
||||
},
|
||||
}));
|
||||
|
||||
import { isTextLike, isPdf, extractText } from '../../../../src/nest/llm-parse/text-extract';
|
||||
|
||||
describe('text-extract', () => {
|
||||
it('classifies text-like and pdf extensions', () => {
|
||||
expect(isTextLike('a.txt')).toBe(true);
|
||||
expect(isTextLike('a.html')).toBe(true);
|
||||
expect(isTextLike('a.eml')).toBe(true);
|
||||
expect(isTextLike('a.pdf')).toBe(false);
|
||||
expect(isPdf('a.PDF')).toBe(true);
|
||||
expect(isPdf('a.txt')).toBe(false);
|
||||
});
|
||||
|
||||
it('decodes plain text', async () => {
|
||||
expect(await extractText(Buffer.from('hello world'), 'a.txt')).toBe('hello world');
|
||||
});
|
||||
|
||||
it('strips markup from html/eml', async () => {
|
||||
const html = '<html><style>x{}</style><body><p>Flight AB123</p><script>1</script></body></html>';
|
||||
const out = await extractText(Buffer.from(html), 'a.html');
|
||||
expect(out).toContain('Flight AB123');
|
||||
expect(out).not.toContain('<p>');
|
||||
expect(out).not.toContain('x{}');
|
||||
});
|
||||
|
||||
it('extracts the embedded text layer from a pdf', async () => {
|
||||
const out = await extractText(Buffer.from('%PDF-1.4'), 'a.pdf');
|
||||
expect(out).toBe('Hotel X — confirmation ABC');
|
||||
expect(getText).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -129,6 +129,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'تم العثور على {count} حجز/حجوزات',
|
||||
'reservations.import.previewEmpty': 'تعذّر استخراج أي حجوزات من الملفات المُحمَّلة.',
|
||||
'reservations.import.removeItem': 'إزالة',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'استيراد {count} حجز/حجوزات',
|
||||
'reservations.import.back': 'رجوع',
|
||||
'reservations.import.success': 'تم استيراد {count} حجز/حجوزات',
|
||||
|
||||
@@ -59,6 +59,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabels': 'تسميات مسارات الحجوزات',
|
||||
'settings.bookingLabelsHint': 'عرض أسماء المحطات/المطارات على الخريطة. عند الإيقاف، يتم عرض الرمز فقط.',
|
||||
'settings.blurBookingCodes': 'إخفاء رموز الحجز',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'تحسين المسار انطلاقًا من مكان الإقامة',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'عند تحسين يوم ما، يبدأ المسار من الفندق الذي تستيقظ فيه وينتهي عند الفندق الذي تسجّل الوصول إليه في تلك الليلة.',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} reserva(s) encontrada(s)',
|
||||
'reservations.import.previewEmpty': 'Nenhuma reserva pôde ser extraída dos arquivos enviados.',
|
||||
'reservations.import.removeItem': 'Remover',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importar {count} reserva(s)',
|
||||
'reservations.import.back': 'Voltar',
|
||||
'reservations.import.success': '{count} reserva(s) importada(s)',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Unidade de distância',
|
||||
'settings.timeFormat': 'Formato de hora',
|
||||
'settings.blurBookingCodes': 'Ocultar códigos de reserva',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Otimizar rota a partir da hospedagem',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Ao otimizar um dia, comece a rota no hotel onde você acorda e termine no hotel em que você faz check-in à noite.',
|
||||
|
||||
@@ -129,6 +129,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'Nalezeno {count} rezervace/í',
|
||||
'reservations.import.previewEmpty': 'Z nahraných souborů se nepodařilo extrahovat žádné rezervace.',
|
||||
'reservations.import.removeItem': 'Odebrat',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importovat {count} rezervaci/í',
|
||||
'reservations.import.back': 'Zpět',
|
||||
'reservations.import.success': '{count} rezervace/í importováno',
|
||||
|
||||
@@ -59,6 +59,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Jednotky vzdálenosti',
|
||||
'settings.timeFormat': 'Formát času',
|
||||
'settings.blurBookingCodes': 'Skrýt rezervační kódy',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Optimalizovat trasu od ubytování',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Při optimalizaci dne začne trasa v hotelu, ve kterém se ráno probudíte, a skončí v hotelu, do kterého se večer ubytujete.',
|
||||
|
||||
@@ -130,6 +130,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} Reservierung(en) gefunden',
|
||||
'reservations.import.previewEmpty': 'Aus den hochgeladenen Dateien konnten keine Reservierungen extrahiert werden.',
|
||||
'reservations.import.removeItem': 'Entfernen',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count} Reservierung(en) importieren',
|
||||
'reservations.import.back': 'Zurück',
|
||||
'reservations.import.success': '{count} Reservierung(en) importiert',
|
||||
|
||||
@@ -62,6 +62,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabels': 'Orts-Labels auf Buchungsrouten',
|
||||
'settings.bookingLabelsHint': 'Zeigt Bahnhofs-/Flughafennamen auf der Karte. Wenn aus, wird nur das Icon angezeigt.',
|
||||
'settings.blurBookingCodes': 'Buchungscodes verbergen',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Route ab der Unterkunft optimieren',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Beim Optimieren eines Tages startet die Route an der Unterkunft, in der du aufwachst, und endet an der, in die du am Abend eincheckst.',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} reservation(s) found',
|
||||
'reservations.import.previewEmpty': 'No reservations could be extracted from the uploaded files.',
|
||||
'reservations.import.removeItem': 'Remove',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Import {count} reservation(s)',
|
||||
'reservations.import.back': 'Back',
|
||||
'reservations.import.success': '{count} reservation(s) imported',
|
||||
|
||||
@@ -64,6 +64,8 @@ const settings: TranslationStrings = {
|
||||
'settings.mapPoiPillHint':
|
||||
'Show a category pill on the trip map to find nearby restaurants, hotels and more from OpenStreetMap.',
|
||||
'settings.blurBookingCodes': 'Blur Booking Codes',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Optimize route from accommodation',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'When optimizing a day, start the route at the hotel you wake up in and end it at the one you check into that evening.',
|
||||
|
||||
@@ -131,6 +131,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} reserva(s) encontrada(s)',
|
||||
'reservations.import.previewEmpty': 'No se pudieron extraer reservas de los archivos subidos.',
|
||||
'reservations.import.removeItem': 'Eliminar',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importar {count} reserva(s)',
|
||||
'reservations.import.back': 'Atrás',
|
||||
'reservations.import.success': '{count} reserva(s) importada(s)',
|
||||
|
||||
@@ -61,6 +61,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Unidad de distancia',
|
||||
'settings.timeFormat': 'Formato de hora',
|
||||
'settings.blurBookingCodes': 'Difuminar códigos de reserva',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Optimizar la ruta desde el alojamiento',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Al optimizar un día, comienza la ruta en el hotel donde despiertas y termínala en aquel en el que te registras esa noche.',
|
||||
|
||||
@@ -132,6 +132,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} réservation(s) trouvée(s)',
|
||||
'reservations.import.previewEmpty': "Aucune réservation n'a pu être extraite des fichiers envoyés.",
|
||||
'reservations.import.removeItem': 'Supprimer',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importer {count} réservation(s)',
|
||||
'reservations.import.back': 'Retour',
|
||||
'reservations.import.success': '{count} réservation(s) importée(s)',
|
||||
|
||||
@@ -62,6 +62,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Unité de distance',
|
||||
'settings.timeFormat': "Format de l'heure",
|
||||
'settings.blurBookingCodes': 'Masquer les codes de réservation',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': "Optimiser l'itinéraire depuis l'hébergement",
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
"Lors de l'optimisation d'une journée, commencez l'itinéraire à l'hôtel où vous vous réveillez et terminez-le à celui où vous arrivez le soir.",
|
||||
|
||||
@@ -131,6 +131,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'Βρέθηκαν {count} κράτηση/κρατήσεις',
|
||||
'reservations.import.previewEmpty': 'Δεν ήταν δυνατή η εξαγωγή κρατήσεων από τα μεταφορτωμένα αρχεία.',
|
||||
'reservations.import.removeItem': 'Αφαίρεση',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Εισαγωγή {count} κράτησης/κρατήσεων',
|
||||
'reservations.import.back': 'Πίσω',
|
||||
'reservations.import.success': '{count} κράτηση/κρατήσεις εισήχθησαν',
|
||||
|
||||
@@ -66,6 +66,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabelsHint':
|
||||
'Εμφάνιση ονομάτων σταθμών / αεροδρομίων στον χάρτη. Όταν είναι απενεργοποιημένο, εμφανίζεται μόνο το εικονίδιο.',
|
||||
'settings.blurBookingCodes': 'Θόλωμα Κωδικών Κρατήσεων',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Βελτιστοποίηση διαδρομής από το κατάλυμα',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Κατά τη βελτιστοποίηση μιας ημέρας, ξεκινήστε τη διαδρομή από το ξενοδοχείο στο οποίο ξυπνάτε και τερματίστε την σε αυτό στο οποίο κάνετε check-in το ίδιο βράδυ.',
|
||||
|
||||
@@ -130,6 +130,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} foglalás találva',
|
||||
'reservations.import.previewEmpty': 'A feltöltött fájlokból nem sikerült foglalásokat kinyerni.',
|
||||
'reservations.import.removeItem': 'Eltávolítás',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count} foglalás importálása',
|
||||
'reservations.import.back': 'Vissza',
|
||||
'reservations.import.success': '{count} foglalás importálva',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Távolság egység',
|
||||
'settings.timeFormat': 'Időformátum',
|
||||
'settings.blurBookingCodes': 'Foglalási kódok elrejtése',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Útvonal optimalizálása a szállástól',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'A nap optimalizálásakor az útvonal annál a szállásnál kezdődjön, ahol felébredsz, és annál érjen véget, ahova este bejelentkezel.',
|
||||
|
||||
@@ -129,6 +129,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} pemesanan ditemukan',
|
||||
'reservations.import.previewEmpty': 'Tidak ada pemesanan yang dapat diekstrak dari file yang diunggah.',
|
||||
'reservations.import.removeItem': 'Hapus',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Impor {count} pemesanan',
|
||||
'reservations.import.back': 'Kembali',
|
||||
'reservations.import.success': '{count} pemesanan berhasil diimpor',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Satuan Jarak',
|
||||
'settings.timeFormat': 'Format Waktu',
|
||||
'settings.blurBookingCodes': 'Sembunyikan Kode Pemesanan',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Optimalkan rute dari akomodasi',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Saat mengoptimalkan suatu hari, mulai rute dari hotel tempatmu bangun pagi dan akhiri di hotel tempatmu check-in malam itu.',
|
||||
|
||||
@@ -129,6 +129,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} prenotazione/i trovata/e',
|
||||
'reservations.import.previewEmpty': 'Nessuna prenotazione è stata estratta dai file caricati.',
|
||||
'reservations.import.removeItem': 'Rimuovi',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importa {count} prenotazione/i',
|
||||
'reservations.import.back': 'Indietro',
|
||||
'reservations.import.success': '{count} prenotazione/i importata/e',
|
||||
|
||||
@@ -61,6 +61,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Unità di Distanza',
|
||||
'settings.timeFormat': 'Formato Ora',
|
||||
'settings.blurBookingCodes': 'Nascondi codici di prenotazione',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': "Ottimizza il percorso dall'alloggio",
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
"Quando ottimizzi un giorno, fa iniziare il percorso dall'hotel in cui ti svegli e terminarlo in quello in cui fai il check-in quella sera.",
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} 件の予約が見つかりました',
|
||||
'reservations.import.previewEmpty': 'アップロードされたファイルから予約を抽出できませんでした。',
|
||||
'reservations.import.removeItem': '削除',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count} 件の予約をインポート',
|
||||
'reservations.import.back': '戻る',
|
||||
'reservations.import.success': '{count} 件の予約をインポートしました',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabels': '予約ルートのラベル',
|
||||
'settings.bookingLabelsHint': '地図に駅・空港名を表示。オフ時はアイコンのみ。',
|
||||
'settings.blurBookingCodes': '予約コードをぼかす',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': '宿泊先を起点にルートを最適化',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'その日を最適化する際、朝に目覚める宿泊先を起点にし、その晩にチェックインする宿泊先を終点としてルートを組みます。',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count}개 예약 발견',
|
||||
'reservations.import.previewEmpty': '업로드된 파일에서 예약을 추출할 수 없었습니다.',
|
||||
'reservations.import.removeItem': '제거',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count}개 예약 가져오기',
|
||||
'reservations.import.back': '뒤로',
|
||||
'reservations.import.success': '{count}개 예약을 가져왔습니다',
|
||||
|
||||
@@ -61,6 +61,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabels': '예약 경로 레이블',
|
||||
'settings.bookingLabelsHint': '지도에 역 / 공항 이름을 표시합니다. 끄면 아이콘만 표시됩니다.',
|
||||
'settings.blurBookingCodes': '예약 코드 흐리게',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': '숙소 기준으로 경로 최적화',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'하루 일정을 최적화할 때, 아침에 머무는 숙소에서 경로를 시작하고 그날 저녁에 체크인하는 숙소에서 경로를 끝냅니다.',
|
||||
|
||||
@@ -130,6 +130,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} reservering(en) gevonden',
|
||||
'reservations.import.previewEmpty': 'Er konden geen reserveringen worden geëxtraheerd uit de geüploade bestanden.',
|
||||
'reservations.import.removeItem': 'Verwijderen',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count} reservering(en) importeren',
|
||||
'reservations.import.back': 'Terug',
|
||||
'reservations.import.success': '{count} reservering(en) geïmporteerd',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Afstandseenheid',
|
||||
'settings.timeFormat': 'Tijdnotatie',
|
||||
'settings.blurBookingCodes': 'Boekingscodes vervagen',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Route optimaliseren vanaf accommodatie',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Begin bij het optimaliseren van een dag de route bij het hotel waar je wakker wordt en eindig bij het hotel waar je die avond incheckt.',
|
||||
|
||||
@@ -129,6 +129,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'Znaleziono {count} rezerwację/rezerwacje',
|
||||
'reservations.import.previewEmpty': 'Nie udało się wyodrębnić rezerwacji z przesłanych plików.',
|
||||
'reservations.import.removeItem': 'Usuń',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Importuj {count} rezerwację/rezerwacje',
|
||||
'reservations.import.back': 'Wstecz',
|
||||
'reservations.import.success': 'Zaimportowano {count} rezerwację/rezerwacje',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Jednostka odległości',
|
||||
'settings.timeFormat': 'Format czasu',
|
||||
'settings.blurBookingCodes': 'Rozmyj kody rezerwacji',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Optymalizuj trasę od zakwaterowania',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Przy optymalizacji dnia rozpocznij trasę w hotelu, w którym się budzisz, a zakończ ją w tym, do którego się zameldujesz tego wieczoru.',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'Найдено {count} бронирование(й)',
|
||||
'reservations.import.previewEmpty': 'Из загруженных файлов не удалось извлечь бронирования.',
|
||||
'reservations.import.removeItem': 'Удалить',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Импортировать {count} бронирование(й)',
|
||||
'reservations.import.back': 'Назад',
|
||||
'reservations.import.success': '{count} бронирование(й) импортировано',
|
||||
|
||||
@@ -59,6 +59,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Единица расстояния',
|
||||
'settings.timeFormat': 'Формат времени',
|
||||
'settings.blurBookingCodes': 'Скрыть коды бронирования',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Оптимизировать маршрут от места проживания',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'При оптимизации дня маршрут начинается от отеля, в котором вы просыпаетесь, и заканчивается у того, в который вы заселяетесь вечером.',
|
||||
|
||||
@@ -130,6 +130,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '{count} rezervasyon bulundu',
|
||||
'reservations.import.previewEmpty': 'Yüklenen dosyalardan hiçbir rezervasyon çıkarılamadı.',
|
||||
'reservations.import.removeItem': 'Kaldır',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '{count} rezervasyonu içe aktar',
|
||||
'reservations.import.back': 'Geri',
|
||||
'reservations.import.success': '{count} rezervasyon içe aktarıldı',
|
||||
|
||||
@@ -61,6 +61,8 @@ const settings: TranslationStrings = {
|
||||
'settings.bookingLabels': 'Rezervasyon rota etiketleri',
|
||||
'settings.bookingLabelsHint': 'Haritada istasyon / havalimanı adlarını göster. Kapalıyken yalnızca simge görünür.',
|
||||
'settings.blurBookingCodes': 'Rezervasyon Kodlarını Bulanıklaştır',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Rotayı konaklamadan optimize et',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Bir günü optimize ederken rotaya o sabah uyandığınız otelden başlayın ve akşam giriş yaptığınız otelde sonlandırın.',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': 'Знайдено {count} бронювання(нь)',
|
||||
'reservations.import.previewEmpty': 'З завантажених файлів не вдалося витягти бронювання.',
|
||||
'reservations.import.removeItem': 'Видалити',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': 'Імпортувати {count} бронювання(нь)',
|
||||
'reservations.import.back': 'Назад',
|
||||
'reservations.import.success': '{count} бронювання(нь) імпортовано',
|
||||
|
||||
@@ -60,6 +60,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': 'Одиниця відстані',
|
||||
'settings.timeFormat': 'Формат часу',
|
||||
'settings.blurBookingCodes': 'Приховати коди бронювання',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': 'Оптимізувати маршрут від житла',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'Під час оптимізації дня починайте маршрут від готелю, у якому ви прокидаєтеся, і завершуйте його тим, у який ви заселяєтеся ввечері.',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '找到 {count} 筆預訂',
|
||||
'reservations.import.previewEmpty': '無法從上傳的檔案中提取任何預訂資訊。',
|
||||
'reservations.import.removeItem': '移除',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '匯入 {count} 筆預訂',
|
||||
'reservations.import.back': '返回',
|
||||
'reservations.import.success': '已匯入 {count} 筆預訂',
|
||||
|
||||
@@ -58,6 +58,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': '距離單位',
|
||||
'settings.timeFormat': '時間格式',
|
||||
'settings.blurBookingCodes': '模糊預訂程式碼',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': '從住宿地點最佳化路線',
|
||||
'settings.optimizeFromAccommodationHint':
|
||||
'最佳化某一天的行程時,路線從你早上起床的飯店出發,並在你當晚入住的飯店結束。',
|
||||
|
||||
@@ -128,6 +128,9 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.previewHeading': '找到 {count} 个预订',
|
||||
'reservations.import.previewEmpty': '无法从上传的文件中提取任何预订信息。',
|
||||
'reservations.import.removeItem': '移除',
|
||||
'reservations.import.needsReview': 'Review',
|
||||
'reservations.import.tryAi': 'Try AI parsing',
|
||||
'reservations.import.aiParsing': 'Parsing with AI…',
|
||||
'reservations.import.confirm': '导入 {count} 个预订',
|
||||
'reservations.import.back': '返回',
|
||||
'reservations.import.success': '已导入 {count} 个预订',
|
||||
|
||||
@@ -58,6 +58,8 @@ const settings: TranslationStrings = {
|
||||
'settings.distance': '距离单位',
|
||||
'settings.timeFormat': '时间格式',
|
||||
'settings.blurBookingCodes': '模糊预订代码',
|
||||
'settings.aiAlwaysRetry': 'Always retry booking imports with AI',
|
||||
'settings.aiAlwaysRetryHint': 'When a file cannot be read by the standard parser, automatically retry it with AI.',
|
||||
'settings.optimizeFromAccommodation': '从住宿地优化路线',
|
||||
'settings.optimizeFromAccommodationHint': '优化某一天时,路线将从您醒来时所在的酒店出发,并在当晚入住的酒店结束。',
|
||||
'settings.notifications': '通知',
|
||||
|
||||
@@ -26,6 +26,7 @@ export * from './packing/packing.schema';
|
||||
export * from './todo/todo.schema';
|
||||
export * from './budget/budget.schema';
|
||||
export * from './reservation/reservation.schema';
|
||||
export * from './reservation/ki-reservation.schema';
|
||||
export * from './airtrail/airtrail.schema';
|
||||
export * from './day/day.schema';
|
||||
export * from './assignment/assignment.schema';
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* schema.org-style reservation JSON-LD — the shape the kitinerary binary emits
|
||||
* and the shape we ask an LLM fallback to produce, so both feed the SAME
|
||||
* `mapReservations()` mapper (server/src/nest/booking-import/kitinerary-mapper.ts).
|
||||
*
|
||||
* Two artifacts live here:
|
||||
* - `kiReservationArraySchema` — a *lenient* Zod schema used server-side to
|
||||
* validate/repair an LLM response before mapping. The mapper already tolerates
|
||||
* missing fields, so validation only guarantees each node is an object with a
|
||||
* string `@type`; everything else passes through untouched.
|
||||
* - `KI_RESERVATION_JSON_SCHEMA` — the JSON Schema handed to the LLM providers
|
||||
* (OpenAI-compatible `response_format`, Anthropic tool `input_schema`). It is
|
||||
* descriptive guidance, kept deliberately permissive on `reservationFor` so the
|
||||
* model can include type-specific fields; the prompt names the exact fields.
|
||||
*/
|
||||
|
||||
/** The `@type` values `mapReservations()` recognises (its switch + flight grouping). */
|
||||
export const KI_RESERVATION_TYPES = [
|
||||
'FlightReservation',
|
||||
'TrainReservation',
|
||||
'BusReservation',
|
||||
'BoatReservation',
|
||||
'LodgingReservation',
|
||||
'FoodEstablishmentReservation',
|
||||
'RentalCarReservation',
|
||||
'EventReservation',
|
||||
'TouristAttractionVisit',
|
||||
] as const;
|
||||
|
||||
/** Lenient validator: require a string `@type`, allow any other keys through. */
|
||||
export const kiReservationSchema = z
|
||||
.object({ '@type': z.string() })
|
||||
.catchall(z.unknown());
|
||||
export type KiReservationDto = z.infer<typeof kiReservationSchema>;
|
||||
|
||||
/** Top-level wrapper the providers return: `{ reservations: KiReservation[] }`. */
|
||||
export const kiReservationArraySchema = z.object({
|
||||
reservations: z.array(kiReservationSchema),
|
||||
});
|
||||
export type KiReservationArrayDto = z.infer<typeof kiReservationArraySchema>;
|
||||
|
||||
/**
|
||||
* JSON Schema for the providers' structured-output entry points. Object root
|
||||
* (both OpenAI and Anthropic want an object, not a bare array). `reservationFor`
|
||||
* is intentionally open (`additionalProperties: true`) so the model can fill the
|
||||
* type-specific sub-object; the prompt enumerates the field names per `@type`.
|
||||
*/
|
||||
export const KI_RESERVATION_JSON_SCHEMA = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
reservations: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: true,
|
||||
properties: {
|
||||
'@type': { type: 'string', enum: [...KI_RESERVATION_TYPES] },
|
||||
reservationNumber: { type: 'string' },
|
||||
// Hotel check-in/out, car pickup/dropoff, event/restaurant start/end —
|
||||
// plain ISO 8601 strings (no KDE QDateTime wrapper).
|
||||
checkinTime: { type: 'string' },
|
||||
checkoutTime: { type: 'string' },
|
||||
pickupTime: { type: 'string' },
|
||||
dropoffTime: { type: 'string' },
|
||||
startTime: { type: 'string' },
|
||||
endTime: { type: 'string' },
|
||||
// Type-specific payload (Flight/Train/Lodging/…): open object.
|
||||
reservationFor: { type: 'object', additionalProperties: true },
|
||||
pickupLocation: { type: 'object', additionalProperties: true },
|
||||
},
|
||||
required: ['@type'],
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['reservations'],
|
||||
} as const;
|
||||
@@ -181,9 +181,28 @@ export const bookingImportPreviewItemSchema = z.object({
|
||||
});
|
||||
export type BookingImportPreviewItem = z.infer<typeof bookingImportPreviewItemSchema>;
|
||||
|
||||
/**
|
||||
* How the preview endpoint should treat the LLM fallback:
|
||||
* - `no-ai` — kitinerary only (default; existing behaviour)
|
||||
* - `fallback-on-empty` — run the LLM for files kitinerary returns nothing for
|
||||
* - `force-ai` — run the LLM on every submitted file (skip kitinerary)
|
||||
*/
|
||||
export const bookingImportModeSchema = z.enum(['no-ai', 'fallback-on-empty', 'force-ai']);
|
||||
export type BookingImportMode = z.infer<typeof bookingImportModeSchema>;
|
||||
|
||||
/** Per-file AI report so the preview UI can offer "Try AI parsing" only where it applies. */
|
||||
export const bookingImportFileReportSchema = z.object({
|
||||
fileName: z.string(),
|
||||
aiAvailable: z.boolean(),
|
||||
aiUsed: z.boolean(),
|
||||
});
|
||||
export type BookingImportFileReport = z.infer<typeof bookingImportFileReportSchema>;
|
||||
|
||||
export const bookingImportPreviewResponseSchema = z.object({
|
||||
items: z.array(bookingImportPreviewItemSchema),
|
||||
warnings: z.array(z.string()),
|
||||
// Optional so existing/no-AI responses stay byte-compatible.
|
||||
files: z.array(bookingImportFileReportSchema).optional(),
|
||||
});
|
||||
export type BookingImportPreviewResponse = z.infer<typeof bookingImportPreviewResponseSchema>;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user