feat(llm-parse): read a photo, as a booking or as a receipt in Costs

Whether the AI Parsing model reads images is an admin setting now, and it
lands with the two features that send a photo, so it has something to
control.

The setting is `vision: 'auto' | 'on' | 'off'` on the addon config. On
Automatic a local Ollama is asked what the model can do (`/api/show`), and a
cloud provider means no; Yes and No are applied as set. The old `multimodal`
flag is not read: the admin screens wrote it false on every save, so a
stored false meant nothing. The per-user toggle keeps its on/off meaning and
is read by the extraction now. The lookup is server-side only: capped read,
validated body, never throws, cached per server and model.
GET /api/llm/capabilities answers it for the signed-in caller, and both
photo pickers below ask it.

The booking import takes JPG, PNG and WEBP when the model reads images.
The photo goes as an image to every provider (image_url, an Anthropic image
block, Ollama's native chat with `images`), capped at 1600px, and skips KDE
Itinerary. A scanned PDF, which used to reach the model as empty text, has
its first two pages drawn by pdf-parse and sent the same way. The rest is
the import as it was: background job, review, the file attached to the
booking, the linked cost. On a real qwen3.5:4b a train ticket came back as
bare clock times until the router gave the model every type's field
meanings, and its price as an object inside a string, which read as 4900;
both are handled.

Costs gets Scan receipt: a dialog laid out like the booking import's takes
one photo, read on the import job queue, then reviewed in the existing
expense editor pre-filled with merchant, total, currency, day and lines,
the photo waiting in Receipts & Invoices to be uploaded on save. No table,
no migration, no category guess, no document-type detection.
This commit is contained in:
RemEx
2026-09-24 17:12:54 +00:00
committed by Maurice
parent 706c6c3959
commit b1c05bca83
183 changed files with 3098 additions and 218 deletions
+2 -1
View File
@@ -54,6 +54,7 @@ import {
type BookingImportPreviewResponse,
type BookingImportConfirmResponse,
type BookingImportMode,
type ReceiptScanResult,
type StorageAdminState,
type StorageBackend,
type StorageConfigPut,
@@ -1475,7 +1476,7 @@ export const reservationsApi = {
return postMultipart(`/trips/${tripId}/reservations/import/booking/async`, fd)
},
// Poll a background job — recovery path when a WebSocket push was missed.
importJobStatus: (tripId: number | string, jobId: string): Promise<{ status: 'running' | 'done' | 'error'; done: number; total: number; result?: BookingImportPreviewResponse; error?: string }> =>
importJobStatus: (tripId: number | string, jobId: string): Promise<{ status: 'running' | 'done' | 'error'; done: number; total: number; result?: BookingImportPreviewResponse | ReceiptScanResult; error?: string }> =>
apiClient.get(`/trips/${tripId}/reservations/import/jobs/${jobId}`).then(r => r.data),
}
@@ -703,7 +703,7 @@ describe('AddonManager', () => {
await user.click(screen.getByRole('button', { name: 'Save' }));
await screen.findByText('Saved');
expect(bodies[0]).toEqual({
config: { provider: 'local', model: 'qwen3:8b', baseUrl: '', apiKey: '••••••••', multimodal: true },
config: { provider: 'local', model: 'qwen3:8b', baseUrl: '', apiKey: '••••••••', vision: 'auto' },
});
await user.click(screen.getByRole('button', { name: 'Save' }));
@@ -737,7 +737,7 @@ describe('AddonManager', () => {
await screen.findByText('Saved');
// The model is trimmed before it is stored, the key is sent verbatim
expect(bodies[0]).toEqual({
config: { provider: 'openai', model: 'mistral:7b', baseUrl: '', apiKey: 'sk-live', multimodal: false },
config: { provider: 'openai', model: 'mistral:7b', baseUrl: '', apiKey: 'sk-live', vision: 'auto' },
});
});
@@ -765,10 +765,40 @@ describe('AddonManager', () => {
await screen.findByText('Saved');
// The stale local base URL must not ride along to Anthropic — it would hijack the endpoint.
expect(bodies[0]).toEqual({
config: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001', baseUrl: '', apiKey: 'sk-ant-live', multimodal: false },
config: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001', baseUrl: '', apiKey: 'sk-ant-live', vision: 'auto' },
});
});
it('FE-ADMIN-ADDON-035: whether the model reads images is auto, on or off, kept as stored and saved as chosen', async () => {
const user = userEvent.setup();
const bodies: unknown[] = [];
server.use(
addonsRoute([llmAddon({ provider: 'openai', model: 'gpt-4.1-mini', baseUrl: '', apiKey: 'sk', vision: 'off' })]),
http.put('/api/admin/addons/llm_parsing', async ({ request }) => {
bodies.push(await request.json());
return HttpResponse.json({ success: true });
}),
);
render(<><ToastContainer /><AddonManager /></>);
// Stored off stays off; a cloud provider is told what Automatic means for it.
await user.click(await screen.findByRole('button', { name: 'No' }));
expect(screen.getByText('Automatic means no for a cloud model. Choose Yes if this model reads images.')).toBeInTheDocument();
await user.click(screen.getByRole('button', { name: 'Yes' }));
await user.click(screen.getByRole('button', { name: 'Save' }));
await screen.findByText('Saved');
expect(bodies[0]).toMatchObject({ config: { provider: 'openai', vision: 'on' } });
expect(bodies[0]).not.toHaveProperty('config.multimodal');
});
it('FE-ADMIN-ADDON-036: with the local provider the hint says Automatic asks the server', async () => {
server.use(addonsRoute([llmAddon({ provider: 'local', model: 'qwen3.5:4b' })]), modelsRoute([]));
render(<AddonManager />);
expect(await screen.findByText('Automatic asks the Ollama server whether this model reads images.')).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Automatic' })).toBeInTheDocument();
});
it('FE-ADMIN-ADDON-027: an error frame in the pull stream aborts the pull and is reported', async () => {
const user = userEvent.setup();
server.use(
+13 -1
View File
@@ -6,6 +6,7 @@ import { useIsDark } from '../../hooks/useIsDark'
import { useToast } from '../shared/Toast'
import { Puzzle, ListChecks, Wallet, FileText, CalendarDays, Globe, Briefcase, Image, Terminal, Link2, Compass, BookOpen, MessageCircle, StickyNote, BarChart3, Sparkles, Luggage, Plane, Server, Cloud, Bookmark, Users, Loader2 } from 'lucide-react'
import CustomSelect from '../shared/CustomSelect'
import { asLlmVision, LLM_VISION_MODES, type LlmVision } from '@trek/shared'
import EmptyState from '../shared/EmptyState'
import DawarichIcon from '../shared/DawarichIcon'
import AirTrailIcon from '../shared/AirTrailIcon'
@@ -371,12 +372,14 @@ const RECOMMENDED_MODELS: { id: string; label: string; note: string; recommended
* provider, it also lists installed Ollama models and can pull NuExtract models.
*/
function LlmParsingConfig({ addon }: { addon: Addon }) {
const { t } = useTranslation()
const toast = useToast()
const cfg = (addon.config ?? {}) as Record<string, unknown>
const [provider, setProvider] = useState<string>((cfg.provider as string) ?? 'local')
const [model, setModel] = useState<string>((cfg.model as string) ?? '')
const [baseUrl, setBaseUrl] = useState<string>((cfg.baseUrl as string) ?? '')
const [apiKey, setApiKey] = useState<string>((cfg.apiKey as string) ?? '')
const [vision, setVision] = useState<LlmVision>(asLlmVision(cfg.vision))
const [saving, setSaving] = useState(false)
// Local-provider model management.
@@ -438,7 +441,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
setSaving(true)
try {
// Send the masked sentinel unchanged so the server keeps the stored key.
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: provider === 'anthropic' ? '' : baseUrl.trim(), apiKey, multimodal: cfg.multimodal === true } })
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: provider === 'anthropic' ? '' : baseUrl.trim(), apiKey, vision } })
toast.success('Saved')
} catch {
toast.error('Failed to save')
@@ -455,6 +458,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
{ value: 'openai', label: 'OpenAI', icon: <Cloud size={14} /> },
{ value: 'anthropic', label: 'Anthropic', icon: <Sparkles size={14} /> },
]
const visionOptions = LLM_VISION_MODES.map(value => ({ value, label: t(`admin.addons.llm.vision.${value}`) }))
/* Lives in the tile's shelf like the collab toggles, so everything is caption-
sized and single-column — the band this used to be had a whole page width. */
@@ -486,6 +490,14 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
<input autoComplete="off" className={fieldCls} value={model} onChange={e => setModel(e.target.value)} placeholder={provider === 'anthropic' ? 'claude-opus-4-8' : provider === 'openai' ? 'gpt-4o' : 'select or pull below'} />
</label>
<div>
<span className={labelCls}>{t('settings.aiParsing.multimodal')}</span>
<CustomSelect value={vision} onChange={v => setVision(asLlmVision(v))} options={visionOptions} />
<p className="mt-1 text-caption text-content-faint">
{t(provider === 'local' ? 'admin.addons.llm.vision.hintLocal' : 'admin.addons.llm.vision.hintCloud')}
</p>
</div>
{/* Local model management (Ollama) */}
{provider === 'local' && (
<div className="space-y-2 rounded-lg border border-edge-secondary bg-surface p-2.5">
@@ -304,3 +304,44 @@ describe('BackgroundTasksWidget: the mark on a finished job (#2477)', () => {
expect(baseElement.querySelector('svg.lucide-alert-triangle')).toBeNull()
})
})
describe('BackgroundTasksWidget — a receipt scanned from Costs', () => {
const RECEIPT = { merchant: 'Café', date: '2026-09-20', total: 12.5, currency: 'EUR', items: [] }
it('FE-W4BGT-025: says it is reading the receipt while the job runs', () => {
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
render(<BackgroundTasksWidget />)
expect(screen.getByText('Reading the receipt…')).toBeInTheDocument()
})
it('FE-W4BGT-026: import:done with a read receipt offers to review the expense, which opens the trip', () => {
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
render(<BackgroundTasksWidget />)
const handler = vi.mocked(addListener).mock.calls[0][0] as WsHandler
act(() => { handler({ type: 'import:done', jobId: 'j1', tripId: 't1', result: { receipt: RECEIPT, warnings: [] } }) })
expect(useBackgroundTasksStore.getState().tasks[0].receipt).toEqual(RECEIPT)
fireEvent.click(screen.getByRole('button', { name: 'Review expense' }))
expect(useBackgroundTasksStore.getState().tasks[0].reviewRequested).toBe(true)
expect(navigate).toHaveBeenCalledWith('/trips/t1')
})
it('FE-W4BGT-027: a photo nothing could be read from says so, with its warning and no AI retry', async () => {
vi.mocked(healthApi.features).mockResolvedValue({ bookingImport: true, aiParsing: true })
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', receipt: null, warnings: ['bill.jpg: no receipt could be read'], sourceFiles: [photo] })] })
render(<BackgroundTasksWidget />)
expect(screen.getByText('No receipt could be read from this photo.')).toBeInTheDocument()
expect(screen.getByText('bill.jpg: no receipt could be read')).toBeInTheDocument()
await waitFor(() => expect(healthApi.features).toHaveBeenCalled())
expect(screen.queryByRole('button', { name: 'Try AI parsing' })).not.toBeInTheDocument()
})
it('FE-W4BGT-028: the reload reconcile reads a receipt job\'s status too', async () => {
vi.mocked(reservationsApi.importJobStatus).mockResolvedValue({ status: 'done', done: 1, total: 1, result: { receipt: RECEIPT, warnings: [] } as never })
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', items: undefined })] })
render(<BackgroundTasksWidget />)
expect(await screen.findByRole('button', { name: 'Review expense' })).toBeInTheDocument()
})
})
@@ -6,7 +6,18 @@ import { useTranslation } from '../../i18n'
import { addListener, removeListener } from '../../api/websocket'
import { reservationsApi, healthApi } from '../../api/client'
import { saveImportFiles } from '../../db/offlineDb'
import { useBackgroundTasksStore, type BackgroundImportTask } from '../../store/backgroundTasksStore'
import { useBackgroundTasksStore, taskFoundSomething, type BackgroundImportTask } from '../../store/backgroundTasksStore'
import type { ReceiptRead } from '@trek/shared'
/**
* One reading of a finished job's result, wherever it arrives from (the push,
* the reload reconcile, the poll): a booking parse answers `items`, a receipt
* scan `receipt`.
*/
function settleJob(id: string, tripId: string, result: unknown): void {
const r = result as { items?: unknown[]; warnings?: string[]; receipt?: ReceiptRead | null } | undefined
useBackgroundTasksStore.getState().setDone(id, tripId, (r?.items ?? []) as never, r?.warnings ?? [], r?.receipt)
}
/**
* Global, route-independent widget (bottom-right) that tracks background booking
@@ -20,7 +31,6 @@ export default function BackgroundTasksWidget() {
const navigate = useNavigate()
const tasks = useBackgroundTasksStore((s) => s.tasks)
const setProgress = useBackgroundTasksStore((s) => s.setProgress)
const setDone = useBackgroundTasksStore((s) => s.setDone)
const setError = useBackgroundTasksStore((s) => s.setError)
const requestReview = useBackgroundTasksStore((s) => s.requestReview)
const dismiss = useBackgroundTasksStore((s) => s.dismiss)
@@ -66,7 +76,7 @@ export default function BackgroundTasksWidget() {
reservationsApi
.importJobStatus(task.tripId, task.id)
.then((s) => {
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
else setProgress(task.id, task.tripId, s.done, s.total)
})
@@ -87,14 +97,12 @@ export default function BackgroundTasksWidget() {
const tripId = String(e.tripId ?? '')
if (!id) return
if (type === 'import:progress') setProgress(id, tripId, Number(e.done ?? 0), Number(e.total ?? 1))
else if (type === 'import:done') {
const result = e.result as { items?: unknown[]; warnings?: string[] } | undefined
setDone(id, tripId, (result?.items ?? []) as never, result?.warnings ?? [])
} else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
else if (type === 'import:done') settleJob(id, tripId, e.result)
else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
}
addListener(handler)
return () => removeListener(handler)
}, [setProgress, setDone, setError])
}, [setProgress, setError])
// Backstop: poll jobs whose state we still need — running ones (in case a WebSocket push
// was missed) and a restored 'done' task whose items haven't been re-fetched yet (so a
@@ -107,7 +115,7 @@ export default function BackgroundTasksWidget() {
reservationsApi
.importJobStatus(task.tripId, task.id)
.then((s) => {
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
else setProgress(task.id, task.tripId, s.done, s.total)
})
@@ -120,7 +128,7 @@ export default function BackgroundTasksWidget() {
}
}, 5000)
return () => clearInterval(iv)
}, [tasks, setProgress, setDone, setError, t])
}, [tasks, setProgress, setError, t])
if (tasks.length === 0) return null
@@ -141,11 +149,11 @@ export default function BackgroundTasksWidget() {
>
<div style={{ flexShrink: 0, marginTop: 1 }}>
{(task.status === 'running' || (task.status === 'done' && task.items === undefined)) && <Loader2 size={16} className="animate-spin" color="var(--accent)" />}
{task.status === 'done' && task.items !== undefined && task.items.length > 0 && <CheckCircle2 size={16} color="var(--success)" />}
{task.status === 'done' && task.items !== undefined && taskFoundSomething(task) && <CheckCircle2 size={16} color="var(--success)" />}
{/* A parse that found nothing is not a success, whatever the job
status says: the same card used to show a green tick next to
"no reservations could be extracted" (#2477). */}
{task.status === 'done' && task.items?.length === 0 && <AlertTriangle size={16} color="var(--warning)" />}
{task.status === 'done' && task.items !== undefined && !taskFoundSomething(task) && <AlertTriangle size={16} color="var(--warning)" />}
{task.status === 'error' && <AlertCircle size={16} color="var(--danger)" />}
</div>
@@ -156,7 +164,7 @@ export default function BackgroundTasksWidget() {
{task.status === 'running' && (
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
{t('reservations.import.parsing')}
{t(task.kind === 'costs' ? 'costs.scan.reading' : 'reservations.import.parsing')}
{task.total > 1 ? ` · ${task.done}/${task.total}` : ''}
</div>
)}
@@ -165,14 +173,14 @@ export default function BackgroundTasksWidget() {
task.items === undefined ? (
// Restored from a reload; items are being re-fetched (see the poll backstop).
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>{t('reservations.import.parsing')}</div>
) : task.items.length > 0 ? (
) : taskFoundSomething(task) ? (
<div>
<button type="button"
onClick={() => review(task)}
className="bg-accent text-accent-text"
style={{ marginTop: 4, border: 'none', borderRadius: 8, padding: '4px 12px', fontSize: 'calc(11.5px * var(--fs-scale-caption, 1))', fontWeight: 600, cursor: 'pointer', fontFamily: 'inherit' }}
>
{t('common.import')}
{t(task.kind === 'costs' ? 'costs.scan.review' : 'common.import')}
</button>
{/* A partly-understood import warns too. Warnings used to be
shown only when nothing at all was found, so anything the
@@ -188,14 +196,14 @@ export default function BackgroundTasksWidget() {
) : (
<div>
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
{t('reservations.import.previewEmpty')}
{t(task.kind === 'costs' ? 'costs.scan.nothingRead' : 'reservations.import.previewEmpty')}
{(task.warnings?.length ?? 0) > 0 && (
<div style={{ color: '#b45309', marginTop: 3, whiteSpace: 'pre-wrap', wordBreak: 'break-word', maxHeight: 96, overflowY: 'auto' }}>
{task.warnings!.join('\n')}
</div>
)}
</div>
{aiParsing && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
{aiParsing && task.kind !== 'costs' && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
<button type="button"
onClick={() => retryWithAi(task)}
disabled={retrying === task.id}
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest'
import { calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, paidByUser, payerSum, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitCents, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
import { calculateTicketShares, expenseEditorFor, newExpenseSeed, receiptToPrefill, finalBudgetFor, finalBudgetSources, hasTicketSplit, paidByUser, payerSum, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitCents, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
describe('splitCents', () => {
it('splits evenly when it divides cleanly', () => {
@@ -310,3 +310,55 @@ describe('finalBudgetSources', () => {
.toEqual({ fronted: [], moved: [], outstanding: [] })
})
})
describe('newExpenseSeed', () => {
it('starts a plain new expense on the base currency, today, with nothing to attach', () => {
expect(newExpenseSeed(undefined, 'eur', [1, 2], '2026-09-24')).toEqual({
currency: 'EUR', day: '2026-09-24', total: '', ticketItems: [], receiptFiles: [],
})
})
it('reads a booking price in the base currency, padded to its decimals', () => {
expect(newExpenseSeed({ amount: 4.9 }, 'EUR', [], '2026-09-24').total).toBe('4.90')
})
it('seeds a scanned receipt: its currency, day, lines shared by everyone, and the photo', () => {
const photo = new File(['x'], 'bill.jpg')
const seed = newExpenseSeed(
{ amount: 1500, currency: 'jpy', date: '2026-09-20', lines: [{ name: 'Ramen', price: 1200 }, { name: 'Tea', price: 300 }], receiptFiles: [photo] },
'EUR', [1, 2], '2026-09-24',
)
expect(seed).toMatchObject({ currency: 'JPY', day: '2026-09-20', total: '1500', receiptFiles: [photo] })
expect(seed.ticketItems.map(i => [i.name, i.price, [...i.participants]])).toEqual([['Ramen', '1200', [1, 2]], ['Tea', '300', [1, 2]]])
})
})
describe('receiptToPrefill', () => {
it('turns a read receipt into an expense prefill, leaving out what was not read', () => {
const photo = new File(['x'], 'bill.jpg')
expect(receiptToPrefill({ merchant: 'Café', date: null, total: 12.5, currency: 'EUR', items: [] }, [photo])).toEqual({
name: 'Café', amount: 12.5, currency: 'EUR', date: undefined, lines: [], receiptFiles: [photo],
})
expect(receiptToPrefill({ merchant: null, date: '2026-09-01', total: null, currency: null, items: [] }, [])).toMatchObject({
name: undefined, amount: undefined, currency: undefined, date: '2026-09-01',
})
})
})
describe('expenseEditorFor', () => {
const closeBooking = () => {}
const closeReceipt = () => {}
it('is nothing when neither opener asks', () => {
expect(expenseEditorFor(null, closeBooking, null, closeReceipt)).toBeNull()
})
it('opens a scanned receipt as a new expense', () => {
expect(expenseEditorFor(null, closeBooking, { name: 'Café' }, closeReceipt)).toEqual({ key: 'receipt', editing: null, prefill: { name: 'Café' }, close: closeReceipt })
})
it('lets the booking win, since it is what was just clicked', () => {
expect(expenseEditorFor({ editing: null, prefill: { amount: 3 } }, closeBooking, { name: 'Café' }, closeReceipt))
.toEqual({ key: 'booking', editing: null, prefill: { amount: 3 }, close: closeBooking })
})
})
@@ -11,8 +11,10 @@
* Amounts are the raw input strings, parsed on use (same as customAmounts).
*/
import type { BudgetParticipantFinal } from '@trek/shared'
import { currencyDecimals } from '../../utils/formatters'
import type { BudgetParticipantFinal, ReceiptRead } from '@trek/shared'
import { amountToInputString, currencyDecimals } from '../../utils/formatters'
import type { ExpensePrefill } from './CostsPanel'
import type { BudgetItem } from '../../types'
// The split and receipt fields guard their own precision on every keystroke, so the
// guard has to follow the currency: a three-decimal one (KWD, BHD, …) seeds three
@@ -264,3 +266,59 @@ export function calculateTicketShares(items: TicketItem[]): { shares: Record<num
return { shares: finalShares, total: totalCents / 100 }
}
/**
* Where a new expense's editor starts from a prefill: a booking's price, or a
* scanned receipt, which also brings its currency, day, lines and photo. Both
* editors (the desktop modal and the phone sheet) seed from this one place.
*
* A prefill without a currency is in `base`, which is also what the currency
* field starts on. Receipt lines seed the Ticket split, shared by everyone like
* a line added by hand, while the split itself stays Equally until the person
* switches.
*/
export function newExpenseSeed(prefill: ExpensePrefill | undefined, base: string, peopleIds: number[], today: string) {
const currency = (prefill?.currency || base).toUpperCase()
return {
currency,
day: prefill?.date || today,
total: prefill?.amount != null ? amountToInputString(prefill.amount, currency) : '',
ticketItems: (prefill?.lines ?? []).map((line, i): TicketItem => ({
id: `receipt-${i}`,
name: line.name,
price: amountToInputString(line.price, currency),
participants: new Set(peopleIds),
})),
receiptFiles: prefill?.receiptFiles ?? [],
}
}
/** A scanned receipt as the prefill of a new expense, with its photo to attach on save. */
export function receiptToPrefill(receipt: ReceiptRead, photos: File[]): ExpensePrefill {
return {
name: receipt.merchant ?? undefined,
amount: receipt.total ?? undefined,
currency: receipt.currency ?? undefined,
date: receipt.date ?? undefined,
lines: receipt.items,
receiptFiles: photos,
}
}
/**
* The expense editor a trip page shows, if any. Two things open it: a booking's
* Costs block (edit that expense, or start one from the booking's price) and a
* receipt scanned from Costs. The booking wins, since it is what the person just
* clicked; `key` remounts the editor when one hands over to the other, so each
* starts from its own seed.
*/
export function expenseEditorFor(
booking: { editing: BudgetItem | null; prefill?: ExpensePrefill } | null,
closeBooking: () => void,
receipt: ExpensePrefill | null,
closeReceipt: () => void,
): { key: string; editing: BudgetItem | null; prefill?: ExpensePrefill; close: () => void } | null {
if (booking) return { key: 'booking', editing: booking.editing, prefill: booking.prefill, close: closeBooking }
if (receipt) return { key: 'receipt', editing: null, prefill: receipt, close: closeReceipt }
return null
}
@@ -1625,6 +1625,33 @@ describe('CostsPanel — expense modal', () => {
expect(onSaved).toHaveBeenCalled()
})
it('FE-W5COSTS-081: offers Scan receipt beside Add expense only when the AI model reads images', async () => {
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })))
const { unmount } = render(<CostsPanel tripId={1} tripMembers={tripMembers} />)
fireEvent.click((await screen.findAllByRole('button', { name: 'Scan receipt' }))[0])
expect(screen.getByRole('dialog', { name: 'Scan a receipt' })).toBeInTheDocument()
unmount()
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
render(<CostsPanel tripId={1} tripMembers={tripMembers} />)
await screen.findAllByRole('button', { name: 'Add expense' })
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
})
it('FE-W5COSTS-080: a scanned receipt opens in its own currency and day, with its photo waiting to be attached', () => {
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
render(
<ExpenseModal tripId={1} base="EUR" people={tripMembers} me={1} editing={null}
prefill={{ name: 'Sushi Dai', amount: 4200, currency: 'JPY', date: '2026-09-20', lines: [{ name: 'Omakase', price: 4200 }], receiptFiles: [photo] }}
onClose={() => {}} onSaved={() => {}} />
)
expect(screen.getByDisplayValue('Sushi Dai')).toBeInTheDocument()
// Yen has no minor unit, so the amount is not padded.
expect(screen.getByDisplayValue('4200')).toBeInTheDocument()
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
})
// The mobile sheet already dates a new expense by the traveller's own clock;
// the desktop modal filed it under the UTC day, so the same expense entered
// late in Tokyo or early in Los Angeles landed on a different day per surface.
+29 -12
View File
@@ -1,6 +1,6 @@
import { Fragment, useState, useEffect, useMemo, useCallback } from 'react'
import { useSearchParams } from 'react-router'
import { ArrowDown, ArrowUp, BarChart3, Plus, Search, ArrowRight, ArrowLeftRight, Check, RotateCcw, Pencil, Trash2, AlertCircle, Download, StickyNote, ChevronDown, Receipt, Paperclip } from 'lucide-react'
import { ArrowDown, ArrowUp, BarChart3, Plus, Search, ArrowRight, ArrowLeftRight, Check, RotateCcw, Pencil, Trash2, AlertCircle, Download, StickyNote, ChevronDown, Receipt, Paperclip, ScanLine } from 'lucide-react'
import { useTripStore } from '../../store/tripStore'
import { useAuthStore } from '../../store/authStore'
import { useSettingsStore } from '../../store/settingsStore'
@@ -19,11 +19,13 @@ import Modal from '../shared/Modal'
import CustomSelect from '../shared/CustomSelect'
import { CustomDatePicker } from '../shared/CustomDateTimePicker'
import { localToday } from '../Planner/today'
import { useReceiptScan } from './useReceiptScan'
import { ReceiptScanModal } from './ReceiptScanModal'
import { SYMBOLS, currenciesWith, SPLIT_COLORS } from './BudgetPanel.constants'
import { amountPattern, calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, NOTE_MAX, paidByUser, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
import { amountPattern, calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, NOTE_MAX, paidByUser, payersBalanced, readTicketItems, newExpenseSeed, readUserNote, rebalancePayers, settlementDate, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
import { COST_CATEGORY_LIST, catMeta } from './costsCategories'
import { ReceiptPreviewModal } from './ReceiptPreviewModal'
import type { BudgetParticipantFinal, BudgetUnconverted } from '@trek/shared'
import type { BudgetParticipantFinal, BudgetUnconverted, ReceiptLine } from '@trek/shared'
import type { BudgetItem, BudgetItemReceipt } from '../../types'
import type { TripMember } from './BudgetPanelMemberChips'
import GuestBadge from '../shared/GuestBadge'
@@ -83,6 +85,7 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
const me = useAuthStore(s => s.user?.id ?? -1)
const can = useCanDo()
const canEdit = can('budget_edit', trip)
const receiptScan = useReceiptScan(tripId, canEdit)
const toast = useToast()
const { t, locale } = useTranslation()
const isMobile = useIsMobile()
@@ -387,7 +390,8 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
<div style={{ maxWidth: '100%', margin: '0 auto' }}>
<CostsToolbar dateMeta={dateMeta} people={people} me={me} colorFor={colorFor}
canEdit={canEdit} canSettle={(settlement?.flows || []).length > 0}
onSettleAll={settleAll} onAddExpense={() => { setEditing(null); setModalOpen(true) }} />
onSettleAll={settleAll} onAddExpense={() => { setEditing(null); setModalOpen(true) }}
onScanReceipt={receiptScan.offered ? receiptScan.open : undefined} />
{/* ── Summary cards ── */}
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(4, 1fr)', gap: 16, marginBottom: 36 }} className="costs-summary">
@@ -509,6 +513,8 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
</div>
</div>)}
<ReceiptScanModal scan={receiptScan} />
{modalOpen && (
<ExpenseModal tripId={tripId} base={base} people={people} me={me} editing={editing}
onClose={() => setModalOpen(false)}
@@ -634,6 +640,13 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
<Plus size={17} /> {t('costs.addExpense')}
</button>
)}
{receiptScan.offered && (
<button type="button" onClick={receiptScan.open}
style={{ marginTop: 8, width: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 8, background: 'transparent', border: '1px solid rgba(255,255,255,0.16)', color: '#fff', padding: 11, borderRadius: 14, fontSize: 'calc(13px * var(--fs-scale-body, 1))', fontWeight: 500, cursor: 'pointer', fontFamily: 'inherit' }} // theme-lint-disable — on the dark total card, drawn like Add expense above it
>
<ScanLine size={16} /> {t('costs.scan.button')}
</button>
)}
</section>
{/* Owe / Owed */}
@@ -1237,6 +1250,11 @@ export interface ExpensePrefill {
reservationId?: number
/** Set when the expense is being created from a place (#1298). */
placeId?: number
/** The rest comes from a scanned receipt: its currency, day and lines, and the photo, attached on save. */
currency?: string
date?: string
lines?: ReceiptLine[]
receiptFiles?: File[]
}
export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClose, onSaved }: {
@@ -1252,17 +1270,16 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
const [name, setName] = useState(editing?.name || prefill?.name || '')
const [cat, setCat] = useState<string>(editing ? catMeta(editing.category).key : (prefill?.category || 'food'))
const [currency, setCurrency] = useState(editingCurrency)
const [day, setDay] = useState(editing?.expense_date || localToday())
const [seed] = useState(() => newExpenseSeed(prefill, base, people.map(p => p.id), localToday()))
const [currency, setCurrency] = useState(editing ? editingCurrency : seed.currency)
const [day, setDay] = useState(editing ? (editing.expense_date || localToday()) : seed.day)
const [note, setNote] = useState(() => readUserNote(editing))
// Edit and prefill seeds are padded to the currency's decimals (#2175): the DB
// returns numbers, so a saved 4,90 would otherwise reopen as "4,9" and a saved
// 5,00 as "5". A prefill has no currency of its own — it is read as `base`,
// which is also what the currency field starts on.
// 5,00 as "5".
const [total, setTotal] = useState<string>(() => {
if (editing) return editing.total_price ? amountToInputString(editing.total_price, editingCurrency) : ''
if (prefill?.amount != null) return amountToInputString(prefill.amount, base)
return ''
return seed.total
})
const [participants, setParticipants] = useState<Set<number>>(() =>
editing ? new Set((editing.members || []).map(m => m.user_id)) : new Set(people.map(p => p.id)))
@@ -1303,7 +1320,7 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
return 'equally'
})
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => readTicketItems(editing))
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => editing ? readTicketItems(editing) : seed.ticketItems)
const [customAmounts, setCustomAmounts] = useState<Record<number, string>>(() => {
const m: Record<number, string> = {}
@@ -1318,7 +1335,7 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
})
const [receipts, setReceipts] = useState<BudgetItemReceipt[]>(() => editing?.receipts || [])
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>([])
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>(() => editing ? [] : seed.receiptFiles)
const [uploadingReceipt, setUploadingReceipt] = useState(false)
const [modalPreviewReceipts, setModalPreviewReceipts] = useState<{ receipts: BudgetItemReceipt[]; initialIndex: number } | null>(null)
@@ -1,4 +1,4 @@
// FE-COSTSBAR-001 to FE-COSTSBAR-007
// FE-COSTSBAR-001 to FE-COSTSBAR-008
import { describe, it, expect, vi } from 'vitest'
import { render, screen, fireEvent } from '../../../tests/helpers/render'
import CostsToolbar from './CostsToolbar'
@@ -66,4 +66,18 @@ describe('CostsToolbar', () => {
expect(screen.queryByRole('button')).toBeNull()
expect(screen.getByText('8 days')).toBeInTheDocument()
})
it('FE-COSTSBAR-008: offers Scan receipt only when handed a way to open it', () => {
const onScanReceipt = vi.fn()
const { unmount } = render(
<CostsToolbar dateMeta={null} people={PEOPLE} me={1} colorFor={() => '#000'} canEdit canSettle
onSettleAll={vi.fn()} onAddExpense={vi.fn()} onScanReceipt={onScanReceipt} />,
)
fireEvent.click(screen.getByRole('button', { name: 'Scan receipt' }))
expect(onScanReceipt).toHaveBeenCalledTimes(1)
unmount()
setup()
expect(screen.queryByRole('button', { name: 'Scan receipt' })).toBeNull()
})
})
+10 -2
View File
@@ -1,4 +1,4 @@
import { Check, Plus } from 'lucide-react'
import { Check, Plus, ScanLine } from 'lucide-react'
import { useTranslation } from '../../i18n'
import type { TripMember } from './BudgetPanelMemberChips'
@@ -13,6 +13,8 @@ interface CostsToolbarProps {
canSettle: boolean
onSettleAll: () => void
onAddExpense: () => void
/** Opens the receipt scan; left out when the AI model reads no images. */
onScanReceipt?: () => void
}
const BODY_SIZE = 'calc(13px * var(--fs-scale-body, 1))'
@@ -22,7 +24,7 @@ const BODY_SIZE = 'calc(13px * var(--fs-scale-body, 1))'
* Bookings, Lists and Files open with: the tab's name, then what the numbers
* cover (the trip's days and who travels), and the two actions on the right.
*/
export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit, canSettle, onSettleAll, onAddExpense }: CostsToolbarProps) {
export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit, canSettle, onSettleAll, onAddExpense, onScanReceipt }: CostsToolbarProps) {
const { t } = useTranslation()
const chip = 'inline-flex items-center whitespace-nowrap rounded-full bg-surface-card px-3 py-1.5 font-medium text-content-muted shadow-sm'
const button = 'inline-flex items-center gap-1.5 rounded-[10px] border-0 px-[14px] py-[9px] font-medium hover:opacity-[0.88]'
@@ -64,6 +66,12 @@ export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit,
<Check size={14} strokeWidth={2.5} />
{t('costs.settleUp')}
</button>
{onScanReceipt && (
<button type="button" onClick={onScanReceipt} className={`${button} bg-surface-card text-content`}>
<ScanLine size={14} strokeWidth={2.5} />
{t('costs.scan.button')}
</button>
)}
<button type="button" onClick={onAddExpense} className={`${button} bg-accent text-accent-text`}>
<Plus size={14} strokeWidth={2.5} />
{t('costs.addExpense')}
@@ -0,0 +1,148 @@
// FE-BUDGET-SCAN-001 to FE-BUDGET-SCAN-011
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { http, HttpResponse } from 'msw'
import { render, screen, fireEvent, waitFor } from '../../../tests/helpers/render'
import { server } from '../../../tests/helpers/msw/server'
import { saveImportFiles } from '../../db/offlineDb'
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
import { useReceiptScan } from './useReceiptScan'
import { ReceiptScanModal } from './ReceiptScanModal'
import MReceiptScanButton from '../../mobile/screens/trip/tabs/MReceiptScanButton'
vi.mock('../../db/offlineDb', () => ({ saveImportFiles: vi.fn(async () => {}) }))
function Harness({ canEdit = true }: { canEdit?: boolean }) {
const scan = useReceiptScan(7, canEdit)
return (
<>
{scan.offered && <button type="button" onClick={scan.open}>Scan receipt</button>}
<ReceiptScanModal scan={scan} />
</>
)
}
const input = () => screen.getByTestId('receipt-scan-input') as HTMLInputElement
const photo = (name = 'bill.jpg', size = 4) => {
const f = new File(['x'.repeat(Math.min(size, 16))], name, { type: 'image/jpeg' })
if (size > 16) Object.defineProperty(f, 'size', { value: size })
return f
}
async function openDialog() {
fireEvent.click(await screen.findByRole('button', { name: 'Scan receipt' }))
return screen.getByRole('dialog', { name: 'Scan a receipt' })
}
beforeEach(() => {
vi.mocked(saveImportFiles).mockClear()
useBackgroundTasksStore.setState({ tasks: [] })
server.use(
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })),
http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({ jobId: 'job-5' })),
)
})
describe('Scan receipt', () => {
it('FE-BUDGET-SCAN-001: offers the scan when the model reads images, and opens a dialog saying what it takes', async () => {
render(<Harness />)
await openDialog()
expect(screen.getByText(/A photo of one receipt \(JPG, PNG or WEBP, up to 10 MB\)/)).toBeInTheDocument()
expect(input().accept).toContain('image/jpeg')
expect(screen.getByRole('button', { name: 'Scan' })).toBeDisabled()
})
it('FE-BUDGET-SCAN-002: offers nothing, and asks nothing, without budget_edit', async () => {
let asked = false
server.use(http.get('/api/llm/capabilities', () => { asked = true; return HttpResponse.json({ images: true }) }))
render(<Harness canEdit={false} />)
await new Promise(r => setTimeout(r, 20))
expect(asked).toBe(false)
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
})
it('FE-BUDGET-SCAN-003: offers nothing when the model reads no images', async () => {
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
render(<Harness />)
await new Promise(r => setTimeout(r, 20))
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
})
it('FE-BUDGET-SCAN-004: a picked photo is named, then Scan starts a costs job the widget follows and closes the dialog', async () => {
render(<Harness />)
await openDialog()
const picked = photo()
fireEvent.change(input(), { target: { files: [picked] } })
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: 'Scan' }))
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks[0]).toMatchObject({ id: 'job-5', tripId: '7', label: 'bill.jpg', kind: 'costs', status: 'running' }))
expect(saveImportFiles).toHaveBeenCalledWith('job-5', [picked])
await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument())
})
it('FE-BUDGET-SCAN-005: a dropped photo is taken like a picked one', async () => {
render(<Harness />)
await openDialog()
const zone = screen.getByText('Drop the receipt photo here, or click to pick or take one').closest('button')!
fireEvent.dragOver(zone)
expect(screen.getByText('Drop the photo to scan it')).toBeInTheDocument()
fireEvent.drop(zone, { dataTransfer: { files: [photo('drop.png')] } })
expect(screen.getByText('drop.png')).toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Scan' })).toBeEnabled()
})
it('FE-BUDGET-SCAN-006: refuses what is not a photo, and a photo over 10 MB, before anything is sent', async () => {
let sent = false
server.use(http.post('/api/trips/7/budget/receipt-scan', () => { sent = true; return HttpResponse.json({ jobId: 'x' }) }))
render(<Harness />)
await openDialog()
fireEvent.change(input(), { target: { files: [new File(['%PDF'], 'bill.pdf', { type: 'application/pdf' })] } })
expect(screen.getByRole('alert')).toHaveTextContent('Only a photo can be scanned: JPG, PNG or WEBP.')
fireEvent.change(input(), { target: { files: [photo('big.jpg', 11 * 1024 * 1024)] } })
expect(screen.getByRole('alert')).toHaveTextContent('File "big.jpg" exceeds 10 MB limit.')
expect(screen.getByRole('button', { name: 'Scan' })).toBeDisabled()
expect(sent).toBe(false)
})
it('FE-BUDGET-SCAN-007: says why when the server refuses, and keeps the dialog open', async () => {
server.use(http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({ error: 'The configured AI model does not read photos' }, { status: 400 })))
render(<Harness />)
await openDialog()
fireEvent.change(input(), { target: { files: [photo()] } })
fireEvent.click(screen.getByRole('button', { name: 'Scan' }))
expect(await screen.findByRole('alert')).toHaveTextContent('The configured AI model does not read photos')
expect(screen.getByRole('dialog')).toBeInTheDocument()
expect(useBackgroundTasksStore.getState().tasks).toEqual([])
})
it('FE-BUDGET-SCAN-008: Cancel, the close button and the backdrop close it; reopening starts empty', async () => {
render(<Harness />)
const dialog = await openDialog()
fireEvent.change(input(), { target: { files: [photo()] } })
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }))
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
await openDialog()
expect(screen.queryByText('bill.jpg')).not.toBeInTheDocument()
fireEvent.click(screen.getByRole('button', { name: 'Close' }))
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
await openDialog()
fireEvent.click(screen.getByRole('dialog').parentElement!)
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
expect(dialog).not.toBeInTheDocument()
})
})
describe('MReceiptScanButton', () => {
it('FE-BUDGET-SCAN-009: the phone header button opens the same dialog', async () => {
render(<MReceiptScanButton tripId={7} canEdit />)
fireEvent.click(await screen.findByRole('button', { name: 'Scan receipt' }))
expect(screen.getByRole('dialog', { name: 'Scan a receipt' })).toBeInTheDocument()
})
it('FE-BUDGET-SCAN-010: draws nothing when the scan is not offered', async () => {
const { container } = render(<MReceiptScanButton tripId={7} canEdit={false} />)
await new Promise(r => setTimeout(r, 20))
expect(container).toBeEmptyDOMElement()
})
})
@@ -0,0 +1,67 @@
import { createPortal } from 'react-dom'
import { useRef, useState } from 'react'
import { ScanLine, X } from 'lucide-react'
import { useTranslation } from '../../i18n'
import { RECEIPT_PHOTO_ACCEPT, type ReceiptScan } from './useReceiptScan'
/**
* The dialog behind "Scan receipt", laid out like the booking import's: what it
* takes, a drop zone that is also the picker (on a phone the picker offers the
* camera), then Scan, which hands the photo to the background read and closes.
*/
export function ReceiptScanModal({ scan }: { scan: ReceiptScan }) {
const { t } = useTranslation()
const inputRef = useRef<HTMLInputElement>(null)
const [dragOver, setDragOver] = useState(false)
if (!scan.isOpen) return null
const onDrop = (e: React.DragEvent) => {
e.preventDefault()
setDragOver(false)
scan.choose(Array.from(e.dataTransfer.files))
}
return createPortal(
<div role="presentation" className="fixed inset-0 z-[99999] flex items-center justify-center bg-black/40 p-4" onClick={(e) => { if (e.target === e.currentTarget) scan.close() }}>
<div role="dialog" aria-modal="true" aria-label={t('costs.scan.title')} className="flex max-h-[90vh] w-full max-w-[540px] flex-col rounded-2xl bg-surface-card p-6 shadow-2xl">
<div className="mb-3.5 flex items-center gap-2">
<h2 className="flex-1 text-subtitle font-bold text-content">{t('costs.scan.title')}</h2>
<button type="button" onClick={scan.close} aria-label={t('common.close')} className="flex rounded-md p-1 text-content-faint">
<X size={16} />
</button>
</div>
<div className="min-h-0 flex-1 overflow-y-auto">
<p className="mb-3.5 text-caption leading-snug text-content-faint">{t('costs.scan.accepted')}</p>
<input ref={inputRef} type="file" accept={RECEIPT_PHOTO_ACCEPT} hidden data-testid="receipt-scan-input"
onChange={(e) => { scan.choose(e.target.files ? Array.from(e.target.files) : []); e.target.value = '' }} />
<button type="button" onClick={() => inputRef.current?.click()}
onDragOver={(e) => { e.preventDefault(); setDragOver(true) }}
onDragEnter={(e) => { e.preventDefault(); setDragOver(true) }}
onDragLeave={(e) => { if (e.target === e.currentTarget) setDragOver(false) }}
onDrop={onDrop}
className={`mb-3 flex min-h-[100px] w-full flex-col items-center justify-center gap-1.5 rounded-xl border-2 border-dashed p-4 text-body font-medium transition-colors ${dragOver ? 'border-accent bg-surface-tertiary' : 'border-edge bg-transparent'}`}>
<ScanLine size={18} strokeWidth={1.8} className={`pointer-events-none ${dragOver ? 'text-accent' : 'text-content-faint'}`} />
<span className={`pointer-events-none break-all text-center ${scan.photo ? 'text-content' : 'text-content-faint'}`}>
{dragOver ? t('costs.scan.dropActive') : scan.photo ? scan.photo.name : t('costs.scan.dropHere')}
</span>
</button>
{scan.error && (
<div role="alert" className="mt-2 rounded-lg border border-danger bg-danger-soft px-2.5 py-2 text-caption text-danger">{scan.error}</div>
)}
</div>
<div className="mt-3.5 flex justify-end gap-2 border-t border-edge-secondary pt-3.5">
<button type="button" onClick={scan.close} className="rounded-lg border border-edge px-4 py-2 text-body font-medium text-content">
{t('common.cancel')}
</button>
<button type="button" onClick={scan.start} disabled={!scan.photo || scan.starting}
className={`rounded-lg px-4 py-2 text-body font-medium ${scan.photo && !scan.starting ? 'bg-accent text-accent-text' : 'bg-surface-tertiary text-content-faint'}`}>
{scan.starting ? t('costs.scan.reading') : t('costs.scan.start')}
</button>
</div>
</div>
</div>,
document.body,
)
}
@@ -0,0 +1,71 @@
import { useEffect, useState } from 'react'
import { useTranslation } from '../../i18n'
import { llmRepo } from '../../repo/llmRepo'
import { saveImportFiles } from '../../db/offlineDb'
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'
/** The picker's filter: the extensions, and the MIME types a phone's camera roll matches on. */
export const RECEIPT_PHOTO_ACCEPT = [...LLM_PHOTO_EXTENSIONS, 'image/jpeg', 'image/png', 'image/webp'].join(',')
const MAX_PHOTO_BYTES = 10 * 1024 * 1024
/**
* "Scan a receipt" in Costs, for both shells. The button is only offered when
* the person may add expenses and their AI model reads images. It opens a
* dialog like the booking import's; the picked photo is read in the background
* like a booking import too: the background tasks widget follows it, and its
* review opens the expense editor pre-filled, with the photo waiting to be
* attached on save. The photo is kept in IndexedDB so a reload mid-read still
* has it for that review.
*/
export function useReceiptScan(tripId: number, canEdit: boolean) {
const { t } = useTranslation()
const addTask = useBackgroundTasksStore((s) => s.addTask)
const [readsPhotos, setReadsPhotos] = useState(false)
const [isOpen, setIsOpen] = useState(false)
const [photo, setPhoto] = useState<File | null>(null)
const [error, setError] = useState('')
const [starting, setStarting] = useState(false)
useEffect(() => {
if (!canEdit) return
let live = true
llmRepo.readsPhotos().then((v) => { if (live) setReadsPhotos(v) }).catch(() => { if (live) setReadsPhotos(false) })
return () => { live = false }
}, [canEdit])
const open = () => { setPhoto(null); setError(''); setIsOpen(true) }
const close = () => { if (!starting) setIsOpen(false) }
/** Take the first of the picked or dropped files, if it is a photo the model can read. */
const choose = (files: File[]) => {
const picked = files[0]
if (!picked) return
const ext = '.' + (picked.name.toLowerCase().split('.').pop() ?? '')
if (!(LLM_PHOTO_EXTENSIONS as readonly string[]).includes(ext)) { setError(t('costs.scan.unsupported')); return }
if (picked.size > MAX_PHOTO_BYTES) { setError(t('reservations.import.fileTooLarge', { name: picked.name })); return }
setPhoto(picked)
setError('')
}
const start = async () => {
if (!photo || starting) return
setStarting(true)
setError('')
try {
const jobId = await llmRepo.scanReceipt(tripId, photo)
await saveImportFiles(jobId, [photo])
addTask({ id: jobId, tripId: String(tripId), label: photo.name, total: 1, files: [photo], kind: 'costs' })
setIsOpen(false)
} catch (err) {
const message = (err as { response?: { data?: { error?: string } } })?.response?.data?.error
setError(message ?? t('costs.scan.failed'))
} finally {
setStarting(false)
}
}
return { offered: canEdit && readsPhotos, isOpen, open, close, photo, choose, error, starting, start }
}
export type ReceiptScan = ReturnType<typeof useReceiptScan>
@@ -197,3 +197,38 @@ describe('BookingImportModal', () => {
expect(onClose).toHaveBeenCalledTimes(1);
});
});
describe('BookingImportModal with a model that reads images', () => {
const jpg = () => new File(['jpeg'], 'ticket.jpg', { type: 'image/jpeg' });
it('FE-PLANNER-BOOKIMP-020: offers photos, and imports one with AI, when the model reads images', async () => {
let mode: string | null = null;
server.use(
http.get('/api/health/features', () => HttpResponse.json({ bookingImport: true, aiParsing: true })),
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })),
http.post('/api/trips/4/reservations/import/booking/async', async ({ request }) => {
mode = (await readMultipart(request)).fields.mode ?? null;
return HttpResponse.json({ jobId: 'job-2' });
}),
);
render(<BookingImportModal {...defaultProps} kind="transports" />);
expect(await screen.findByText(/Photos \(JPG, PNG, WEBP\) are read by the AI model\./)).toBeInTheDocument();
expect(fileInput().accept).toContain('.jpg');
fireEvent.change(fileInput(), { target: { files: [jpg()] } });
expect(screen.getByText('ticket.jpg')).toBeInTheDocument();
await userEvent.click(screen.getByRole('button', { name: 'Import' }));
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks.find(t => t.id === 'job-2')).toMatchObject({ kind: 'transports' }));
expect(mode).toBe('fallback-on-empty');
});
it('FE-PLANNER-BOOKIMP-021: refuses a photo while the model reads no images', async () => {
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })));
render(<BookingImportModal {...defaultProps} />);
await waitFor(() => expect(fileInput().accept).not.toContain('.jpg'));
fireEvent.change(fileInput(), { target: { files: [jpg()] } });
expect(screen.getByText('Unsupported file format. Use EML, PDF, PKPass, HTML, or TXT.')).toBeInTheDocument();
expect(screen.queryByText(/Photos \(JPG/)).not.toBeInTheDocument();
});
});
@@ -3,6 +3,8 @@ import { useState, useRef, useEffect } from 'react'
import { Upload, X } from 'lucide-react'
import { useTranslation } from '../../i18n'
import { reservationsApi, healthApi } from '../../api/client'
import { llmRepo } from '../../repo/llmRepo'
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
import { saveImportFiles } from '../../db/offlineDb'
@@ -36,6 +38,9 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
const [loading, setLoading] = useState(false)
const [error, setError] = useState('')
const [aiParsing, setAiParsing] = useState(false)
const [readsPhotos, setReadsPhotos] = useState(false)
// Only a model that reads images can do anything with a photo, so photos are offered only then.
const acceptedExts: readonly string[] = readsPhotos ? [...ACCEPTED_EXTS, ...LLM_PHOTO_EXTENSIONS] : ACCEPTED_EXTS
const reset = () => {
setFiles([])
@@ -53,13 +58,14 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
useEffect(() => {
if (!isOpen) return
healthApi.features().then((f) => setAiParsing(!!f.aiParsing)).catch(() => setAiParsing(false))
llmRepo.readsPhotos().then(setReadsPhotos).catch(() => setReadsPhotos(false))
}, [isOpen])
const handleClose = () => { reset(); onClose() }
const validateFile = (f: File): string | null => {
const ext = ('.' + f.name.toLowerCase().split('.').pop()) as string
if (!ACCEPTED_EXTS.includes(ext)) return t('reservations.import.unsupportedFormat')
if (!acceptedExts.includes(ext)) return t('reservations.import.unsupportedFormat')
if (f.size > MAX_FILE_BYTES) return t('reservations.import.fileTooLarge', { name: f.name })
return null
}
@@ -143,12 +149,13 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
<div style={{ flex: 1, overflowY: 'auto', minHeight: 0 }}>
<div style={{ fontSize: 'calc(12px * var(--fs-scale-body, 1))', color: 'var(--text-faint)', marginBottom: 14, lineHeight: 1.45 }}>
{t('reservations.import.acceptedFormats')}
{readsPhotos && <> {t('reservations.import.acceptedPhotos')}</>}
</div>
<input
ref={fileInputRef}
type="file"
accept={ACCEPTED_EXTS.join(',')}
accept={acceptedExts.join(',')}
multiple
style={{ display: 'none' }}
onChange={handleInputChange}
@@ -40,7 +40,7 @@ function renderSection() {
}
function multimodalToggle(): HTMLElement {
const row = screen.getByText('Send documents as images').parentElement as HTMLElement;
const row = screen.getByText('Model reads images').parentElement as HTMLElement;
return row.querySelector('button') as HTMLElement;
}
@@ -12,6 +12,7 @@ import DawarichIcon from '../../../components/shared/DawarichIcon'
import AirTrailIcon from '../../../components/shared/AirTrailIcon'
import { DOCUMENT_PROVIDER_ICONS } from '../../../components/shared/DocumentProviderIcons'
import MToggle from '../../components/MToggle'
import { asLlmVision, LLM_VISION_MODES, type LlmVision } from '@trek/shared'
import { MAdminButton, MAdminCard, MAdminField, MAdminInput, MAdminSecretInput } from './MAdminUi'
const ICON_MAP = {
@@ -391,12 +392,14 @@ const RECOMMENDED_MODELS: { id: string; label: string; note: string; recommended
* provider, it also lists installed Ollama models and can pull NuExtract models.
*/
function LlmParsingConfig({ addon }: { addon: Addon }) {
const { t } = useTranslation()
const toast = useToast()
const cfg = (addon.config ?? {}) as Record<string, unknown>
const [provider, setProvider] = useState<string>((cfg.provider as string) ?? 'local')
const [model, setModel] = useState<string>((cfg.model as string) ?? '')
const [baseUrl, setBaseUrl] = useState<string>((cfg.baseUrl as string) ?? '')
const [apiKey, setApiKey] = useState<string>((cfg.apiKey as string) ?? '')
const [vision, setVision] = useState<LlmVision>(asLlmVision(cfg.vision))
const [saving, setSaving] = useState(false)
// Local-provider model management.
@@ -458,7 +461,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
setSaving(true)
try {
// Send the masked sentinel unchanged so the server keeps the stored key.
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: baseUrl.trim(), apiKey, multimodal: cfg.multimodal === true } })
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: baseUrl.trim(), apiKey, vision } })
toast.success('Saved')
} catch {
toast.error('Failed to save')
@@ -546,6 +549,31 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
placeholder={provider === 'anthropic' ? 'claude-opus-4-8' : provider === 'openai' ? 'gpt-4o' : 'select or pull below'}
/>
<MAdminField label={t('settings.aiParsing.multimodal')}>
<div role="radiogroup" aria-label={t('settings.aiParsing.multimodal')} className="flex gap-[6px]">
{LLM_VISION_MODES.map((mode) => {
const active = vision === mode
return (
<button
key={mode}
type="button"
role="radio"
aria-checked={active}
onClick={() => setVision(mode)}
className={`flex-1 rounded-xl border px-3 py-[10px] text-[0.8125rem] font-semibold text-m-ink ${
active ? 'border-[color:var(--m-act)] bg-[color:var(--m-ic)]' : 'border-[color:var(--m-rowbr)]'
}`}
>
{t(`admin.addons.llm.vision.${mode}`)}
</button>
)
})}
</div>
</MAdminField>
<p className="font-geist text-[0.625rem] leading-relaxed text-m-faint">
{t(provider === 'local' ? 'admin.addons.llm.vision.hintLocal' : 'admin.addons.llm.vision.hintCloud')}
</p>
{/* Local model management (Ollama) */}
{provider === 'local' && (
<div className="space-y-3 rounded-xl border border-[color:var(--m-rowbr)] bg-[color:var(--m-ic)] p-3">
@@ -13,6 +13,7 @@ import MMapArea from './map/MMapArea'
import MPlacesBrowser from './places/MPlacesBrowser'
import MTripTabPanel from './tabs/MTripTabPanel'
import MTripSheets from './sheets/MTripSheets'
import MReceiptScanButton from './tabs/MReceiptScanButton'
import MTripLoadingSplash from './MTripLoadingSplash'
import { usePluginDayTints, dayTintBackground } from '../../../components/Plugins/PluginDaySchedule'
import { stageOf } from '../../../components/Roadtrip/roadtripRowModel'
@@ -593,6 +594,7 @@ export default function MTripShell({
{trTab === 'finanzplan' && (
<div className="absolute left-1/2 top-1/2 flex -translate-x-1/2 -translate-y-1/2 items-center gap-[7px]">
<PrimaryPill label={t('costs.addExpense')} onClick={() => setAddExpenseSignal(s => s + 1)} />
<MReceiptScanButton tripId={tripId} canEdit={planner.can('budget_edit', trip)} />
<MIconBtn ariaLabel={t('budget.exportCsv')} onClick={() => setExportCostsCsvSignal(s => s + 1)} size={40} className="text-m-muted backdrop-blur-[24px] backdrop-saturate-[1.7]">
<FileDown size={15} strokeWidth={2} />
</MIconBtn>
@@ -15,7 +15,7 @@ import { splitShareLabel, useExpenseFx } from '../../../../components/Budget/exp
import { SYMBOLS, SPLIT_COLORS, currenciesWith } from '../../../../components/Budget/BudgetPanel.constants'
import { COST_CATEGORY_LIST, catMeta } from '../../../../components/Budget/costsCategories'
import { localToday } from '../../../../components/Planner/today'
import { amountPattern, calculateTicketShares, hasTicketSplit, NOTE_MAX, readTicketItems, readUserNote, splitEqualShares, writeTicketItems, type TicketItem } from '../../../../components/Budget/CostsPanel.helpers'
import { amountPattern, calculateTicketShares, hasTicketSplit, newExpenseSeed, NOTE_MAX, readTicketItems, readUserNote, splitEqualShares, writeTicketItems, type TicketItem } from '../../../../components/Budget/CostsPanel.helpers'
import type { ExpensePrefill } from '../../../../components/Budget/CostsPanel'
import { payersBalanced, rebalancePayers } from '../../../../components/Budget/CostsPanel.helpers'
import GuestBadge from '../../../../components/shared/GuestBadge'
@@ -76,15 +76,14 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
const [cat, setCat] = useState<string>(editing ? catMeta(editing.category).key : (prefill?.category || 'food'))
const [catOpen, setCatOpen] = useState(false)
const [note, setNote] = useState(() => readUserNote(editing))
const [currency, setCurrency] = useState(editingCurrency)
const [day, setDay] = useState(editing?.expense_date || localToday())
const [seed] = useState(() => newExpenseSeed(prefill, base, people.map(p => p.id), localToday()))
const [currency, setCurrency] = useState(editing ? editingCurrency : seed.currency)
const [day, setDay] = useState(editing ? (editing.expense_date || localToday()) : seed.day)
// Edit and prefill seeds are padded to the currency's decimals (#2175), same
// as the desktop modal: a saved 4,90 must reopen as "4,90", not "4,9". A
// prefill has no currency of its own and is read as `base`.
// as the desktop modal: a saved 4,90 must reopen as "4,90", not "4,9".
const [total, setTotal] = useState<string>(() => {
if (editing) return editing.total_price ? amountToInputString(editing.total_price, editingCurrency) : ''
if (prefill?.amount != null) return amountToInputString(prefill.amount, base)
return ''
return seed.total
})
const [participants, setParticipants] = useState<Set<number>>(() =>
editing ? new Set((editing.members || []).map(m => m.user_id)) : new Set(people.map(p => p.id)))
@@ -115,7 +114,7 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
return 'equally'
})
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => readTicketItems(editing))
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => editing ? readTicketItems(editing) : seed.ticketItems)
const [customAmounts, setCustomAmounts] = useState<Record<number, string>>(() => {
const m: Record<number, string> = {}
@@ -128,7 +127,7 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
})
const [receipts, setReceipts] = useState<BudgetItemReceipt[]>(() => editing?.receipts || [])
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>([])
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>(() => editing ? [] : seed.receiptFiles)
const [uploadingReceipt, setUploadingReceipt] = useState(false)
const [previewReceipts, setPreviewReceipts] = useState<{ receipts: BudgetItemReceipt[]; initialIndex: number } | null>(null)
@@ -5,6 +5,7 @@ import AirTrailImportModal from '../../../../components/Planner/AirTrailImportMo
import TripFormModal from '../../../../components/Trips/TripFormModal'
import TripMembersModal from '../../../../components/Trips/TripMembersModal'
import type { ExpensePrefill } from '../../../../components/Budget/CostsPanel'
import { expenseEditorFor } from '../../../../components/Budget/CostsPanel.helpers'
import { useAuthStore } from '../../../../store/authStore'
import { useSettingsStore } from '../../../../store/settingsStore'
import { useTripStore } from '../../../../store/tripStore'
@@ -59,6 +60,7 @@ export default function MTripSheets({ planner, shell }: MTripSheetsProps) {
else if (req.prefill) setBookingExpense({ editing: null, prefill: req.prefill })
}
const costsBase = (displayCurrency || trip?.currency || 'EUR').toUpperCase()
const expenseEditor = expenseEditorFor(bookingExpense, () => setBookingExpense(null), planner.receiptExpense, planner.clearReceiptExpense)
return (
<>
@@ -142,16 +144,17 @@ export default function MTripSheets({ planner, shell }: MTripSheetsProps) {
/>
)}
{bookingExpense && (
{expenseEditor && (
<MCostSheet
key={expenseEditor.key}
tripId={tripId}
base={costsBase}
people={planner.tripMembers}
me={meId}
editing={bookingExpense.editing}
prefill={bookingExpense.prefill}
onClose={() => setBookingExpense(null)}
onSaved={() => { setBookingExpense(null); loadBudgetItems(tripId) }}
editing={expenseEditor.editing}
prefill={expenseEditor.prefill}
onClose={expenseEditor.close}
onSaved={() => { expenseEditor.close(); loadBudgetItems(tripId) }}
/>
)}
@@ -0,0 +1,20 @@
import { ScanLine } from 'lucide-react'
import MIconBtn from '../../../components/MIconBtn'
import { useReceiptScan } from '../../../../components/Budget/useReceiptScan'
import { ReceiptScanModal } from '../../../../components/Budget/ReceiptScanModal'
import { useTranslation } from '../../../../i18n'
/** The Costs header's "Scan receipt" button on a phone; the dialog and the scan are the shared ones. */
export default function MReceiptScanButton({ tripId, canEdit }: { tripId: number; canEdit: boolean }) {
const { t } = useTranslation()
const scan = useReceiptScan(tripId, canEdit)
if (!scan.offered) return null
return (
<>
<MIconBtn ariaLabel={t('costs.scan.button')} onClick={scan.open} size={40} className="text-m-muted backdrop-blur-[24px] backdrop-saturate-[1.7]">
<ScanLine size={15} strokeWidth={2} />
</MIconBtn>
<ReceiptScanModal scan={scan} />
</>
)
}
+11 -5
View File
@@ -28,6 +28,7 @@ import AirTrailImportModal from '../components/Planner/AirTrailImportModal'
import ApplyTemplateButton from '../components/Packing/ApplyTemplateButton'
import PackingExportMenu from '../components/Packing/PackingExportMenu'
import type { ExpensePrefill } from '../components/Budget/CostsPanel'
import { expenseEditorFor } from '../components/Budget/CostsPanel.helpers'
import type { BookingExpenseRequest } from '../components/Planner/BookingCostsSection.types'
import type { BudgetItem } from '../types'
import PluginFrame from '../components/Plugins/PluginFrame'
@@ -309,6 +310,7 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
transportModalDayId, setTransportModalDayId,
transportModalAutomated, setTransportModalAutomated, transitPrefill, setTransitPrefill, transitJourney, setTransitJourney,
reservationPrefill, transportPrefill, importReviewActive, advanceImportReview,
receiptExpense, clearReceiptExpense,
routeShown, setRouteShown, transitRoutesShown, routeProfile, setRouteProfile, routeVias, fitKey, setFitKey,
mobileSidebarOpen, setMobileSidebarOpen, mobilePlanScrollTopRef, mobilePlacesScrollTopRef,
deletePlaceId, setDeletePlaceId, deletePlaceIds, setDeletePlaceIds, deletePlaceNote, deletePlacesNote,
@@ -375,6 +377,9 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
if (req.editItem) setBookingExpense({ editing: req.editItem })
else if (req.prefill) setBookingExpense({ editing: null, prefill: req.prefill })
}
// One expense editor for both openers: a booking's Costs block, and a scanned
// receipt sent here from the background tasks widget.
const expenseEditor = expenseEditorFor(bookingExpense, () => setBookingExpense(null), receiptExpense, clearReceiptExpense)
if (isLoading || !splashDone) {
return (
@@ -1174,18 +1179,19 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
}}
/>
)}
{bookingExpense && (
{expenseEditor && (
<ErrorBoundary boundaryId="planner-panel:expense" fallback={null}>
<Suspense fallback={null}>
<ExpenseModal
key={expenseEditor.key}
tripId={tripId}
base={costsBase}
people={tripMembers}
me={meId}
editing={bookingExpense.editing}
prefill={bookingExpense.prefill}
onClose={() => setBookingExpense(null)}
onSaved={() => { setBookingExpense(null); loadBudgetItems(tripId) }}
editing={expenseEditor.editing}
prefill={expenseEditor.prefill}
onClose={expenseEditor.close}
onSaved={() => { expenseEditor.close(); loadBudgetItems(tripId) }}
/>
</Suspense>
</ErrorBoundary>
@@ -17,7 +17,7 @@ import {
healthApi, airtrailApi, mapsApi,
} from '../../api/client'
import { accommodationRepo } from '../../repo/accommodationRepo'
import { offlineDb } from '../../db/offlineDb'
import { offlineDb, saveImportFiles, getImportFiles } from '../../db/offlineDb'
import { getCached, fetchPhoto } from '../../services/photoService'
import type { Accommodation, Place, Reservation, Settings } from '../../types'
@@ -2104,6 +2104,66 @@ describe('useTripPlanner — booking import review', () => {
})
})
describe('useTripPlanner — a receipt scanned from Costs', () => {
const RECEIPT = { merchant: 'Café', date: '2026-09-20', total: 12.5, currency: 'EUR', items: [{ name: 'Tart', price: 12.5 }] }
it('FE-TP-HOOK-160: a read receipt opens the expense editor pre-filled, with the photo to attach, and clears the widget', async () => {
seedTrip()
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
useBackgroundTasksStore.setState({
tasks: [{
id: 'job-r', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
reviewRequested: true, items: [], receipt: RECEIPT, sourceFiles: [photo],
}] as never,
})
const { result } = await renderPlanner()
await waitFor(() => expect(result.current.receiptExpense).not.toBeNull())
expect(result.current.receiptExpense).toEqual({
name: 'Café', amount: 12.5, currency: 'EUR', date: '2026-09-20', lines: [{ name: 'Tart', price: 12.5 }], receiptFiles: [photo],
})
expect(result.current.showReservationModal).toBe(false)
expect(useBackgroundTasksStore.getState().tasks).toHaveLength(0)
act(() => { result.current.clearReceiptExpense() })
expect(result.current.receiptExpense).toBeNull()
})
it('FE-TP-HOOK-162: after a reload the photo comes back from IndexedDB for the review', async () => {
seedTrip()
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
await saveImportFiles('job-db', [photo])
useBackgroundTasksStore.setState({
tasks: [{
id: 'job-db', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
reviewRequested: true, items: [], receipt: RECEIPT,
}] as never,
})
const { result } = await renderPlanner()
await waitFor(() => expect(result.current.receiptExpense).not.toBeNull())
expect(result.current.receiptExpense?.receiptFiles?.map(f => f.name)).toEqual(['bill.jpg'])
await waitFor(async () => expect(await getImportFiles('job-db')).toEqual([]))
})
it('FE-TP-HOOK-161: a scan that read nothing just clears the widget', async () => {
seedTrip()
useBackgroundTasksStore.setState({
tasks: [{
id: 'job-n', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
reviewRequested: true, items: [], receipt: null,
}] as never,
})
const { result } = await renderPlanner()
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks).toHaveLength(0))
expect(result.current.receiptExpense).toBeNull()
})
})
describe('useTripPlanner — misc state', () => {
it('FE-TP-HOOK-095: the map tile url falls back to the default basemap', async () => {
seedTrip()
+20 -2
View File
@@ -22,6 +22,8 @@ import { accommodationRepo } from '../../repo/accommodationRepo'
import { offlineDb, getImportFiles, deleteImportFiles } from '../../db/offlineDb'
import { isEffectivelyOffline } from '../../sync/networkMode'
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
import { receiptToPrefill } from '../../components/Budget/CostsPanel.helpers'
import type { ExpensePrefill } from '../../components/Budget/CostsPanel'
import { useAuthStore } from '../../store/authStore'
import { useResizablePanels } from '../../hooks/useResizablePanels'
import { useTripWebSocket } from '../../hooks/useTripWebSocket'
@@ -435,6 +437,8 @@ export function useTripPlanner() {
const [reservationPrefill, setReservationPrefill] = useState<BookingReviewDraft | null>(null)
const [transportPrefill, setTransportPrefill] = useState<BookingReviewDraft | null>(null)
const [importReviewActive, setImportReviewActive] = useState(false)
// The expense a scanned receipt pre-fills, opened by the page's expense editor.
const [receiptExpense, setReceiptExpense] = useState<ExpensePrefill | null>(null)
const importQueueRef = useRef<BookingImportPreviewItem[]>([])
// The files this import was parsed from, so each reviewed booking can attach its source doc.
const importSourceFilesRef = useRef<File[]>([])
@@ -2727,13 +2731,26 @@ export function useTripPlanner() {
const task = bgTasks.find(
(tk) => tk.tripId === String(tripId) && tk.status === 'done' && tk.reviewRequested && !tk.consumed,
)
if (task && task.items && task.items.length > 0) {
if (task && task.kind === 'costs') {
// A scanned receipt is reviewed in the expense editor, pre-filled with what
// was read and with the photo waiting to be attached when it is saved.
const receipt = task.receipt
const jobId = task.id
const inMemory = task.sourceFiles
dismissBgTask(jobId)
if (!receipt) return
void (async () => {
const files = inMemory && inMemory.length ? inMemory : await getImportFiles(jobId)
deleteImportFiles(jobId)
setReceiptExpense(receiptToPrefill(receipt, files))
})()
} else if (task && task.items && task.items.length > 0) {
// Hand the items (and the source files, to attach to each booking) to the review flow
// and clear the widget entry — once the user hit "review", the background card is done.
const items = task.items
const jobId = task.id
const inMemory = task.sourceFiles
const kind = task.kind ?? 'bookings'
const kind = task.kind === 'transports' ? 'transports' : 'bookings'
dismissBgTask(jobId)
// Prefer the in-memory files (immediate path); after a reload they live in IndexedDB.
void (async () => {
@@ -2866,6 +2883,7 @@ export function useTripPlanner() {
transportModalDayId, setTransportModalDayId,
transportModalAutomated, setTransportModalAutomated, transitPrefill, setTransitPrefill, transitJourney, setTransitJourney,
reservationPrefill, transportPrefill, importReviewActive, startImportReview, advanceImportReview,
receiptExpense, clearReceiptExpense: () => setReceiptExpense(null),
routeShown, setRouteShown, autoShowRoute, transitRoutesShown, routeProfile, setRouteProfile, routeVias, fitKey, setFitKey,
mobileSidebarOpen, setMobileSidebarOpen, mobilePlanScrollTopRef, mobilePlacesScrollTopRef,
deletePlaceId, setDeletePlaceId, deletePlaceIds, setDeletePlaceIds, deletePlaceNote, deletePlacesNote,
+49
View File
@@ -0,0 +1,49 @@
// FE-REPO-LLM-001 to FE-REPO-LLM-004
import { describe, it, expect, beforeEach } from 'vitest'
import { http, HttpResponse } from 'msw'
import { server } from '../../tests/helpers/msw/server'
import { readMultipart } from '../../tests/helpers/multipart'
import { llmRepo } from './llmRepo'
function setOnline(v: boolean): void {
Object.defineProperty(navigator, 'onLine', { value: v, writable: true, configurable: true })
}
beforeEach(() => setOnline(true))
describe('llmRepo', () => {
it('FE-REPO-LLM-001: readsPhotos answers what the server says the model reads', async () => {
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })))
await expect(llmRepo.readsPhotos()).resolves.toBe(true)
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
await expect(llmRepo.readsPhotos()).resolves.toBe(false)
})
it('FE-REPO-LLM-002: readsPhotos is false offline, without asking', async () => {
let asked = false
server.use(http.get('/api/llm/capabilities', () => { asked = true; return HttpResponse.json({ images: true }) }))
setOnline(false)
await expect(llmRepo.readsPhotos()).resolves.toBe(false)
expect(asked).toBe(false)
})
it('FE-REPO-LLM-003: scanReceipt uploads the photo as `file` and answers the job id', async () => {
let sent: string[] = []
server.use(http.post('/api/trips/7/budget/receipt-scan', async ({ request }) => {
sent = (await readMultipart(request)).filenames
return HttpResponse.json({ jobId: 'job-9' })
}))
const photo = new File(['jpeg'], 'bill.jpg', { type: 'image/jpeg' })
await expect(llmRepo.scanReceipt(7, photo)).resolves.toBe('job-9')
// One file part; jsdom names every File part 'blob' on the wire, so the name is not checked.
expect(sent).toHaveLength(1)
})
it('FE-REPO-LLM-004: scanReceipt refuses offline and rejects an answer without a job id', async () => {
setOnline(false)
await expect(llmRepo.scanReceipt(7, new File(['x'], 'a.jpg'))).rejects.toThrow(/connection/)
setOnline(true)
server.use(http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({})))
await expect(llmRepo.scanReceipt(7, new File(['x'], 'a.jpg'))).rejects.toThrow()
})
})
+25
View File
@@ -0,0 +1,25 @@
import { llmCapabilitiesResponseSchema, receiptScanStartResponseSchema } from '@trek/shared'
import { apiClient, postMultipart } from '../api/client'
import { isEffectivelyOffline } from '../sync/networkMode'
/**
* What the AI model can read, and the receipt scan that depends on it. Both are
* online-only: the model runs on the server, so offline a photo is simply not
* offered.
*/
export const llmRepo = {
/** Whether a photo is worth offering to this user: their AI model reads images. */
async readsPhotos(): Promise<boolean> {
if (isEffectivelyOffline()) return false
const reply = await apiClient.get('/llm/capabilities')
return llmCapabilitiesResponseSchema.parse(reply.data).images
},
/** Start reading one receipt photo in the background; answers the import job id. */
async scanReceipt(tripId: number | string, photo: File): Promise<string> {
if (isEffectivelyOffline()) throw new Error('Scanning a receipt requires a connection')
const form = new FormData()
form.append('file', photo)
return receiptScanStartResponseSchema.parse(await postMultipart(`/trips/${tripId}/budget/receipt-scan`, form)).jobId
},
}
@@ -0,0 +1,29 @@
import { describe, it, expect, beforeEach } from 'vitest'
import { useBackgroundTasksStore, taskFoundSomething, type BackgroundImportTask } from './backgroundTasksStore'
const RECEIPT = { merchant: 'Café', date: null, total: 4, currency: 'EUR', items: [] }
beforeEach(() => useBackgroundTasksStore.setState({ tasks: [] }))
describe('backgroundTasksStore with a receipt scan', () => {
it('keeps what the scan read on the finished task', () => {
const s = useBackgroundTasksStore.getState()
s.addTask({ id: 'j1', tripId: '7', label: 'bill.jpg', total: 1, kind: 'costs' })
s.setDone('j1', '7', [], [], RECEIPT)
expect(useBackgroundTasksStore.getState().tasks[0]).toMatchObject({ status: 'done', kind: 'costs', receipt: RECEIPT, items: [] })
})
})
describe('taskFoundSomething', () => {
const task = (over: Partial<BackgroundImportTask>): BackgroundImportTask => ({ id: 'j', tripId: '7', label: 'x', status: 'done', done: 1, total: 1, ...over })
it('asks a receipt scan for its receipt', () => {
expect(taskFoundSomething(task({ kind: 'costs', items: [], receipt: RECEIPT }))).toBe(true)
expect(taskFoundSomething(task({ kind: 'costs', items: [], receipt: null }))).toBe(false)
})
it('asks a booking parse for its items', () => {
expect(taskFoundSomething(task({ kind: 'bookings', items: [{} as never] }))).toBe(true)
expect(taskFoundSomething(task({ items: [] }))).toBe(false)
})
})
+16 -6
View File
@@ -1,6 +1,6 @@
import { create } from 'zustand'
import { persist } from 'zustand/middleware'
import type { BookingImportPreviewItem, BookingImportMode } from '@trek/shared'
import type { BookingImportPreviewItem, BookingImportMode, ReceiptRead } from '@trek/shared'
/**
* Tracks booking-import parses that run in the BACKGROUND (the async endpoint).
@@ -15,6 +15,8 @@ import type { BookingImportPreviewItem, BookingImportMode } from '@trek/shared'
* on mount. We deliberately persist neither the parsed `items` (re-fetched) nor the
* transient review flags (so a reload never auto-reopens the review flow).
*/
export type BackgroundTaskKind = 'transports' | 'bookings' | 'costs'
export interface BackgroundImportTask {
id: string // server job id
tripId: string
@@ -23,6 +25,8 @@ export interface BackgroundImportTask {
done: number
total: number
items?: BookingImportPreviewItem[]
/** A receipt scanned from Costs (`kind: 'costs'`): what the model read, or null when it read nothing. */
receipt?: ReceiptRead | null
warnings?: string[]
error?: string
reviewRequested?: boolean // user clicked "review" — the trip page consumes it
@@ -37,16 +41,17 @@ export interface BackgroundImportTask {
* when its type is guessed (#2076), and unlike the other transient flags it IS
* persisted: the parse deliberately outlives navigation and reload, and the
* review is triggered by the global widget on any page, so a component-local
* value is gone by the time it is needed.
* value is gone by the time it is needed. `costs` is a receipt scan, reviewed
* in the expense editor instead of the booking forms.
*/
kind?: 'transports' | 'bookings'
kind?: BackgroundTaskKind
}
interface BackgroundTasksState {
tasks: BackgroundImportTask[]
addTask: (task: { id: string; tripId: string; label: string; total: number; files?: File[]; mode?: BookingImportMode; kind?: 'transports' | 'bookings' }) => void
addTask: (task: { id: string; tripId: string; label: string; total: number; files?: File[]; mode?: BookingImportMode; kind?: BackgroundTaskKind }) => void
setProgress: (id: string, tripId: string, done: number, total: number) => void
setDone: (id: string, tripId: string, items: BookingImportPreviewItem[], warnings: string[]) => void
setDone: (id: string, tripId: string, items: BookingImportPreviewItem[], warnings: string[], receipt?: ReceiptRead | null) => void
setError: (id: string, tripId: string, error: string) => void
requestReview: (id: string) => void
markConsumed: (id: string) => void
@@ -73,7 +78,7 @@ export const useBackgroundTasksStore = create<BackgroundTasksState>()(
tasks: [],
addTask: ({ id, tripId, label, total, files, mode, kind }) => upsert(id, tripId, { label, total, status: 'running', done: 0, sourceFiles: files, mode, kind }),
setProgress: (id, tripId, done, total) => upsert(id, tripId, { done, total, status: 'running' }),
setDone: (id, tripId, items, warnings) => upsert(id, tripId, { status: 'done', items, warnings, done: items?.length ?? 0 }),
setDone: (id, tripId, items, warnings, receipt) => upsert(id, tripId, { status: 'done', items, warnings, receipt, done: items?.length ?? 0 }),
setError: (id, tripId, error) => upsert(id, tripId, { status: 'error', error }),
requestReview: (id) => set((s) => ({ tasks: s.tasks.map((t) => (t.id === id ? { ...t, reviewRequested: true } : t)) })),
markConsumed: (id) => set((s) => ({ tasks: s.tasks.map((t) => (t.id === id ? { ...t, consumed: true, reviewRequested: false } : t)) })),
@@ -96,3 +101,8 @@ export const useBackgroundTasksStore = create<BackgroundTasksState>()(
},
),
)
/** Whether a finished task has something to review: a read receipt, or at least one booking. */
export function taskFoundSomething(task: BackgroundImportTask): boolean {
return task.kind === 'costs' ? Boolean(task.receipt) : (task.items?.length ?? 0) > 0
}
@@ -1,6 +1,8 @@
import { http, HttpResponse } from 'msw';
export const addonHandlers = [
// Whose AI model reads images: off unless a test says otherwise, so no photo picker shows by default.
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })),
http.get('/api/addons', () => {
return HttpResponse.json({
bagTracking: false,
@@ -649,7 +649,7 @@ describe('MAdminAddonManager', () => {
model: 'gpt-4o',
baseUrl: 'https://api.openai.com/v1',
apiKey: '••••••••',
multimodal: true,
vision: 'auto',
},
});
@@ -657,6 +657,30 @@ describe('MAdminAddonManager', () => {
await screen.findByText('Failed to save');
});
it('FE-MOB-AADD-030: whether the model reads images is a three-way choice, saved as picked', async () => {
const user = userEvent.setup();
const bodies: unknown[] = [];
server.use(
addonsRoute([llmAddon({ provider: 'local', model: 'llava:7b', baseUrl: '', apiKey: '', vision: 'on' })]),
http.get('/api/admin/llm/local/models', () => HttpResponse.json({ models: [] })),
http.put('/api/admin/addons/llm_parsing', async ({ request }) => {
bodies.push(await request.json());
return HttpResponse.json({ success: true });
}),
);
render(<><ToastContainer /><MAdminAddonManager /></>);
const group = await screen.findByRole('radiogroup', { name: 'Model reads images' });
expect(within(group).getByRole('radio', { name: 'Yes' })).toHaveAttribute('aria-checked', 'true');
expect(screen.getByText('Automatic asks the Ollama server whether this model reads images.')).toBeInTheDocument();
await user.click(within(group).getByRole('radio', { name: 'No' }));
expect(within(group).getByRole('radio', { name: 'No' })).toHaveAttribute('aria-checked', 'true');
await user.click(screen.getByRole('button', { name: 'Save' }));
await screen.findByText('Saved');
expect(bodies[0]).toMatchObject({ config: { vision: 'off' } });
});
it('FE-MOB-AADD-024: the API key field can be revealed', async () => {
const user = userEvent.setup();
server.use(addonsRoute([llmAddon({ provider: 'anthropic', apiKey: 'sk-secret' })]));
@@ -66,7 +66,7 @@ describe('MLlmConnectionSection', () => {
expect(screen.getByRole('button', { name: /OpenAI/ })).toBeInTheDocument();
expect(screen.getByDisplayValue('gpt-4o-mini')).toBeInTheDocument();
expect(screen.getByRole('switch', { name: 'Send documents as images' })).toHaveAttribute('aria-checked', 'true');
expect(screen.getByRole('switch', { name: 'Model reads images' })).toHaveAttribute('aria-checked', 'true');
});
it('FE-MOB-SETLLM-003: nothing is hydrated while the settings are still loading', () => {
@@ -158,7 +158,7 @@ describe('MLlmConnectionSection', () => {
const updateSettings = seedLlm();
renderSection();
await user.click(screen.getByRole('switch', { name: 'Send documents as images' }));
await user.click(screen.getByRole('switch', { name: 'Model reads images' }));
await user.click(screen.getByRole('button', { name: /Save/ }));
expect(updateSettings).toHaveBeenCalledWith(expect.objectContaining({ llm_multimodal: true }));
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ExpensePrefill } from '../../../../src/components/Budget/CostsPanel'
import MCostSheet from '../../../../src/mobile/screens/trip/sheets/MCostSheet'
import type { TripMember } from '../../../../src/components/Budget/BudgetPanelMemberChips'
import { clearExchangeRateCache } from '../../../../src/hooks/useExchangeRates'
@@ -35,7 +36,7 @@ interface SheetOverrides {
me?: number
base?: string
editing?: BudgetItem | null
prefill?: { name?: string; category?: string; amount?: number; reservationId?: number; placeId?: number }
prefill?: ExpensePrefill
}
function renderSheet(overrides: SheetOverrides = {}) {
@@ -181,6 +182,14 @@ describe('MCostSheet', () => {
}))
})
it('FE-MOB-COSTSH-003c: a scanned receipt opens with its currency and its photo waiting to be attached', () => {
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
renderSheet({ prefill: { name: 'Sushi Dai', amount: 4200, currency: 'JPY', date: '2026-09-20', receiptFiles: [photo] } })
expect(nameField()).toHaveValue('Sushi Dai')
expect(screen.getByDisplayValue('4200')).toBeInTheDocument()
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
})
it('FE-MOB-COSTSH-003b: a place prefill carries the place link into the payload (#1298)', async () => {
renderSheet({ prefill: { name: 'Louvre', category: 'activities', placeId: 12 } })
expect(nameField()).toHaveValue('Louvre')
+2 -1
View File
@@ -57,6 +57,7 @@ import { StorageModule } from './storage/storage.module';
import { BackupModule } from './backup/backup.module';
import { BookingImportModule } from './booking-import/booking-import.module';
import { ReservationImportModule } from './reservation-import/reservation-import.module';
import { ReceiptScanModule } from './receipt-scan/receipt-scan.module';
import { LlmParseModule } from './llm-parse/llm-parse.module';
import { AuthModule } from './auth/auth.module';
import { OidcModule } from './oidc/oidc.module';
@@ -82,7 +83,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
* migrated.
*/
@Module({
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, ReceiptScanModule, LlmParseModule, ManagedExtModule],
providers: [
// Default-deny: a route is authenticated unless it carries @Public() or
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
@@ -4,6 +4,7 @@ import { PermissionsService } from '../permissions/permissions.service';
import { ReservationsService } from '../reservations/reservations.service';
import { PlacesService } from '../places/places.service';
import { BudgetService } from '../budget/budget.service';
import { imageMimeType } from '../llm-parse/image-input';
import { AddonsService } from '../addons/addons.service';
import { ADDON_IDS } from '../../addons';
import { MapsService } from '../maps/maps.service';
@@ -56,6 +57,11 @@ export class BookingImportService {
return this.llmParse.isAvailable(userId);
}
/** Whether a photo this user imports can be read (see LlmParseService.readsImages). */
readsImages(userId: number): Promise<boolean> {
return this.llmParse.readsImages(userId);
}
/**
* Parse uploaded files and return a preview list. Does NOT persist anything.
* Runs kitinerary first; depending on `mode`, falls back to the LLM:
@@ -155,8 +161,10 @@ export class BookingImportService {
let kiItems: KiReservation[] = [];
let aiUsed = false;
// Stage 1: kitinerary (skipped entirely when forcing AI).
if (mode !== 'force-ai' && kitineraryAvailable) {
// Stage 1: kitinerary (skipped entirely when forcing AI, and for a photo,
// which only a model can read).
const photo = imageMimeType(file.originalname) !== null;
if (mode !== 'force-ai' && kitineraryAvailable && !photo) {
try {
kiItems = await this.extractor.extract(file.buffer, file.originalname);
} catch (err) {
@@ -2,7 +2,8 @@ import { Injectable } from '@nestjs/common';
import { randomUUID } from 'node:crypto';
import { RealtimeService } from '../realtime/realtime.service';
import { BookingImportService } from './booking-import.service';
import type { BookingImportMode, BookingImportPreviewResponse, TrekWsPayload } from '@trek/shared';
import { LlmParseService } from '../llm-parse/llm-parse.service';
import type { BookingImportMode, BookingImportPreviewResponse, ReceiptScanResult, TrekWsPayload } from '@trek/shared';
type JobStatus = 'running' | 'done' | 'error';
@@ -13,7 +14,7 @@ interface ImportJob {
status: JobStatus;
done: number;
total: number;
result?: BookingImportPreviewResponse;
result?: BookingImportPreviewResponse | ReceiptScanResult;
error?: string;
createdAt: number;
}
@@ -28,7 +29,9 @@ const JOB_TTL_MS = 10 * 60_000;
* user's sockets via `broadcastToUser` (which reaches them on ANY page, not just the
* trip room). This is what lets the upload modal close at once and a background widget
* track the work while the user keeps navigating. The actual parsing is the same
* `BookingImportService.preview` the synchronous endpoint uses.
* `BookingImportService.preview` the synchronous endpoint uses. A receipt scanned
* from the Costs tab rides the same queue, with `LlmParseService.readReceipt` as
* its work: it is the same slow model on the same server.
*/
@Injectable()
export class ImportJobsService {
@@ -36,17 +39,44 @@ export class ImportJobsService {
/** Tail of each user's job chain — parses run one at a time per user, not all at once. */
private readonly chains = new Map<number, Promise<void>>();
constructor(private readonly bookingImport: BookingImportService, private readonly realtime: RealtimeService) {}
constructor(
private readonly bookingImport: BookingImportService,
private readonly realtime: RealtimeService,
private readonly llmParse: LlmParseService,
) {}
/** Create a job and queue it behind the user's other parses; returns the job id at once. */
start(tripId: string, files: Express.Multer.File[], mode: BookingImportMode, userId: number): string {
return this.enqueue(tripId, userId, files.length, (job) =>
this.bookingImport.preview(files, mode, job.userId, (done, total, fileName) => {
job.done = done;
this.push(job, 'import:progress', { status: 'running', done, total, fileName });
}),
);
}
/** Queue the read of one receipt photo, behind the same user's other parses. */
startReceipt(tripId: string, file: Express.Multer.File, userId: number): string {
return this.enqueue(tripId, userId, 1, async (job) => {
const result = await this.llmParse.readReceipt({ buffer: file.buffer, originalName: file.originalname }, job.userId);
job.done = 1;
return result;
});
}
private enqueue(
tripId: string,
userId: number,
total: number,
work: (job: ImportJob) => Promise<BookingImportPreviewResponse | ReceiptScanResult>,
): string {
const id = randomUUID();
const job: ImportJob = { id, tripId, userId, status: 'running', done: 0, total: files.length, createdAt: Date.now() };
const job: ImportJob = { id, tripId, userId, status: 'running', done: 0, total, createdAt: Date.now() };
this.jobs.set(id, job);
// Chain onto the user's previous parse so they run sequentially (one CPU-heavy
// inference at a time), while the request returns immediately.
const prev = this.chains.get(userId) ?? Promise.resolve();
const next = prev.then(() => this.run(job, files, mode)).catch(() => {});
const next = prev.then(() => this.run(job, work)).catch(() => {});
this.chains.set(userId, next);
void next.finally(() => {
if (this.chains.get(userId) === next) this.chains.delete(userId);
@@ -59,13 +89,10 @@ export class ImportJobsService {
return job && job.userId === userId ? job : undefined;
}
private async run(job: ImportJob, files: Express.Multer.File[], mode: BookingImportMode): Promise<void> {
private async run(job: ImportJob, work: (job: ImportJob) => Promise<BookingImportPreviewResponse | ReceiptScanResult>): Promise<void> {
this.push(job, 'import:progress', { status: 'running', done: 0, total: job.total });
try {
const result = await this.bookingImport.preview(files, mode, job.userId, (done, total, fileName) => {
job.done = done;
this.push(job, 'import:progress', { status: 'running', done, total, fileName });
});
const result = await work(job);
job.status = 'done';
job.result = result;
this.push(job, 'import:done', { result });
@@ -6,13 +6,14 @@ import { UnreadableLlmResponse } from './openai-compatible.client';
const MAX_TOKENS = 8192;
const ANTHROPIC_VERSION = '2023-06-01';
const TOOL_NAME = 'emit_reservations';
/**
* Anthropic Messages API client. Structured output via forced tool-use: a single
* `emit_reservations` tool whose `input_schema` is the reservations schema, with
* `tool_choice` forcing it — the documented, reliable way to get structured JSON.
* PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs).
* `emit_<rootKey>` tool (`emit_reservations` by default) whose `input_schema` is
* the caller's schema, with `tool_choice` forcing it — the documented, reliable
* way to get structured JSON.
* PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs), photos
* as `image` blocks.
* Raw fetch (no SDK) to match the codebase's HTTP style.
*/
export class AnthropicClient implements LlmExtractionClient {
@@ -24,16 +25,21 @@ export class AnthropicClient implements LlmExtractionClient {
const base = (input.baseUrl ?? 'https://api.anthropic.com').replace(/(?<!\/)\/+$/, '');
const url = `${base}/v1/messages`;
const rootKey = input.rootKey ?? 'reservations';
const toolName = `emit_${rootKey}`;
const userText = input.userText ?? USER_TEXT;
const content: unknown[] = [];
if (input.file) {
for (const file of input.file ? [input.file, ...(input.pageImages ?? [])] : []) {
content.push({
type: 'document',
source: { type: 'base64', media_type: input.file.mimeType, data: input.file.data.toString('base64') },
// A photo is an image block; the document block only takes a PDF.
type: file.mimeType.startsWith('image/') ? 'image' : 'document',
source: { type: 'base64', media_type: file.mimeType, data: file.data.toString('base64') },
});
}
content.push({
type: 'text',
text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT,
text: input.text ? `${userText}\n\n${input.text}` : userText,
});
const body = {
@@ -42,12 +48,12 @@ export class AnthropicClient implements LlmExtractionClient {
system: input.prompt,
tools: [
{
name: TOOL_NAME,
description: 'Return the travel reservations extracted from the document.',
name: toolName,
description: `Return the ${rootKey} extracted from the document.`,
input_schema: input.jsonSchema,
},
],
tool_choice: { type: 'tool', name: TOOL_NAME },
tool_choice: { type: 'tool', name: toolName },
messages: [{ role: 'user', content }],
};
@@ -78,7 +84,7 @@ export class AnthropicClient implements LlmExtractionClient {
const data = (await res.json()) as {
stop_reason?: string;
content?: { type: string; name?: string; input?: { reservations?: unknown } }[];
content?: { type: string; name?: string; input?: Record<string, unknown> }[];
};
if (data.stop_reason === 'refusal') {
@@ -95,9 +101,9 @@ export class AnthropicClient implements LlmExtractionClient {
);
}
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === TOOL_NAME);
if (!toolUse) throw new UnreadableLlmResponse(`the model answered without calling ${TOOL_NAME}`);
return toReservationList(toolUse.input?.reservations);
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === toolName);
if (!toolUse) throw new UnreadableLlmResponse(`the model answered without calling ${toolName}`);
return toReservationList(toolUse.input?.[rootKey], rootKey);
}
}
@@ -110,7 +110,7 @@ function clean(value: unknown): unknown {
* treating the right-most separator as the decimal point. Returns null when there
* is no parseable amount.
*/
function parseAmount(raw: unknown): number | null {
export function parseAmount(raw: unknown): number | null {
if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null;
if (typeof raw !== 'string') return null;
let s = raw.replace(/[^\d.,]/g, '');
@@ -84,18 +84,21 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
const base = (input.baseUrl ?? 'https://api.openai.com/v1').replace(/(?<!\/)\/+$/, '');
const url = `${base}/chat/completions`;
const nuextract = isNuExtractModel(input.model);
const rootKey = input.rootKey ?? 'reservations';
const userText = input.userText ?? USER_TEXT;
const userContent: unknown[] = nuextract
? [{ type: 'text', text: buildNuExtractUserText(input.text ?? '') }]
: [{ type: 'text', text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT }];
: [{ type: 'text', text: input.text ? `${userText}\n\n${input.text}` : userText }];
// Only genuine images go natively (as image_url) — OpenAI-compatible servers
// (notably Ollama) reject `file`/PDF content parts. PDFs reach this client as
// pre-extracted text (see llm-parse.service.ts), never as bytes.
if (!nuextract && input.file && input.file.mimeType.startsWith('image/')) {
const b64 = input.file.data.toString('base64');
// pre-extracted text (see llm-parse.service.ts), or a scan as its pages drawn
// as images, never as PDF bytes.
for (const file of !nuextract && input.file ? [input.file, ...(input.pageImages ?? [])] : []) {
if (!file.mimeType.startsWith('image/')) continue;
userContent.push({
type: 'image_url',
image_url: { url: `data:${input.file.mimeType};base64,${b64}` },
image_url: { url: `data:${file.mimeType};base64,${file.data.toString('base64')}` },
});
}
@@ -130,7 +133,7 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
...baseBody,
response_format: shape.jsonObject
? { type: 'json_object' as const }
: { type: 'json_schema' as const, json_schema: { name: 'reservations', schema: input.jsonSchema, strict: false } },
: { type: 'json_schema' as const, json_schema: { name: rootKey, schema: input.jsonSchema, strict: false } },
};
};
@@ -185,7 +188,7 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
choices?: { message?: { content?: string } }[];
};
const content = data.choices?.[0]?.message?.content;
return nuextract ? parseNuExtract(content) : parseReservations(content);
return nuextract ? parseNuExtract(content) : parseReservations(content, rootKey);
}
private async send(url: string, body: unknown, apiKey?: string): Promise<Response> {
@@ -216,9 +219,9 @@ function parseNuExtract(content: string | undefined | null): Record<string, unkn
const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
/** Tolerant parse: strip code fences, JSON(5).parse, pull `reservations`. */
function parseReservations(content: string | undefined | null): Record<string, unknown>[] {
return toReservationList(readAnswer(content));
/** Tolerant parse: strip code fences, JSON(5).parse, pull the `rootKey` array. */
function parseReservations(content: string | undefined | null, rootKey: string): Record<string, unknown>[] {
return toReservationList(readAnswer(content), rootKey);
}
/**
+67
View File
@@ -0,0 +1,67 @@
import { extname } from 'node:path';
import { Jimp } from 'jimp';
import { PDFParse } from 'pdf-parse';
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared';
/** A photographed document on its way to a vision model, by the extensions shared/ lists. */
const IMAGE_MIME_BY_EXT: Record<(typeof LLM_PHOTO_EXTENSIONS)[number], string> = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.webp': 'image/webp',
};
export const IMAGE_EXTENSIONS: readonly string[] = LLM_PHOTO_EXTENSIONS;
/** The image MIME type for a file name, or null when it is not a photo TREK sends. */
export function imageMimeType(fileName: string): string | null {
return (IMAGE_MIME_BY_EXT as Record<string, string>)[extname(fileName).toLowerCase()] ?? null;
}
/**
* Long edge in pixels. A phone photo is around twelve megapixels, which a model
* pays for in prompt tokens without reading the print any better: the same
* receipt on a local qwen3.5:4b took 252s whole and 37-124s at this size.
*/
export const IMAGE_MAX_EDGE = 1600;
const IMAGE_JPEG_QUALITY = 82;
/**
* The image, no larger than the model needs. Returns the bytes unchanged when
* they are already small enough, or when they cannot be decoded (Jimp has no
* WebP): a photo that might be read beats one refused over a failed resize.
*/
export async function capImage(data: Buffer, mimeType: string): Promise<{ data: Buffer; mimeType: string }> {
try {
const image = await Jimp.read(data);
if (Math.max(image.width, image.height) <= IMAGE_MAX_EDGE) return { data, mimeType };
if (image.width >= image.height) image.resize({ w: IMAGE_MAX_EDGE });
else image.resize({ h: IMAGE_MAX_EDGE });
const resized = await image.getBuffer('image/jpeg', { quality: IMAGE_JPEG_QUALITY });
return { data: Buffer.from(resized), mimeType: 'image/jpeg' };
} catch {
return { data, mimeType };
}
}
/**
* Pages of a PDF read the only way a scan can be: as pictures. The booking
* details sit on the first pages, and every page costs the model a photo's
* worth of prompt, so only the first two are drawn.
*/
export const PDF_PAGES_AS_IMAGES = 2;
/** The first pages of a PDF drawn as images, capped like a photo; empty when none could be drawn. */
export async function renderPdfPages(data: Buffer): Promise<{ data: Buffer; mimeType: string }[]> {
const parser = new PDFParse({ data: new Uint8Array(data) });
try {
const shot = await parser.getScreenshot({ first: PDF_PAGES_AS_IMAGES, desiredWidth: IMAGE_MAX_EDGE, imageBuffer: true, imageDataUrl: false });
const pages: { data: Buffer; mimeType: string }[] = [];
for (const page of shot.pages) {
if (page.data?.length) pages.push(await capImage(Buffer.from(page.data), 'image/png'));
}
return pages;
} finally {
await parser.destroy().catch(() => {});
}
}
+6 -3
View File
@@ -60,13 +60,16 @@ export function parseLenientJson(content: string | undefined | null): unknown {
* would have been accepted inside an array is accepted alone rather than thrown
* away without a word (#2375). It is still not guessing: an object without that
* key stays refused.
*
* `rootKey` names the wrapper for a caller that asks for another list than
* reservations (the receipt read asks for `receipts`).
*/
export function toReservationList(value: unknown): Record<string, unknown>[] {
export function toReservationList(value: unknown, rootKey = 'reservations'): Record<string, unknown>[] {
const list = (v: unknown): Record<string, unknown>[] | null => {
if (Array.isArray(v)) return v as Record<string, unknown>[];
if (v && typeof v === 'object') {
const o = v as { reservations?: unknown; '@type'?: unknown };
if (Array.isArray(o.reservations)) return o.reservations as Record<string, unknown>[];
const o = v as Record<string, unknown>;
if (Array.isArray(o[rootKey])) return o[rootKey] as Record<string, unknown>[];
if (typeof o['@type'] === 'string') return [v as Record<string, unknown>];
}
return null;
@@ -0,0 +1,22 @@
import { Controller, Get, UseGuards } from '@nestjs/common';
import type { LlmCapabilitiesResponse } from '@trek/shared';
import type { User } from '../../types';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { LlmParseService } from './llm-parse.service';
/**
* What the caller's AI model can be handed. Per user, unlike the public
* /api/health/features: whose model applies depends on who is asking. The
* photo pickers read it to decide whether a photo is offered at all.
*/
@Controller('api/llm')
@UseGuards(JwtAuthGuard)
export class LlmCapabilitiesController {
constructor(private readonly llmParse: LlmParseService) {}
@Get('capabilities')
async capabilities(@CurrentUser() user: User): Promise<LlmCapabilitiesResponse> {
return { images: await this.llmParse.readsImages(user.id) };
}
}
@@ -1,5 +1,6 @@
import { ADDON_IDS } from '../../addons';
import { AddonsService } from '../addons/addons.service';
import { asLlmVision } from '@trek/shared';
import { decryptLlmApiKey, LLM_PROVIDERS, type LlmProvider, type ResolvedLlmConfig } from './llm-config';
import { DatabaseService } from '../database/database.service';
import { SettingsService } from '../settings/settings.service';
@@ -53,7 +54,7 @@ export class LlmConfigResolver {
model,
baseUrl: typeof cfg.baseUrl === 'string' && cfg.baseUrl.trim() ? cfg.baseUrl.trim() : undefined,
apiKey: decryptLlmApiKey(cfg.apiKey),
multimodal: cfg.multimodal === true,
vision: asLlmVision(cfg.vision),
};
}
@@ -88,7 +89,9 @@ export class LlmConfigResolver {
model,
baseUrl,
apiKey,
multimodal: settings.llm_multimodal === true,
// The personal switch has only ever been on or off, and was saved as the
// person set it, so it is read as exactly that.
vision: settings.llm_multimodal === true ? 'on' : 'off',
};
}
}
+8 -2
View File
@@ -1,3 +1,4 @@
import { asLlmVision, type LlmVision } from '@trek/shared';
import { maybe_encrypt_api_key, decrypt_api_key } from '../common/crypto/apiKeyCrypto';
/**
@@ -21,7 +22,7 @@ export interface ResolvedLlmConfig {
model: string;
baseUrl?: string;
apiKey?: string;
multimodal: boolean;
vision: LlmVision;
}
/** Shape of the admin instance config stored in `addons.config` (apiKey encrypted). */
@@ -30,7 +31,11 @@ export interface LlmAddonConfig {
model?: string;
baseUrl?: string;
apiKey?: string;
multimodal?: boolean;
/**
* Replaces `multimodal`, which the admin screens wrote as `false` on every
* save whatever the model: a stored `false` means nothing, so it is not read.
*/
vision?: LlmVision;
}
export const LLM_PROVIDERS: LlmProvider[] = ['local', 'openai', 'anthropic'];
@@ -46,6 +51,7 @@ export function prepareLlmAddonConfigForWrite(
existingStored: Record<string, unknown> | undefined,
): Record<string, unknown> {
const out: Record<string, unknown> = { ...incoming };
if ('vision' in incoming) out.vision = asLlmVision(incoming.vision);
const key = incoming.apiKey;
if (key === undefined || key === null || key === '' || key === MASKED_VALUE) {
// Keep the existing encrypted key untouched (mask echoed or no key supplied).
@@ -1,5 +1,17 @@
import { Injectable, HttpException } from '@nestjs/common';
import { z } from 'zod';
import { safeFetchLlm } from '../../utils/ssrfGuard';
import { readCappedJson } from '../../utils/cappedFetch';
/** `/api/show` carries the licence and the template: tens of KB, never megabytes. */
const MAX_SHOW_BYTES = 1024 * 1024;
const SHOW_TIMEOUT_MS = 5_000;
/** A model's capabilities only change when it is re-pulled; ten minutes of staleness is harmless. */
const CAPABILITIES_TTL_MS = 10 * 60_000;
/** An unreachable server is asked again soon, so a restart does not cost ten minutes. */
const CAPABILITIES_MISS_TTL_MS = 60_000;
const showResponseSchema = z.object({ capabilities: z.array(z.string()).optional() });
/**
* Admin helpers for managing a local OpenAI-compatible LLM server (Ollama).
@@ -11,6 +23,8 @@ import { safeFetchLlm } from '../../utils/ssrfGuard';
*/
@Injectable()
export class LlmLocalService {
private readonly capabilityCache = new Map<string, { capabilities: string[] | null; expires: number }>();
/** Derive the Ollama root from a configured base URL (strip a trailing /v1). */
ollamaRoot(baseUrl: string | undefined): string {
const raw = (baseUrl ?? 'http://localhost:11434').trim();
@@ -65,4 +79,42 @@ export class LlmLocalService {
if (!res.ok || !res.body) throw new HttpException({ error: `Pull failed (${res.status})` }, 502);
return res.body;
}
/**
* What the local server says the model can do (`['completion', 'vision', …]`),
* or null when it cannot say: a bad URL, a server that is down, a model it
* does not have, an answer that is not what Ollama sends. Never throws, since
* every caller treats "unknown" as "no", and remembers the answer per server
* and model so a page load does not become a request to the model server.
*/
async modelCapabilities(baseUrl: string | undefined, model: string): Promise<string[] | null> {
const now = Date.now();
const key = `${baseUrl ?? ''}\n${model}`;
const cached = this.capabilityCache.get(key);
if (cached && cached.expires > now) return cached.capabilities;
let capabilities: string[] | null = null;
try {
const root = this.ollamaRoot(baseUrl);
const res = await safeFetchLlm(`${root}/api/show`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ model }),
signal: AbortSignal.timeout(SHOW_TIMEOUT_MS),
});
if (res.ok) {
const parsed = showResponseSchema.safeParse(await readCappedJson<unknown>(res, MAX_SHOW_BYTES));
if (parsed.success) capabilities = parsed.data.capabilities ?? [];
} else {
void res.body?.cancel().catch(() => {});
}
} catch {
capabilities = null;
}
this.capabilityCache.set(key, {
capabilities,
expires: now + (capabilities ? CAPABILITIES_TTL_MS : CAPABILITIES_MISS_TTL_MS),
});
return capabilities;
}
}
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
import { LlmParseService } from './llm-parse.service';
import { LlmLocalService } from './llm-local.service';
import { LlmLocalController } from './llm-local.controller';
import { LlmCapabilitiesController } from './llm-capabilities.controller';
import { LlmConfigResolver } from './llm-config.resolver';
import { SettingsModule } from '../settings/settings.module';
import { AddonsModule } from '../addons/addons.module';
@@ -13,7 +14,7 @@ import { AppConfigModule } from '../app-config/app-config.module';
*/
@Module({
imports: [AppConfigModule, SettingsModule, AddonsModule],
controllers: [LlmLocalController],
controllers: [LlmLocalController, LlmCapabilitiesController],
providers: [LlmParseService, LlmLocalService, LlmConfigResolver],
exports: [LlmParseService, LlmConfigResolver],
})
+121 -14
View File
@@ -1,13 +1,25 @@
import type { KiReservation } from '../booking-import/kitinerary.types';
import { createLlmClient } from './llm-client.factory';
import { LlmConfigResolver } from './llm-config.resolver';
import type { ResolvedLlmConfig } from './llm-config';
import { LlmLocalService } from './llm-local.service';
import { capImage, imageMimeType, renderPdfPages } from './image-input';
import { extractEnforced } from './router/ollama-format.client';
import {
buildReceiptPrompt,
RECEIPT_JSON_SCHEMA,
RECEIPT_LIST_JSON_SCHEMA,
RECEIPT_ROOT_KEY,
RECEIPT_USER_TEXT,
toReceiptRead,
} from './receipt-read';
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt';
import type { LlmExtractionInput } from './llm-provider.interface';
import { isPdf, extractText } from './text-extract';
import { routeExtraction, detectFlightNumbers, extractTotalPrice } from './router/extraction-router';
import { routeExtraction, routeImageExtraction, detectFlightNumbers, extractTotalPrice } from './router/extraction-router';
import { toIsoCurrency } from './currency-code';
import { Injectable } from '@nestjs/common';
import { kiReservationSchema } from '@trek/shared';
import { kiReservationSchema, type ReceiptScanResult } from '@trek/shared';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
const MIME_BY_EXT: Record<string, string> = {
@@ -21,7 +33,8 @@ export interface LlmParseResult {
/**
* Orchestrates the LLM fallback: resolve config → pick client → build input
* (native bytes vs extracted text by the `multimodal` flag) → call provider →
* (a photo as an image when the model reads one, a PDF natively for Anthropic,
* extracted text otherwise) → call provider →
* validate the response → return schema.org `KiReservation[]` for the shared
* mapper. Never throws for content/provider reasons — degrades to `[]` + a
* warning, mirroring the kitinerary extractor's tolerance.
@@ -31,6 +44,7 @@ export class LlmParseService {
constructor(
private readonly llmConfig: LlmConfigResolver,
private readonly env: RuntimeEnvService,
private readonly local: LlmLocalService,
) {}
/** True when the addon is enabled AND a usable config resolves for this user. */
@@ -38,6 +52,77 @@ export class LlmParseService {
return this.llmConfig.resolve(userId) !== null;
}
/** Whether a photo this user uploads can be read: AI parsing is set up and its model takes images. */
async readsImages(userId: number): Promise<boolean> {
const config = this.llmConfig.resolve(userId);
return config ? this.visionFor(config) : false;
}
/**
* Read a photographed receipt for the Costs tab. Like parse(), it never throws
* for content or provider reasons: a refusal comes back as a null receipt and
* a warning naming the file.
*/
async readReceipt(file: { buffer: Buffer; originalName: string }, userId: number): Promise<ReceiptScanResult> {
const config = this.llmConfig.resolve(userId);
if (!config) return { receipt: null, warnings: ['AI parsing is not configured'] };
const imageType = imageMimeType(file.originalName);
if (!imageType) return { receipt: null, warnings: [`${file.originalName}: not a photo`] };
if (!(await this.visionFor(config))) {
return { receipt: null, warnings: [`${file.originalName}: the configured model does not read images`] };
}
const image = await capImage(file.buffer, imageType);
const prompt = buildReceiptPrompt();
let raw: unknown;
try {
if (config.provider === 'local') {
// Ollama's own chat API, as the booking router uses: `think: false` and
// `num_ctx` exist there and not on /v1, and a hybrid model that is left
// to reason spends the whole token budget on it and answers nothing.
raw = await extractEnforced({
baseUrl: config.baseUrl ?? 'http://localhost:11434/v1',
model: config.model,
apiKey: config.apiKey,
system: prompt,
user: RECEIPT_USER_TEXT,
schema: RECEIPT_JSON_SCHEMA,
images: [image.data.toString('base64')],
numCtx: 16384,
// A supermarket receipt is a long list of lines.
numPredict: 2048,
});
} else {
const list = await createLlmClient(config).extract({
prompt,
jsonSchema: RECEIPT_LIST_JSON_SCHEMA,
rootKey: RECEIPT_ROOT_KEY,
userText: RECEIPT_USER_TEXT,
model: config.model,
baseUrl: config.baseUrl,
apiKey: config.apiKey,
file: image,
});
raw = list[0];
}
} catch (err) {
console.error(`[llm-parse] Receipt read failed for "${file.originalName}" (provider=${config.provider}):`, err instanceof Error ? err.message : err);
return { receipt: null, warnings: [`${file.originalName}: AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
}
const receipt = toReceiptRead(raw);
return receipt ? { receipt, warnings: [] } : { receipt: null, warnings: [`${file.originalName}: no receipt could be read`] };
}
private async visionFor(config: ResolvedLlmConfig): Promise<boolean> {
if (config.vision !== 'auto') return config.vision === 'on';
// Only a local Ollama can be asked. A cloud model is not assumed to read
// images: an admin who knows it does says so.
if (config.provider !== 'local') return false;
const capabilities = await this.local.modelCapabilities(config.baseUrl, config.model);
return capabilities?.includes('vision') ?? false;
}
async parse(file: { buffer: Buffer; originalName: string }, userId: number): Promise<LlmParseResult> {
const config = this.llmConfig.resolve(userId);
if (!config) return { kiItems: [], warnings: ['AI parsing is not configured'] };
@@ -51,11 +136,22 @@ export class LlmParseService {
apiKey: config.apiKey,
};
// Native PDF only for Anthropic (its document block reads text AND scans).
// OpenAI-compatible servers (incl. Ollama/NuExtract) can't ingest PDFs/`file`
// parts, so every other provider gets extracted text.
const imageType = imageMimeType(file.originalName);
if (imageType && !(await this.visionFor(config))) {
return { kiItems: [], warnings: [`${file.originalName}: the configured model does not read images`] };
}
// A photo goes as an image to every provider. Native PDF only for Anthropic
// (its document block reads text AND scans). OpenAI-compatible servers
// (incl. Ollama/NuExtract) can't ingest PDFs/`file` parts, so every other
// provider gets extracted text.
try {
if (config.provider === 'anthropic' && isPdf(file.originalName)) {
if (imageType) {
input.file = await capImage(file.buffer, imageType);
console.debug(
`[DEBUG] Photo (${input.file.data.length} bytes) sent to ${config.provider}: ${file.originalName}`,
);
} else if (config.provider === 'anthropic' && isPdf(file.originalName)) {
input.file = { mimeType: MIME_BY_EXT['.pdf'], data: file.buffer };
console.debug(
`[DEBUG] Extracted (native PDF, ${file.buffer.length} bytes) sent to ${config.provider}: ${file.originalName}`,
@@ -78,10 +174,18 @@ export class LlmParseService {
console.debug(`[DEBUG] Extracted text from ${file.originalName} (${input.text.length} chars):\n`, input.text);
}
if (!input.text.trim()) {
return {
kiItems: [],
warnings: [`${file.originalName}: no readable text found (a scanned PDF needs a cloud/vision provider)`],
};
// A scan has no text layer, but a model that reads images can read its pages.
const pages = isPdf(file.originalName) && (await this.visionFor(config)) ? await renderPdfPages(file.buffer) : [];
if (pages.length === 0) {
return {
kiItems: [],
warnings: [`${file.originalName}: no readable text found (a scanned PDF needs a model that reads images)`],
};
}
input.text = undefined;
input.file = pages[0];
input.pageImages = pages.slice(1);
console.debug(`[DEBUG] Scanned PDF sent to ${config.provider} as ${pages.length} page image(s): ${file.originalName}`);
}
}
} catch (err) {
@@ -96,13 +200,16 @@ export class LlmParseService {
// templates → decompose + grammar-enforced per-reservation extraction → validate
// + repair. Far more reliable on small CPU models than the single-shot path below
// (which stays for cloud providers, whose strong models handle one-shot well).
if (config.provider === 'local' && input.text) {
if (config.provider === 'local' && (input.text || input.file)) {
try {
const routed = await routeExtraction(input.text, {
const ctx = {
baseUrl: config.baseUrl ?? 'http://localhost:11434/v1',
model: config.model,
apiKey: config.apiKey,
});
};
const routed = input.file
? await routeImageExtraction([input.file, ...(input.pageImages ?? [])].map((f) => f.data), ctx)
: await routeExtraction(input.text ?? '', ctx);
return { kiItems: routed.kiItems, warnings: [...warnings, ...routed.warnings] };
} catch (err) {
console.error(`[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`, err instanceof Error ? err.message : err);
@@ -1,4 +1,4 @@
/** A single binary file (e.g. a PDF) sent natively to a multimodal provider. */
/** A single binary file (a PDF, or a photo) sent natively to a provider that reads it. */
export interface LlmExtractionFile {
mimeType: string;
data: Buffer;
@@ -15,8 +15,17 @@ export interface LlmExtractionInput {
apiKey?: string;
/** Pre-extracted text (text-like files, or text-only-model mode). */
text?: string;
/** Native binary (PDF) for multimodal providers. */
/** Native binary: a PDF for Anthropic, a photo for a model that reads images. */
file?: LlmExtractionFile;
/** The following pages of a scanned PDF, as images after `file`, which holds the first. */
pageImages?: LlmExtractionFile[];
/**
* The key the answer's array sits under, `jsonSchema` being `{ [rootKey]: [...] }`.
* Defaults to `reservations`; the receipt read asks for `receipts`.
*/
rootKey?: string;
/** The user-turn instruction. Defaults to the booking import's. */
userText?: string;
}
/**
+108
View File
@@ -0,0 +1,108 @@
import { receiptReadSchema, type ReceiptLine, type ReceiptRead } from '@trek/shared';
import { toIsoCurrency } from './currency-code';
import { parseAmount } from './clients/nuextract';
/**
* Reading a photographed receipt for the Costs tab: the prompt, the schema the
* answer is held to, and the step that turns the answer into a `ReceiptRead`.
* Pure, so every piece is testable without a model.
*
* The field list is flat and short on purpose. A till roll is printed small and
* photographed under bad light, and a small local model fills a handful of named
* fields far more reliably than the nested schema.org shape bookings use.
*/
/** Key the cloud clients read the answer's array under (`{ receipts: [...] }`). */
export const RECEIPT_ROOT_KEY = 'receipts';
export const RECEIPT_USER_TEXT = 'Read the receipt in the attached photo.';
/** One receipt, flat. Ollama compiles it to a grammar, so the answer has this shape or none. */
export const RECEIPT_JSON_SCHEMA = {
type: 'object',
properties: {
merchant: { type: 'string' },
date: { type: 'string' },
total: { type: 'number' },
currency: { type: 'string' },
items: {
type: 'array',
items: {
type: 'object',
properties: { name: { type: 'string' }, price: { type: 'number' } },
required: ['name', 'price'],
},
},
},
// Every field is required. Ollama's grammar lets a small model skip an
// optional key, and it does: qwen3.5:4b answered the total and the lines of a
// receipt printing both its name and its date, and nothing else. A field the
// receipt does not show comes back empty, which toReceiptRead reads as absent.
required: ['merchant', 'date', 'total', 'currency', 'items'],
} as const;
/** The same receipt wrapped in the list the cloud clients' structured output returns. */
export const RECEIPT_LIST_JSON_SCHEMA = {
type: 'object',
properties: { [RECEIPT_ROOT_KEY]: { type: 'array', items: RECEIPT_JSON_SCHEMA } },
required: [RECEIPT_ROOT_KEY],
} as const;
export function buildReceiptPrompt(today: Date = new Date()): string {
const todayIso = today.toISOString().slice(0, 10);
return [
'You read a photographed receipt, bill or invoice: often a paper till roll, possibly crumpled, skewed or in a foreign language.',
'Fill "total" with the grand total actually paid, after tax and tip: the final TOTAL or AMOUNT DUE line, not a subtotal.',
'Fill "currency" with its ISO 4217 code, from the symbol and the country when the code is not printed.',
'Fill "merchant" with the business name printed at the top, and "date" as YYYY-MM-DD.',
// A till roll prints "26-08-23" and means one of three dates. The merchant's
// country settles day/month order; nothing on the paper settles which pair is
// the year, and a receipt is scanned soon after it is paid.
`Today is ${todayIso}. When a date could be read several ways, take the reading nearest to today and never one in the future.`,
'Fill "items" with the purchased lines as printed, each with its line total as "price". Leave out subtotal, tax, discount, tip and total lines.',
'Never invent a value: when the receipt does not show a field, answer an empty string for it (0 for "total", [] for "items"). Numbers are plain numbers: 12.5, not "12,50 €".',
].join('\n');
}
function toText(value: unknown): string | null {
return typeof value === 'string' && value.trim() ? value.trim() : null;
}
/** A real calendar date as YYYY-MM-DD, or null. */
function toDate(value: unknown): string | null {
const s = toText(value);
if (!s || !/^\d{4}-\d{2}-\d{2}$/.test(s)) return null;
const d = new Date(`${s}T00:00:00Z`);
return !Number.isNaN(d.getTime()) && d.toISOString().slice(0, 10) === s ? s : null;
}
/**
* The model's answer as a `ReceiptRead`, or null when it holds nothing a person
* could use: no total, no merchant and no line. Each field is kept only when it
* reads as what it claims to be, so a garbled one is left for the person to fill
* rather than saved wrong.
*/
export function toReceiptRead(raw: unknown): ReceiptRead | null {
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
const node = raw as Record<string, unknown>;
const total = parseAmount(node.total);
const items: ReceiptLine[] = [];
if (Array.isArray(node.items)) {
for (const line of node.items) {
if (!line || typeof line !== 'object') continue;
const name = toText((line as Record<string, unknown>).name);
const price = parseAmount((line as Record<string, unknown>).price);
if (name && price !== null) items.push({ name, price });
}
}
const read = {
merchant: toText(node.merchant),
date: toDate(node.date),
total: total !== null && total > 0 ? total : null,
currency: toIsoCurrency(node.currency) ?? null,
items,
};
if (read.total === null && read.merchant === null && items.length === 0) return null;
const parsed = receiptReadSchema.safeParse(read);
return parsed.success ? parsed.data : null;
}
@@ -23,6 +23,7 @@ import type { KiReservation } from '../../booking-import/kitinerary.types';
import { nuExtractToKiReservations } from '../clients/nuextract';
import { FLAT_SCHEMA_BY_TYPE, FLAT_TYPES, FLIGHTS_ARRAY_SCHEMA, UNION_SINGLE_SCHEMA, type FlatType, type FlatLike } from './flat-schemas';
import { extractEnforced } from './ollama-format.client';
import { parseLenientJson } from '../lenient-json';
export interface RouterContext {
baseUrl: string;
@@ -45,6 +46,24 @@ const TYPE_HINT: Record<FlatType, string> = {
event: 'event/attraction. name = the event/ticket, address = the venue, start_time/end_time = full ISO, price/currency = ticket price.',
};
/**
* The union prompt for a photo. Text has keywords that pick a type-specific
* schema and hint before the call; a photo has none, so the model gets every
* type's field meanings at once. Without them qwen3.5:4b read a train ticket
* as bare clock times with no stations, and its price as a JSON object inside
* the string.
*/
const PHOTO_UNION_HINT = [
'Pick the correct "type", then fill its fields:',
'- flight: vehicle_number = flight number, from_code/to_code = IATA codes.',
'- train, bus, ferry: from_name/to_name = stations, stops or ports, vehicle_number = train or line number.',
'- car: operator = rental company, from_name = pick-up location, to_name = return location.',
'- hotel: name, address, checkin_time/checkout_time.',
'- restaurant, event: name, address = the venue, start_time (and end_time).',
"Every time is full ISO 'YYYY-MM-DDTHH:MM:00' (a transport uses departure_time/arrival_time).",
'price = the total paid as a plain number like 49.00, currency = its ISO 4217 code.',
].join('\n');
/**
* Keyword → reservation type, so an obvious document skips the costlier union/strong path.
*
@@ -213,21 +232,23 @@ async function extractFlights(text: string, ctx: RouterContext): Promise<FlatLik
/** One enforced call for a single reservation — a type-specific schema when the type is
* obvious from keywords, else a union schema the model fills with the type it picks. */
async function extractSingle(text: string, ctx: RouterContext): Promise<FlatLike> {
async function extractSingle(text: string, ctx: RouterContext, images?: Buffer[]): Promise<FlatLike> {
const known = detectType(text);
const call = (schema: Record<string, unknown>, hint: string) =>
extractEnforced({
baseUrl: ctx.baseUrl, model: ctx.model, apiKey: ctx.apiKey,
system: `Extract the single reservation from the document into the flat fields. ${hint} Omit any field that is truly absent.`,
user: `Document:\n${text}`,
user: images?.length ? 'The document is the attached photo.' : `Document:\n${text}`,
schema,
// A photo costs far more prompt tokens than the text of the same page.
...(images?.length ? { images: images.map((image) => image.toString('base64')), numCtx: 16384 } : {}),
});
if (known) {
const out = (await call(FLAT_SCHEMA_BY_TYPE[known], `It is a ${TYPE_HINT[known]}`)) ?? {};
return fixArrivalDate(normalizeDates({ ...out, type: known }));
}
const out = (await call(UNION_SINGLE_SCHEMA, 'Pick the correct "type".')) ?? {};
const out = (await call(UNION_SINGLE_SCHEMA, images?.length ? PHOTO_UNION_HINT : 'Pick the correct "type".')) ?? {};
// "I could not tell" used to be written down as "hotel", and a hotel is a
// booking, so a ferry voucher imported from the Transport tab opened the
// booking form with no transport type to pick and stayed 'other' (#2076).
@@ -298,3 +319,38 @@ export async function routeExtraction(text: string, ctx: RouterContext): Promise
const kiItems = nuExtractToKiReservations(flats as unknown as Record<string, unknown>[]) as unknown as KiReservation[];
return { kiItems, warnings };
}
/**
* The router for a photographed document: one enforced call with the photo
* attached, or the pages of a scanned PDF, one image each. There is no text to
* spot a flight number or a vendor in, so the model picks the type, and the
* booking-wide fields come from its answer alone.
*/
export async function routeImageExtraction(images: Buffer[], ctx: RouterContext): Promise<{ kiItems: KiReservation[]; warnings: string[] }> {
let flat: FlatLike;
try {
flat = unwrapPrice(await extractSingle('', ctx, images));
} catch (err) {
return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
}
const kiItems = nuExtractToKiReservations([flat] as unknown as Record<string, unknown>[]) as unknown as KiReservation[];
return { kiItems, warnings: [] };
}
/**
* The price as the amount alone. With no document text to take the total from,
* the model's `price` is all there is, and qwen3.5:4b writes it as an object
* inside the string (`{"amount": 49.00, "currency": "EUR"}`), which the amount
* parser then reads as 4900.
*/
function unwrapPrice(flat: FlatLike): FlatLike {
if (typeof flat.price !== 'string' || !flat.price.trim().startsWith('{')) return flat;
const inner = parseLenientJson(flat.price);
if (!inner || typeof inner !== 'object' || Array.isArray(inner)) return flat;
const { amount, currency } = inner as { amount?: unknown; currency?: unknown };
const out: FlatLike = { ...flat };
if (typeof amount === 'number' || typeof amount === 'string') out.price = String(amount);
else delete out.price;
if (!out.currency && typeof currency === 'string') out.currency = currency;
return out;
}
@@ -27,6 +27,8 @@ export interface EnforcedExtractInput {
numPredict?: number;
/** Context window. 8192 fits a typical multi-section booking; raise for long itineraries. */
numCtx?: number;
/** Base64 images attached to the user turn, for a vision model reading a photo. */
images?: string[];
}
/** Resolve the native API base from a config base URL that may end in `/v1`. */
@@ -59,7 +61,7 @@ export async function extractEnforced(input: EnforcedExtractInput): Promise<Reco
options: { temperature: 0, num_predict: input.numPredict ?? 512, num_ctx: input.numCtx ?? 8192 },
messages: [
{ role: 'system', content: input.system },
{ role: 'user', content: input.user },
{ role: 'user', content: input.user, ...(input.images?.length ? { images: input.images } : {}) },
],
};
@@ -0,0 +1,68 @@
import { Controller, HttpException, Param, Post, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { memoryStorage } from 'multer';
import type { ReceiptScanStartResponse } from '@trek/shared';
import type { User } from '../../types';
import { ADDON_IDS } from '../../addons';
import { AddonsService } from '../addons/addons.service';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { DatabaseService } from '../database/database.service';
import { PermissionsService } from '../permissions/permissions.service';
import { ImportJobsService } from '../booking-import/import-jobs.service';
import { LlmParseService } from '../llm-parse/llm-parse.service';
import { imageMimeType } from '../llm-parse/image-input';
const MAX_FILE_BYTES = 10 * 1024 * 1024;
/**
* Scan a receipt photo into an expense draft.
*
* It only reads: the job answers a `ReceiptRead` the client pre-fills the
* expense editor with, and the expense (and the photo, attached to it) is saved
* by that editor's own path once the person has checked it. The job is a
* booking-import job like any other, so the background tasks widget follows it
* and `GET …/reservations/import/jobs/:jobId` answers for it.
*
* Trip access, the addon and the right are checked in the handler, not by
* guards: a guard answers before multer has read the body, and a response sent
* while the client is still uploading reaches it as ECONNRESET, not as the 404
* or 403 (see files.controller.ts). The answers are the guards' own.
*
* There is no MCP twin, for the reason the booking-file import has none: the
* route takes an uploaded photo, and a tool caller has no file to hand it.
*/
@Controller('api/trips/:tripId/budget/receipt-scan')
@UseGuards(JwtAuthGuard)
export class ReceiptScanController {
constructor(
private readonly importJobs: ImportJobsService,
private readonly llmParse: LlmParseService,
private readonly db: DatabaseService,
private readonly permissions: PermissionsService,
private readonly addons: AddonsService,
) {}
@Post()
@UseInterceptors(FileInterceptor('file', { storage: memoryStorage(), limits: { fileSize: MAX_FILE_BYTES, files: 1 }, defParamCharset: 'utf8' }))
async scan(
@CurrentUser() user: User,
@Param('tripId') tripId: string,
@UploadedFile() file: Express.Multer.File | undefined,
): Promise<ReceiptScanStartResponse> {
const trip = this.db.canAccessTrip(tripId, user.id);
if (!trip) throw new HttpException({ error: 'Trip not found' }, 404);
if (!this.addons.isAddonEnabled(ADDON_IDS.BUDGET)) throw new HttpException({ error: 'Costs addon is not enabled' }, 404);
if (!this.permissions.checkPermission('budget_edit', user.role, trip.user_id, user.id, trip.user_id !== user.id)) {
throw new HttpException({ error: 'No permission' }, 403);
}
if (!file) throw new HttpException({ error: 'No file uploaded' }, 400);
if (!imageMimeType(file.originalname)) {
throw new HttpException({ error: `Unsupported file type: ${file.originalname}. Accepted: JPG, PNG, WEBP` }, 400);
}
if (!(await this.llmParse.readsImages(user.id))) {
throw new HttpException({ error: 'The configured AI model does not read photos' }, 400);
}
return { jobId: this.importJobs.startReceipt(tripId, file, user.id) };
}
}
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { ReceiptScanController } from './receipt-scan.controller';
import { BookingImportModule } from '../booking-import/booking-import.module';
import { LlmParseModule } from '../llm-parse/llm-parse.module';
import { AddonsModule } from '../addons/addons.module';
import { PermissionsModule } from '../permissions/permissions.module';
/** The Costs tab's receipt scan: an HTTP surface over the import job queue. */
@Module({
imports: [BookingImportModule, LlmParseModule, AddonsModule, PermissionsModule],
controllers: [ReceiptScanController],
})
export class ReceiptScanModule {}
@@ -27,8 +27,9 @@ import type { AirtrailImportResult } from '@trek/shared';
import { bookingImportModeSchema } from '@trek/shared';
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse, BookingImportMode } from '@trek/shared';
import { BookingImportConfirmDto, BookingImportPreviewDto } from './reservation-import.dto';
import { IMAGE_EXTENSIONS, imageMimeType } from '../llm-parse/image-input';
const ACCEPTED_EXTS = new Set(['.eml', '.pdf', '.pkpass', '.html', '.htm', '.txt']);
const ACCEPTED_EXTS = new Set(['.eml', '.pdf', '.pkpass', '.html', '.htm', '.txt', ...IMAGE_EXTENSIONS]);
const MAX_FILE_BYTES = 10 * 1024 * 1024;
const MAX_FILES = 5;
@@ -99,7 +100,7 @@ export class ReservationImportController {
/** Shared validation for both the sync and async import endpoints; returns the parsed mode. */
private validateImport(tripId: string, user: User, files: Express.Multer.File[] | undefined, rawMode?: string): BookingImportMode {
private async validateImport(tripId: string, user: User, files: Express.Multer.File[] | undefined, rawMode?: string): Promise<BookingImportMode> {
const modeResult = bookingImportModeSchema.safeParse(rawMode ?? 'no-ai');
if (!modeResult.success) throw new HttpException({ error: 'Invalid mode' }, 400);
@@ -118,12 +119,19 @@ export class ReservationImportController {
throw new HttpException({ error: `Unsupported file type: ${f.originalname}. Accepted: EML, PDF, PKPass, HTML, TXT` }, 400);
}
}
// A photo has no text layer and no structure: only a model that reads images
// can do anything with it, so it is refused up front rather than coming back
// as an empty preview.
if (files.some((f) => imageMimeType(f.originalname)) && (mode === 'no-ai' || !(await this.bookingImport.readsImages(user.id)))) {
throw new HttpException({ error: 'The configured AI model does not read photos' }, 400);
}
return mode;
}
/**
* POST /api/trips/:tripId/reservations/import/booking
* Accepts up to 5 booking confirmation files (EML, PDF, PKPass, HTML, TXT).
* Accepts up to 5 booking confirmation files (EML, PDF, PKPass, HTML, TXT, and
* photos when the AI model reads images).
* Returns a preview list without persisting anything.
*/
@RequirePermission('reservation_edit')
@@ -135,7 +143,7 @@ export class ReservationImportController {
@UploadedFiles() files: Express.Multer.File[] | undefined,
@Body() body: BookingImportPreviewDto,
): Promise<BookingImportPreviewResponse> {
const mode = this.validateImport(tripId, user, files, body?.mode);
const mode = await this.validateImport(tripId, user, files, body?.mode);
return this.bookingImport.preview(files!, mode, user.id);
}
@@ -155,7 +163,7 @@ export class ReservationImportController {
@UploadedFiles() files: Express.Multer.File[] | undefined,
@Body() body: BookingImportPreviewDto,
): Promise<{ jobId: string }> {
const mode = this.validateImport(tripId, user, files, body?.mode);
const mode = await this.validateImport(tripId, user, files, body?.mode);
const jobId = this.importJobs.start(tripId, files!, mode, user.id);
return { jobId };
}
+204
View File
@@ -0,0 +1,204 @@
/**
* Photos through the AI Parsing addon, end to end: the booking import taking a
* photo, the Costs receipt scan and its job, and the capability the pickers ask.
*
* Real: the guards, multer, LlmParseService with the instance's `vision`
* setting, the image cap, the OpenAI-compatible client, the import job queue
* and its status route. Replaced: the kitinerary binary (absent) and the one
* outbound call, `safeFetchLlm`, which answers as the provider would.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import cookieParser from 'cookie-parser';
import type { Server } from 'http';
import { Test } from '@nestjs/testing';
import { Jimp } from 'jimp';
import { sessionCookie } from './harness';
const { canAccessTrip, safeFetchLlm } = vi.hoisted(() => ({ canAccessTrip: vi.fn(), safeFetchLlm: vi.fn() }));
vi.mock('../../src/db/database', async () => {
const { default: Database } = await import('better-sqlite3');
const tmp = new Database(':memory:');
tmp.exec('PRAGMA journal_mode = WAL');
tmp.exec('PRAGMA foreign_keys = ON');
return { db: tmp, canAccessTrip, isOwner: vi.fn(() => true), getPlaceWithTags: vi.fn(), closeDb: () => {}, reinitialize: () => {} };
});
vi.mock('../../src/websocket', () => ({ broadcast: vi.fn(), broadcastToUser: vi.fn() }));
vi.mock('../../src/utils/ssrfGuard', async (orig) => ({ ...(await orig<Record<string, unknown>>()), safeFetchLlm }));
import { db } from '../../src/db/database';
import { createTables } from '../../src/db/schema';
import { runMigrations } from '../../src/db/migrations';
import { DatabaseModule } from '../../src/nest/database/database.module';
import { RealtimeModule } from '../../src/nest/realtime/realtime.module';
import { ReservationImportModule } from '../../src/nest/reservation-import/reservation-import.module';
import { ReceiptScanModule } from '../../src/nest/receipt-scan/receipt-scan.module';
import { LlmParseModule } from '../../src/nest/llm-parse/llm-parse.module';
import { KitineraryExtractorService } from '../../src/nest/booking-import/kitinerary-extractor.service';
import { NotificationsService } from '../../src/nest/notifications/notifications.service';
import { PermissionsService } from '../../src/nest/permissions/permissions.service';
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
function setVision(vision: string) {
db.prepare("UPDATE addons SET config = ? WHERE id = 'llm_parsing'").run(
JSON.stringify({ provider: 'openai', model: 'gpt-4.1-mini', apiKey: '', vision }),
);
}
/** The provider's chat-completions answer, carrying `payload` as its content. */
function providerAnswers(payload: unknown) {
safeFetchLlm.mockResolvedValue({
ok: true,
status: 200,
json: async () => ({ choices: [{ message: { content: JSON.stringify(payload) } }] }),
text: async () => '',
} as unknown as Response);
}
const sentUserContent = () => JSON.parse((safeFetchLlm.mock.calls[0][1] as RequestInit).body as string).messages[1].content;
describe('Photos through AI Parsing e2e', () => {
let server: Server;
let app: Awaited<ReturnType<typeof build>>;
let tripId: number;
let photo: Buffer;
async function build() {
const moduleRef = await Test.createTestingModule({ imports: [DatabaseModule, RealtimeModule, LlmParseModule, ReservationImportModule, ReceiptScanModule] })
.overrideProvider(KitineraryExtractorService)
.useValue({ onModuleInit: () => {}, isAvailable: () => false, extract: vi.fn(), describe: () => ({ available: false }) })
.overrideProvider(NotificationsService)
.useValue({ send: vi.fn().mockResolvedValue(undefined) })
.compile();
const nest = moduleRef.createNestApplication();
nest.use(cookieParser());
nest.useGlobalFilters(new TrekExceptionFilter());
nest.useGlobalPipes(new ZodValidationPipe());
await nest.init();
return nest;
}
beforeAll(async () => {
createTables(db);
runMigrations(db);
db.prepare(
"INSERT INTO users (id, username, email, password_hash, role, password_version) VALUES (1, 'e2e-user', 'e2e@example.test', 'x', 'user', 0)",
).run();
tripId = Number(db.prepare("INSERT INTO trips (user_id, title) VALUES (1, 'Lyon')").run().lastInsertRowid);
db.prepare(
`INSERT INTO addons (id, name, type, enabled, config) VALUES ('llm_parsing', 'AI Parsing', 'integration', 1, '{}')
ON CONFLICT(id) DO UPDATE SET enabled = 1`,
).run();
db.prepare(
`INSERT INTO addons (id, name, type, enabled) VALUES ('budget', 'Costs', 'trip', 1)
ON CONFLICT(id) DO UPDATE SET enabled = 1`,
).run();
photo = Buffer.from(await new Jimp({ width: 40, height: 60, color: 0xffffffff }).getBuffer('image/png'));
app = await build();
vi.spyOn(app.get(PermissionsService), 'checkPermission').mockReturnValue(true);
server = app.getHttpServer();
});
beforeEach(() => {
canAccessTrip.mockImplementation((id: unknown) => db.prepare('SELECT * FROM trips WHERE id = ?').get(id));
safeFetchLlm.mockReset();
setVision('on');
});
afterAll(async () => {
await app.close();
});
describe('GET /api/llm/capabilities', () => {
it('401 without a cookie', async () => {
expect((await request(server).get('/api/llm/capabilities')).status).toBe(401);
});
it('answers the instance setting, and no for a cloud model on Automatic', async () => {
expect((await request(server).get('/api/llm/capabilities').set('Cookie', sessionCookie(1))).body).toEqual({ images: true });
setVision('auto');
expect((await request(server).get('/api/llm/capabilities').set('Cookie', sessionCookie(1))).body).toEqual({ images: false });
expect(safeFetchLlm).not.toHaveBeenCalled();
});
});
describe('booking import', () => {
const upload = () =>
request(server)
.post(`/api/trips/${tripId}/reservations/import/booking`)
.set('Cookie', sessionCookie(1))
.field('mode', 'fallback-on-empty')
.attach('files', photo, { filename: 'ticket.png', contentType: 'image/png' });
it('sends a photo to the model as an image, and maps its answer', async () => {
providerAnswers({ reservations: [{ '@type': 'TrainReservation', reservationFor: { departureStation: { name: 'Lyon Part-Dieu' }, arrivalStation: { name: 'Paris Gare de Lyon' }, departureTime: '2026-10-03T08:04:00' } }] });
const res = await upload();
expect(res.status).toBe(201);
expect(res.body.items).toHaveLength(1);
expect(res.body.files).toEqual([{ fileName: 'ticket.png', aiAvailable: true, aiUsed: true }]);
const image = sentUserContent().find((p: { type: string }) => p.type === 'image_url');
expect(image.image_url.url).toMatch(/^data:image\/png;base64,/);
});
it('refuses a photo with 400 when the model reads no images', async () => {
setVision('off');
const res = await upload();
expect(res.status).toBe(400);
expect(res.body.error).toBe('The configured AI model does not read photos');
expect(safeFetchLlm).not.toHaveBeenCalled();
});
});
describe('POST /api/trips/:tripId/budget/receipt-scan', () => {
const scan = (name = 'bill.png') =>
request(server)
.post(`/api/trips/${tripId}/budget/receipt-scan`)
.set('Cookie', sessionCookie(1))
.attach('file', photo, { filename: name, contentType: name.endsWith('.png') ? 'image/png' : 'application/pdf' });
it('401 without a cookie, 404 for a trip out of reach', async () => {
expect((await request(server).post(`/api/trips/${tripId}/budget/receipt-scan`)).status).toBe(401);
canAccessTrip.mockReturnValue(undefined);
expect((await scan()).status).toBe(404);
});
it('reads the receipt in a job whose status answers what was read', async () => {
providerAnswers({ receipts: [{ merchant: 'Boulangerie du Port', date: '2026-09-21', total: 18.1, currency: 'EUR', items: [{ name: 'Quiche', price: 6.9 }] }] });
const res = await scan();
expect(res.status).toBe(201);
const { jobId } = res.body;
let status: request.Response | undefined;
await vi.waitFor(async () => {
status = await request(server).get(`/api/trips/${tripId}/reservations/import/jobs/${jobId}`).set('Cookie', sessionCookie(1));
expect(status.body.status).toBe('done');
});
expect(status!.body.result).toEqual({
receipt: { merchant: 'Boulangerie du Port', date: '2026-09-21', total: 18.1, currency: 'EUR', items: [{ name: 'Quiche', price: 6.9 }] },
warnings: [],
});
const body = JSON.parse((safeFetchLlm.mock.calls[0][1] as RequestInit).body as string);
expect(body.response_format.json_schema.name).toBe('receipts');
});
it('refuses a file that is not a photo, and any photo when the model reads no images', async () => {
expect((await scan('bill.pdf')).status).toBe(400);
setVision('off');
expect((await scan()).status).toBe(400);
expect(safeFetchLlm).not.toHaveBeenCalled();
});
it('404 while Costs is off', async () => {
db.prepare("UPDATE addons SET enabled = 0 WHERE id = 'budget'").run();
try {
const res = await scan();
expect(res.status).toBe(404);
expect(res.body.error).toBe('Costs addon is not enabled');
} finally {
db.prepare("UPDATE addons SET enabled = 1 WHERE id = 'budget'").run();
}
});
});
});
@@ -79,6 +79,18 @@ describe('BookingImportService.preview', () => {
expect(res.files![0].aiUsed).toBe(false);
});
it('fallback-on-empty: a photo goes straight to the LLM, kitinerary is not asked', async () => {
const { svc, extractor, llmParse } = make({
kit: true, ai: true,
extract: async () => [HOTEL_KI],
parse: async () => ({ kiItems: [HOTEL_KI], warnings: [] }),
});
const res = await svc.preview([file('ticket.jpg')], 'fallback-on-empty', 1);
expect(extractor.extract).not.toHaveBeenCalled();
expect(llmParse.parse).toHaveBeenCalledWith({ buffer: expect.any(Buffer), originalName: 'ticket.jpg' }, 1);
expect(res.items[0].needs_review).toBe(true);
});
it('force-ai: skips kitinerary entirely and uses the LLM', async () => {
const { svc, extractor, llmParse } = make({
kit: true, ai: true,
@@ -7,8 +7,8 @@ import { ImportJobsService } from '../../../../src/nest/booking-import/import-jo
import { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
type Preview = ReturnType<typeof vi.fn>;
function makeService(preview: Preview) {
return new ImportJobsService({ preview } as never, new RealtimeService());
function makeService(preview: Preview, readReceipt: Preview = vi.fn()) {
return new ImportJobsService({ preview } as never, new RealtimeService(), { readReceipt } as never);
}
const files = (n: number) => Array.from({ length: n }, (_, i) => ({ originalname: `f${i}.pdf` })) as never;
const eventsFor = (jobId: string) => broadcastToUser.mock.calls.map((c) => c[1]).filter((p) => p.jobId === jobId);
@@ -101,3 +101,30 @@ describe('ImportJobsService', () => {
expect(svc.get(id, 99)).toBeUndefined();
});
});
describe('ImportJobsService.startReceipt', () => {
it('reads the receipt off-request and pushes the read as the result', async () => {
const result = { receipt: { merchant: 'Shop', date: null, total: 4, currency: 'EUR', items: [] }, warnings: [] };
const readReceipt = vi.fn(async () => result);
const svc = makeService(vi.fn(), readReceipt);
const id = svc.startReceipt('7', { originalname: 'r.jpg', buffer: Buffer.from('x') } as never, 42);
await vi.waitFor(() => expect(svc.get(id, 42)?.status).toBe('done'));
expect(svc.get(id, 42)).toMatchObject({ result, done: 1, total: 1 });
expect(readReceipt).toHaveBeenCalledWith({ buffer: Buffer.from('x'), originalName: 'r.jpg' }, 42);
expect(eventsFor(id).find((p) => p.type === 'import:done')).toMatchObject({ result, tripId: '7' });
});
it('queues behind the same user\'s booking parse rather than running beside it', async () => {
let release!: () => void;
const preview = vi.fn(() => new Promise((resolve) => { release = () => resolve({ items: [] }); }));
const readReceipt = vi.fn(async () => ({ receipt: null, warnings: [] }));
const svc = makeService(preview, readReceipt);
svc.start('7', files(1), 'force-ai', 42);
const id = svc.startReceipt('7', { originalname: 'r.jpg', buffer: Buffer.from('x') } as never, 42);
await new Promise((r) => setTimeout(r, 10));
expect(readReceipt).not.toHaveBeenCalled();
release();
await vi.waitFor(() => expect(svc.get(id, 42)?.status).toBe('done'));
});
});
@@ -14,6 +14,7 @@ function make(over: Partial<BookingImportService> = {}) {
canEdit: vi.fn(() => true),
isAvailable: vi.fn(() => true),
aiAvailable: vi.fn(() => true),
readsImages: vi.fn(async () => true),
preview: vi.fn(async () => ({ items: [], warnings: [], files: [] })),
...over,
} as unknown as BookingImportService;
@@ -55,6 +56,28 @@ describe('ReservationImportController.preview', () => {
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'fallback-on-empty', 1);
});
it('takes a photo when the model reads images, without asking otherwise', async () => {
const { c, svc } = make();
await c.preview(user, 't1', [file('ticket.JPG')], { mode: 'fallback-on-empty' });
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'fallback-on-empty', 1);
(svc.readsImages as ReturnType<typeof vi.fn>).mockClear();
await c.preview(user, 't1', [file('a.pdf')], { mode: 'fallback-on-empty' });
expect(svc.readsImages).not.toHaveBeenCalled();
});
it('refuses a photo with 400 when the model does not read images, or AI is not asked', async () => {
const { c, svc } = make({ readsImages: vi.fn(async () => false) });
expect(await status(() => c.preview(user, 't1', [file('a.pdf'), file('ticket.png')], { mode: 'fallback-on-empty' }))).toBe(400);
const { c: c2 } = make();
expect(await status(() => c2.preview(user, 't1', [file('ticket.png')], { mode: 'no-ai' }))).toBe(400);
expect(svc.preview).not.toHaveBeenCalled();
});
it('still refuses a file type it does not know, HEIC included', async () => {
const { c } = make();
expect(await status(() => c.preview(user, 't1', [file('IMG_1.heic')], { mode: 'fallback-on-empty' }))).toBe(400);
});
it('defaults the mode to no-ai when omitted', async () => {
const { c, svc } = make();
await c.preview(user, 't1', [file()], {});
@@ -275,6 +275,15 @@ describe('OpenAiCompatibleClient', () => {
expect(safeFetchLlmMock).toHaveBeenCalledTimes(1);
});
it('sends each page of a scanned PDF as its own image_url, in order', async () => {
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
const page = (n: string) => ({ mimeType: 'image/png', data: Buffer.from(n) });
await new OpenAiCompatibleClient().extract({ ...baseInput, text: undefined, file: page('p1'), pageImages: [page('p2')] });
const parts = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string).messages[1].content;
const urls = parts.filter((p: { type: string }) => p.type === 'image_url').map((p: { image_url: { url: string } }) => p.image_url.url);
expect(urls).toEqual([`data:image/png;base64,${Buffer.from('p1').toString('base64')}`, `data:image/png;base64,${Buffer.from('p2').toString('base64')}`]);
});
it('sends an image natively as image_url but never a file/pdf part', async () => {
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
await new OpenAiCompatibleClient().extract({ ...baseInput, file: { mimeType: 'image/png', data: Buffer.from('IMG') } });
@@ -380,6 +389,28 @@ describe('OpenAiCompatibleClient — NuExtract path', () => {
});
describe('AnthropicClient', () => {
it('sends the pages of a scanned PDF as image blocks after the first', async () => {
const fetchFn = mockFetch(() =>
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }),
);
const page = (n: string) => ({ mimeType: 'image/png', data: Buffer.from(n) });
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: page('p1'), pageImages: [page('p2')] });
const content = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string).messages[0].content;
expect(content.map((b: { type: string }) => b.type)).toEqual(['image', 'image', 'text']);
expect(content[1].source.data).toBe(Buffer.from('p2').toString('base64'));
});
it('sends a photo as an image block and a PDF as a document block', async () => {
const fetchFn = mockFetch(() =>
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }),
);
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: { mimeType: 'image/jpeg', data: Buffer.from('jpg') } });
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: { mimeType: 'application/pdf', data: Buffer.from('pdf') } });
const blockOf = (i: number) => JSON.parse((fetchFn.mock.calls[i][1] as RequestInit).body as string).messages[0].content[0];
expect(blockOf(0)).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: Buffer.from('jpg').toString('base64') } });
expect(blockOf(1).type).toBe('document');
});
it('forces the emit_reservations tool and reads its input', async () => {
const fetchFn = mockFetch(() =>
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [{ '@type': 'LodgingReservation' }] } }] }),
@@ -15,6 +15,7 @@ import {
detectFlightNumbers,
fixArrivalDate,
routeExtraction,
routeImageExtraction,
} from '../../../../src/nest/llm-parse/router/extraction-router';
const CTX = { baseUrl: 'http://ollama:11434/v1', model: 'qwen3:8b' };
@@ -257,3 +258,64 @@ describe('printed 12-hour clocks and unreadable types (#2094, #2076)', () => {
expect(flats[0].type_guessed).toBeUndefined();
});
});
describe('routeImageExtraction', () => {
it('makes one enforced call with the photo attached and lets the model pick the type', async () => {
extractEnforced.mockResolvedValue({ type: 'train', from_name: 'Lyon', to_name: 'Paris', price: '42', currency: 'EUR' });
const out = await routeImageExtraction([Buffer.from('photo')], CTX);
expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [] });
expect(extractEnforced).toHaveBeenCalledTimes(1);
const call = extractEnforced.mock.calls[0][0];
expect(call.images).toEqual([Buffer.from('photo').toString('base64')]);
expect(call.numCtx).toBe(16384);
expect(call.user).not.toMatch(/Document:/);
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ type: 'train', from_name: 'Lyon', price: '42', currency: 'EUR' });
});
it('tells the model what each type\'s fields mean, since a photo has no keyword to pick a schema', async () => {
extractEnforced.mockResolvedValue({ type: 'train' });
await routeImageExtraction([Buffer.from('photo')], CTX);
const system = extractEnforced.mock.calls[0][0].system;
expect(system).toMatch(/from_name\/to_name = stations/);
expect(system).toMatch(/full ISO/);
});
it('reads a price the model wrapped as an object inside the string', async () => {
extractEnforced.mockResolvedValue({ type: 'train', price: '{"amount": 49.00, "currency": "EUR"}' });
await routeImageExtraction([Buffer.from('photo')], CTX);
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ price: '49', currency: 'EUR' });
mapToKi.mockClear();
extractEnforced.mockResolvedValue({ type: 'train', price: '{"amount": "12,50"}', currency: 'CHF' });
await routeImageExtraction([Buffer.from('photo')], CTX);
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ price: '12,50', currency: 'CHF' });
mapToKi.mockClear();
extractEnforced.mockResolvedValue({ type: 'train', price: '{"note": "see below"}' });
await routeImageExtraction([Buffer.from('photo')], CTX);
expect(mapToKi.mock.calls[0][0][0]).not.toHaveProperty('price');
});
it('leaves a plain price, or one that only looks like an object, as it came', async () => {
for (const price of ['49,00 EUR', '{not json']) {
mapToKi.mockClear();
extractEnforced.mockResolvedValue({ type: 'train', price });
await routeImageExtraction([Buffer.from('photo')], CTX);
expect(mapToKi.mock.calls[0][0][0].price).toBe(price);
}
});
it('attaches every page of a scanned PDF to the one call', async () => {
extractEnforced.mockResolvedValue({ type: 'hotel' });
await routeImageExtraction([Buffer.from('p1'), Buffer.from('p2')], CTX);
expect(extractEnforced).toHaveBeenCalledTimes(1);
expect(extractEnforced.mock.calls[0][0].images).toEqual([Buffer.from('p1').toString('base64'), Buffer.from('p2').toString('base64')]);
});
it('degrades to a warning when the call throws', async () => {
extractEnforced.mockRejectedValue(new Error('timeout'));
const out = await routeImageExtraction([Buffer.from('photo')], CTX);
expect(out.kiItems).toEqual([]);
expect(out.warnings[0]).toMatch(/timeout/);
});
});
@@ -0,0 +1,94 @@
import { describe, it, expect } from 'vitest';
import { Jimp } from 'jimp';
import { capImage, imageMimeType, IMAGE_MAX_EDGE, renderPdfPages } from '../../../../src/nest/llm-parse/image-input';
async function png(w: number, h: number): Promise<Buffer> {
const image = new Jimp({ width: w, height: h, color: 0xffffffff });
return Buffer.from(await image.getBuffer('image/png'));
}
describe('imageMimeType', () => {
it('knows the photo formats a provider reads, whatever the case', () => {
expect(imageMimeType('a.jpg')).toBe('image/jpeg');
expect(imageMimeType('A.JPEG')).toBe('image/jpeg');
expect(imageMimeType('a.png')).toBe('image/png');
expect(imageMimeType('a.webp')).toBe('image/webp');
});
it('is null for everything else, HEIC included', () => {
for (const name of ['a.heic', 'a.pdf', 'a.txt', 'jpg', 'a.jpg.pdf']) expect(imageMimeType(name)).toBeNull();
});
});
describe('capImage', () => {
it('shrinks the long edge of a large photo to the cap, as a JPEG', async () => {
const out = await capImage(await png(IMAGE_MAX_EDGE * 2, 1000), 'image/png');
expect(out.mimeType).toBe('image/jpeg');
const back = await Jimp.read(out.data);
expect(back.width).toBe(IMAGE_MAX_EDGE);
expect(back.height).toBe(500);
});
it('caps a portrait photo by its height', async () => {
const back = await Jimp.read((await capImage(await png(800, IMAGE_MAX_EDGE * 2), 'image/png')).data);
expect(back.height).toBe(IMAGE_MAX_EDGE);
expect(back.width).toBe(400);
});
it('leaves a photo that is already small enough untouched', async () => {
const data = await png(400, 300);
expect(await capImage(data, 'image/png')).toEqual({ data, mimeType: 'image/png' });
});
it('passes bytes it cannot decode through unchanged', async () => {
const data = Buffer.from('RIFF....WEBP');
expect(await capImage(data, 'image/webp')).toEqual({ data, mimeType: 'image/webp' });
});
});
/** A PDF with no text layer: each page is one JPEG, as a scanner writes it. */
async function scannedPdf(pages: number): Promise<Buffer> {
const jpg = Buffer.from(await new Jimp({ width: 300, height: 400, color: 0xddddddff }).getBuffer('image/jpeg'));
const objects: Buffer[] = [];
const kids = Array.from({ length: pages }, (_, i) => `${3 + i * 3} 0 R`).join(' ');
objects.push(Buffer.from('<< /Type /Catalog /Pages 2 0 R >>'));
objects.push(Buffer.from(`<< /Type /Pages /Kids [${kids}] /Count ${pages} >>`));
for (let i = 0; i < pages; i++) {
const page = 3 + i * 3;
const content = 'q 300 0 0 400 0 0 cm /Im0 Do Q';
objects.push(Buffer.from(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /XObject << /Im0 ${page + 2} 0 R >> >> /Contents ${page + 1} 0 R >>`));
objects.push(Buffer.from(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`));
objects.push(Buffer.concat([
Buffer.from(`<< /Type /XObject /Subtype /Image /Width 300 /Height 400 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode /Length ${jpg.length} >>\nstream\n`),
jpg,
Buffer.from('\nendstream'),
]));
}
const parts: Buffer[] = [Buffer.from('%PDF-1.4\n')];
const offsets: number[] = [];
let length = parts[0].length;
objects.forEach((body, i) => {
offsets.push(length);
const obj = Buffer.concat([Buffer.from(`${i + 1} 0 obj\n`), body, Buffer.from('\nendobj\n')]);
parts.push(obj);
length += obj.length;
});
const rows = offsets.map((o) => `${String(o).padStart(10, '0')} 00000 n \n`).join('');
parts.push(Buffer.from(`xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${rows}trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${length}\n%%EOF\n`));
return Buffer.concat(parts);
}
describe('renderPdfPages', () => {
it('draws the first two pages of a scan as images no larger than a capped photo', async () => {
const pages = await renderPdfPages(await scannedPdf(3));
expect(pages).toHaveLength(2);
for (const page of pages) {
const image = await Jimp.read(page.data);
expect(Math.max(image.width, image.height)).toBeLessThanOrEqual(IMAGE_MAX_EDGE);
}
}, 30_000);
it('throws for bytes that are not a PDF, which the caller reports as unreadable', async () => {
await expect(renderPdfPages(Buffer.from('not a pdf'))).rejects.toThrow();
});
});
@@ -0,0 +1,20 @@
import { describe, it, expect, vi } from 'vitest';
import { GUARDS_METADATA } from '@nestjs/common/constants';
import { LlmCapabilitiesController } from '../../../../src/nest/llm-parse/llm-capabilities.controller';
import { JwtAuthGuard } from '../../../../src/nest/auth/jwt-auth.guard';
import type { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
import type { User } from '../../../../src/types';
describe('LlmCapabilitiesController', () => {
it("answers whether the caller's model reads images", async () => {
const readsImages = vi.fn(async (id: number) => id === 1);
const c = new LlmCapabilitiesController({ readsImages } as unknown as LlmParseService);
await expect(c.capabilities({ id: 1 } as User)).resolves.toEqual({ images: true });
await expect(c.capabilities({ id: 2 } as User)).resolves.toEqual({ images: false });
expect(readsImages).toHaveBeenCalledWith(2);
});
it('needs a session', () => {
expect(Reflect.getMetadata(GUARDS_METADATA, LlmCapabilitiesController)).toContain(JwtAuthGuard);
});
});
@@ -64,16 +64,25 @@ describe('resolveLlmConfig', () => {
});
it('uses instance config when present (and decrypts the key)', () => {
setInstanceConfig({ provider: 'anthropic', model: 'claude-opus-4-8', apiKey: 'sk-plain', multimodal: true });
setInstanceConfig({ provider: 'anthropic', model: 'claude-opus-4-8', apiKey: 'sk-plain', vision: 'on' });
expect(resolver.resolve(1)).toEqual({
provider: 'anthropic',
model: 'claude-opus-4-8',
baseUrl: undefined,
apiKey: 'sk-plain',
multimodal: true,
vision: 'on',
});
});
it('reads the instance vision setting as auto when it is missing, unknown, or only the old multimodal flag', () => {
for (const extra of [{}, { vision: 'maybe' }, { multimodal: false }, { multimodal: true }]) {
setInstanceConfig({ provider: 'local', model: 'qwen3.5:4b', ...extra });
expect(resolver.resolve(1)?.vision).toBe('auto');
}
setInstanceConfig({ provider: 'local', model: 'qwen3.5:4b', vision: 'off' });
expect(resolver.resolve(1)?.vision).toBe('off');
});
it('instance config with a base URL still wins for a plain user (#1772 does not touch it)', () => {
setInstanceConfig({ provider: 'local', model: 'nuextract', baseUrl: 'http://ollama:11434' });
setRole('user');
@@ -89,7 +98,7 @@ describe('resolveLlmConfig', () => {
model: 'claude-sonnet',
baseUrl: undefined,
apiKey: 'user-key',
multimodal: true,
vision: 'on',
});
expect(getDecryptedUserSetting).toHaveBeenCalledWith(7, 'llm_api_key');
});
@@ -114,7 +123,7 @@ describe('resolveLlmConfig', () => {
model: 'gpt-4o-mini',
baseUrl: undefined,
apiKey: 'sk-user',
multimodal: false,
vision: 'off',
});
});
@@ -0,0 +1,19 @@
import { describe, it, expect } from 'vitest';
import { prepareLlmAddonConfigForWrite } from '../../../../src/nest/llm-parse/llm-config';
describe('prepareLlmAddonConfigForWrite: the vision setting', () => {
it('stores a known mode as sent', () => {
for (const vision of ['auto', 'on', 'off']) {
expect(prepareLlmAddonConfigForWrite({ provider: 'local', vision }, undefined).vision).toBe(vision);
}
});
it('stores anything else as auto, so the resolver never meets a value it cannot read', () => {
expect(prepareLlmAddonConfigForWrite({ vision: 'maybe' }, undefined).vision).toBe('auto');
expect(prepareLlmAddonConfigForWrite({ vision: true }, undefined).vision).toBe('auto');
});
it('adds no vision key to a config that sent none', () => {
expect(prepareLlmAddonConfigForWrite({ provider: 'openai', model: 'm' }, undefined)).not.toHaveProperty('vision');
});
});
@@ -68,3 +68,67 @@ describe('LlmLocalService.pull', () => {
expect(JSON.parse(init.body)).toEqual({ model: 'nuextract', stream: true });
});
});
describe('LlmLocalService.modelCapabilities', () => {
const showOk = (body: unknown, headers: Record<string, string> = {}) => ({
ok: true,
headers: { get: (name: string) => headers[name] ?? null },
json: async () => body,
});
it('asks /api/show for the model and answers its capabilities', async () => {
const fetchFn = mockFetch(async () => showOk({ capabilities: ['completion', 'vision'] }));
await expect(svc().modelCapabilities('http://ollama:11434/v1', 'qwen3.5:4b')).resolves.toEqual(['completion', 'vision']);
expect(fetchFn.mock.calls[0][0]).toBe('http://ollama:11434/api/show');
expect(JSON.parse(fetchFn.mock.calls[0][1].body)).toEqual({ model: 'qwen3.5:4b' });
});
it('asks the server once per server and model, not once per call', async () => {
const fetchFn = mockFetch(async () => showOk({ capabilities: ['completion'] }));
const s = svc();
await s.modelCapabilities('http://ollama:11434', 'qwen3:8b');
await s.modelCapabilities('http://ollama:11434', 'qwen3:8b');
expect(fetchFn).toHaveBeenCalledTimes(1);
await s.modelCapabilities('http://ollama:11434', 'qwen3.5:4b');
expect(fetchFn).toHaveBeenCalledTimes(2);
});
it('answers null for a URL it cannot use, without throwing and without a request', async () => {
await expect(svc().modelCapabilities('not a url', 'm')).resolves.toBeNull();
await expect(svc().modelCapabilities('ftp://x', 'm')).resolves.toBeNull();
expect(safeFetchLlmMock).not.toHaveBeenCalled();
});
it('answers null for an unreachable server, a model it lacks, or a body that is not Ollama\'s', async () => {
mockFetch(async () => { throw new Error('ECONNREFUSED'); });
await expect(svc().modelCapabilities('http://a:1', 'm')).resolves.toBeNull();
mockFetch(async () => ({ ok: false, status: 404, body: null }));
await expect(svc().modelCapabilities('http://b:1', 'm')).resolves.toBeNull();
mockFetch(async () => showOk({ capabilities: 'vision' }));
await expect(svc().modelCapabilities('http://c:1', 'm')).resolves.toBeNull();
});
it('does not read a body that declares itself larger than the cap', async () => {
const json = vi.fn(async () => ({ capabilities: ['vision'] }));
mockFetch(async () => ({ ...showOk({}), headers: { get: () => String(2 * 1024 * 1024) }, json }));
await expect(svc().modelCapabilities('http://d:1', 'm')).resolves.toBeNull();
expect(json).not.toHaveBeenCalled();
});
it('asks again a minute after a miss, and keeps an answer for ten', async () => {
vi.useFakeTimers();
try {
const s = svc();
mockFetch(async () => { throw new Error('down'); });
await s.modelCapabilities('http://e:1', 'm');
vi.advanceTimersByTime(61_000);
mockFetch(async () => showOk({ capabilities: ['vision'] }));
await expect(s.modelCapabilities('http://e:1', 'm')).resolves.toEqual(['vision']);
vi.advanceTimersByTime(9 * 60_000);
mockFetch(async () => showOk({ capabilities: [] }));
await expect(s.modelCapabilities('http://e:1', 'm')).resolves.toEqual(['vision']);
} finally {
vi.useRealTimers();
}
});
});
@@ -10,30 +10,49 @@ const { createLlmClient, extract } = vi.hoisted(() => {
});
vi.mock('../../../../src/nest/llm-parse/llm-client.factory', () => ({ createLlmClient }));
const { extractEnforced } = vi.hoisted(() => ({ extractEnforced: vi.fn() }));
const { renderPdfPages } = vi.hoisted(() => ({ renderPdfPages: vi.fn() }));
vi.mock('../../../../src/nest/llm-parse/image-input', async (orig) => {
const actual = await orig() as Record<string, unknown>;
return { ...actual, renderPdfPages };
});
vi.mock('../../../../src/nest/llm-parse/router/ollama-format.client', async (orig) => {
const actual = await orig() as Record<string, unknown>;
return { ...actual, extractEnforced };
});
const { extractText } = vi.hoisted(() => ({ extractText: vi.fn(async () => 'Flight AB123') }));
vi.mock('../../../../src/nest/llm-parse/text-extract', async (orig) => {
const actual = await orig() as Record<string, unknown>;
return { ...actual, extractText };
});
const { routeExtraction, detectFlightNumbers } = vi.hoisted(() => ({
const { routeExtraction, routeImageExtraction, detectFlightNumbers } = vi.hoisted(() => ({
routeExtraction: vi.fn(),
routeImageExtraction: vi.fn(),
detectFlightNumbers: vi.fn(() => [] as string[]),
}));
// The router's pure helpers stay real: the currency fallback reads the document's
// total through extractTotalPrice.
vi.mock('../../../../src/nest/llm-parse/router/extraction-router', async (orig) => {
const actual = await orig() as Record<string, unknown>;
return { ...actual, routeExtraction, detectFlightNumbers };
return { ...actual, routeExtraction, routeImageExtraction, detectFlightNumbers };
});
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver';
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
const cfg = (over: Record<string, unknown> = {}) => ({ provider: 'openai', model: 'm', multimodal: false, ...over });
const cfg = (over: Record<string, unknown> = {}) => ({ provider: 'openai', model: 'm', vision: 'off', ...over });
const llmConfigStub = { resolve: resolveLlmConfig } as unknown as LlmConfigResolver;
const svc = () => new LlmParseService(llmConfigStub, { isManaged: () => false } as unknown as RuntimeEnvService);
const modelCapabilities = vi.fn();
const svc = () => new LlmParseService(
llmConfigStub,
{ isManaged: () => false } as unknown as RuntimeEnvService,
{ modelCapabilities } as unknown as LlmLocalService,
);
const file = (name: string, body = 'Flight AB123') => ({ buffer: Buffer.from(body), originalName: name });
beforeEach(() => {
@@ -43,6 +62,9 @@ beforeEach(() => {
extractText.mockResolvedValue('Flight AB123');
detectFlightNumbers.mockReturnValue([]);
routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [] });
routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [] });
modelCapabilities.mockResolvedValue(null);
renderPdfPages.mockResolvedValue([]);
});
describe('LlmParseService', () => {
@@ -284,3 +306,165 @@ describe('LlmParseService: cleaning up an AI answer (#2477)', () => {
expect(res.kiItems).toEqual([plain]);
});
});
describe('LlmParseService: a photo', () => {
it('readsImages follows on and off whatever the provider, and never asks a server for them', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on' }));
await expect(svc().readsImages(1)).resolves.toBe(true);
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'off' }));
await expect(svc().readsImages(1)).resolves.toBe(false);
resolveLlmConfig.mockReturnValue(cfg({ provider: 'openai', vision: 'on' }));
await expect(svc().readsImages(1)).resolves.toBe(true);
expect(modelCapabilities).not.toHaveBeenCalled();
});
it('readsImages on auto asks a local server, and answers no for a cloud provider', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'auto', baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b' }));
modelCapabilities.mockResolvedValue(['completion', 'vision']);
await expect(svc().readsImages(1)).resolves.toBe(true);
expect(modelCapabilities).toHaveBeenCalledWith('http://ollama:11434/v1', 'qwen3.5:4b');
modelCapabilities.mockResolvedValue(['completion']);
await expect(svc().readsImages(1)).resolves.toBe(false);
modelCapabilities.mockResolvedValue(null);
await expect(svc().readsImages(1)).resolves.toBe(false);
modelCapabilities.mockClear();
for (const provider of ['openai', 'anthropic']) {
resolveLlmConfig.mockReturnValue(cfg({ provider, vision: 'auto' }));
await expect(svc().readsImages(1)).resolves.toBe(false);
}
expect(modelCapabilities).not.toHaveBeenCalled();
});
it('readsImages is false when AI parsing is not configured', async () => {
resolveLlmConfig.mockReturnValue(null);
await expect(svc().readsImages(1)).resolves.toBe(false);
});
it('refuses a photo the model cannot read, without calling it', async () => {
const res = await svc().parse(file('ticket.jpg', 'jpeg'), 1);
expect(res.kiItems).toEqual([]);
expect(res.warnings[0]).toMatch(/does not read images/);
expect(extract).not.toHaveBeenCalled();
expect(extractText).not.toHaveBeenCalled();
});
it('sends a photo as an image to a cloud provider, with no text', async () => {
for (const provider of ['openai', 'anthropic']) {
extract.mockClear();
resolveLlmConfig.mockReturnValue(cfg({ provider, vision: 'on' }));
await svc().parse(file('ticket.PNG', 'not really a png'), 1);
const input = extract.mock.calls[0][0];
// Bytes that do not decode go as they came rather than being refused.
expect(input.file).toEqual({ mimeType: 'image/png', data: Buffer.from('not really a png') });
expect(input.text).toBeUndefined();
}
expect(extractText).not.toHaveBeenCalled();
});
it('sends a photo through the local router with the image attached', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on', baseUrl: 'http://ollama:11434/v1' }));
const res = await svc().parse(file('ticket.jpeg', 'jpeg bytes'), 1);
expect(res.kiItems).toEqual([{ '@type': 'TrainReservation' }]);
expect(routeImageExtraction).toHaveBeenCalledWith([Buffer.from('jpeg bytes')], expect.objectContaining({ baseUrl: 'http://ollama:11434/v1' }));
expect(routeExtraction).not.toHaveBeenCalled();
expect(extract).not.toHaveBeenCalled();
});
it('leaves a PDF on its text path when the model reads images', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
await svc().parse(file('a.pdf', '%PDF'), 1);
expect(extract.mock.calls[0][0].file).toBeUndefined();
expect(extractText).toHaveBeenCalled();
});
});
describe('LlmParseService.readReceipt', () => {
const RECEIPT = { merchant: 'Bäckerei', date: '2026-09-20', total: 7.5, currency: 'EUR', items: [{ name: 'Brezel', price: 2.5 }] };
it('says why when AI parsing is not set up, the file is not a photo, or the model reads no images', async () => {
resolveLlmConfig.mockReturnValue(null);
expect((await svc().readReceipt(file('r.jpg'), 1)).warnings[0]).toMatch(/not configured/);
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
expect(await svc().readReceipt(file('r.pdf'), 1)).toEqual({ receipt: null, warnings: ['r.pdf: not a photo'] });
resolveLlmConfig.mockReturnValue(cfg({ vision: 'off' }));
expect((await svc().readReceipt(file('r.jpg'), 1)).warnings[0]).toMatch(/does not read images/);
expect(extract).not.toHaveBeenCalled();
expect(extractEnforced).not.toHaveBeenCalled();
});
it('reads through Ollama\'s native chat on a local provider, with the photo and the receipt schema', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on', baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b' }));
extractEnforced.mockResolvedValue(RECEIPT);
await expect(svc().readReceipt(file('r.jpg', 'jpeg'), 1)).resolves.toEqual({ receipt: RECEIPT, warnings: [] });
const call = extractEnforced.mock.calls[0][0];
expect(call).toMatchObject({ baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b', images: [Buffer.from('jpeg').toString('base64')] });
expect(call.schema.required).toContain('total');
expect(extract).not.toHaveBeenCalled();
});
it('asks a cloud client for a receipts list with the photo attached, and keeps the first', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic', vision: 'on' }));
extract.mockResolvedValue([RECEIPT, { total: 1 }]);
await expect(svc().readReceipt(file('r.png', 'png'), 1)).resolves.toEqual({ receipt: RECEIPT, warnings: [] });
const input = extract.mock.calls[0][0];
expect(input.rootKey).toBe('receipts');
expect(input.file).toEqual({ mimeType: 'image/png', data: Buffer.from('png') });
expect(input.text).toBeUndefined();
});
it('answers a warning, not a throw, when the model fails or reads nothing', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
extract.mockRejectedValue(new Error('429'));
const failed = await svc().readReceipt(file('r.jpg'), 1);
expect(failed.receipt).toBeNull();
expect(failed.warnings[0]).toMatch(/AI parsing failed — 429/);
extract.mockResolvedValue([]);
expect(await svc().readReceipt(file('r.jpg'), 1)).toEqual({ receipt: null, warnings: ['r.jpg: no receipt could be read'] });
});
});
describe('LlmParseService: a scanned PDF', () => {
const page = (n: string) => ({ mimeType: 'image/jpeg', data: Buffer.from(n) });
beforeEach(() => extractText.mockResolvedValue(' '));
it('sends the pages as images when the text layer is empty and the model reads images', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
renderPdfPages.mockResolvedValue([page('p1'), page('p2')]);
await svc().parse(file('scan.pdf', '%PDF'), 1);
const input = extract.mock.calls[0][0];
expect(input.file).toEqual(page('p1'));
expect(input.pageImages).toEqual([page('p2')]);
expect(input.text).toBeUndefined();
});
it('hands the local router every page', async () => {
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on' }));
renderPdfPages.mockResolvedValue([page('p1'), page('p2')]);
await svc().parse(file('scan.pdf', '%PDF'), 1);
expect(routeImageExtraction).toHaveBeenCalledWith([Buffer.from('p1'), Buffer.from('p2')], expect.anything());
expect(routeExtraction).not.toHaveBeenCalled();
});
it('keeps the warning, and draws nothing, when the model reads no images', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'off' }));
const res = await svc().parse(file('scan.pdf', '%PDF'), 1);
expect(res.warnings[0]).toMatch(/no readable text found \(a scanned PDF needs a model that reads images\)/);
expect(renderPdfPages).not.toHaveBeenCalled();
expect(extract).not.toHaveBeenCalled();
});
it('keeps the warning when no page could be drawn', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
const res = await svc().parse(file('scan.pdf', '%PDF'), 1);
expect(res.warnings[0]).toMatch(/no readable text found/);
expect(extract).not.toHaveBeenCalled();
});
it('does not draw an empty text file', async () => {
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
await svc().parse(file('empty.txt', ''), 1);
expect(renderPdfPages).not.toHaveBeenCalled();
});
});
@@ -52,6 +52,15 @@ describe('extractEnforced', () => {
expect((init as RequestInit).headers).not.toHaveProperty('authorization');
});
it('attaches images to the user turn only when given', async () => {
const fetchFn = mockFetch(() => jsonResponse({ message: { content: '{}' } }));
await extractEnforced({ ...INPUT, images: ['aW1n'] });
await extractEnforced(INPUT);
const userTurn = (i: number) => JSON.parse((fetchFn.mock.calls[i][1] as RequestInit).body as string).messages[1];
expect(userTurn(0).images).toEqual(['aW1n']);
expect(userTurn(1)).not.toHaveProperty('images');
});
it('sends a bearer header only when an apiKey is given', async () => {
const fetchFn = mockFetch(() => jsonResponse({ message: { content: '{}' } }));
await extractEnforced({ ...INPUT, apiKey: 'sk-123', numPredict: 900, numCtx: 16000 });
@@ -0,0 +1,52 @@
import { describe, it, expect } from 'vitest';
import { buildReceiptPrompt, RECEIPT_LIST_JSON_SCHEMA, RECEIPT_ROOT_KEY, toReceiptRead } from '../../../../src/nest/llm-parse/receipt-read';
describe('toReceiptRead', () => {
it('keeps a clean answer as it came', () => {
expect(toReceiptRead({
merchant: 'Café de Flore', date: '2026-09-20', total: 23.4, currency: 'EUR',
items: [{ name: '2 x Espresso', price: 8 }, { name: 'Croque', price: 15.4 }],
})).toEqual({
merchant: 'Café de Flore', date: '2026-09-20', total: 23.4, currency: 'EUR',
items: [{ name: '2 x Espresso', price: 8 }, { name: 'Croque', price: 15.4 }],
});
});
it('reads amounts written as text, in either decimal convention', () => {
expect(toReceiptRead({ total: '12,50 €' })?.total).toBe(12.5);
expect(toReceiptRead({ total: '1.234,50' })?.total).toBe(1234.5);
expect(toReceiptRead({ total: '1,234.50' })?.total).toBe(1234.5);
expect(toReceiptRead({ total: 7, items: [{ name: 'Tea', price: '3,20' }] })?.items).toEqual([{ name: 'Tea', price: 3.2 }]);
});
it('turns a currency symbol or name into its code, and drops one nobody can convert', () => {
expect(toReceiptRead({ total: 5, currency: '€' })?.currency).toBe('EUR');
expect(toReceiptRead({ total: 5, currency: 'euros' })?.currency).toBe('EUR');
expect(toReceiptRead({ total: 5, currency: 'Lek' })?.currency).toBeNull();
});
it('keeps only a real calendar date', () => {
expect(toReceiptRead({ total: 5, date: '2026-02-28' })?.date).toBe('2026-02-28');
for (const date of ['2026-02-30', '28/02/2026', '2026-2-8', '']) expect(toReceiptRead({ total: 5, date })?.date).toBeNull();
});
it('leaves out a total that is not a positive amount, and lines without a name or price', () => {
const read = toReceiptRead({ merchant: 'Shop', total: 0, items: [{ name: '', price: 2 }, { name: 'Bread' }, 'x', { name: 'Milk', price: 1.1 }] });
expect(read).toEqual({ merchant: 'Shop', date: null, total: null, currency: null, items: [{ name: 'Milk', price: 1.1 }] });
});
it('is null when nothing usable came back', () => {
for (const raw of [null, undefined, 'text', [], {}, { total: 'n/a', merchant: ' ' }]) expect(toReceiptRead(raw)).toBeNull();
});
});
describe('the receipt prompt and schema', () => {
it('pins today so an ambiguous date is read towards it', () => {
expect(buildReceiptPrompt(new Date('2026-09-24T10:00:00Z'))).toContain('Today is 2026-09-24.');
});
it('wraps one receipt under the key the cloud clients read', () => {
expect(RECEIPT_LIST_JSON_SCHEMA.required).toEqual([RECEIPT_ROOT_KEY]);
expect(RECEIPT_LIST_JSON_SCHEMA.properties[RECEIPT_ROOT_KEY].items.required).toEqual(['merchant', 'date', 'total', 'currency', 'items']);
});
});
@@ -0,0 +1,55 @@
import { describe, it, expect, vi } from 'vitest';
import { HttpException } from '@nestjs/common';
import { ReceiptScanController } from '../../../src/nest/receipt-scan/receipt-scan.controller';
import type { ImportJobsService } from '../../../src/nest/booking-import/import-jobs.service';
import type { LlmParseService } from '../../../src/nest/llm-parse/llm-parse.service';
import type { DatabaseService } from '../../../src/nest/database/database.service';
import type { PermissionsService } from '../../../src/nest/permissions/permissions.service';
import type { AddonsService } from '../../../src/nest/addons/addons.service';
import type { User } from '../../../src/types';
const user = { id: 3, role: 'user' } as User;
const photo = (name = 'r.jpg') => ({ originalname: name, buffer: Buffer.from('x') } as Express.Multer.File);
function make(over: { readsImages?: boolean; trip?: unknown; addon?: boolean; allowed?: boolean } = {}) {
const startReceipt = vi.fn(() => 'job-1');
const checkPermission = vi.fn(() => over.allowed ?? true);
const c = new ReceiptScanController(
{ startReceipt } as unknown as ImportJobsService,
{ readsImages: vi.fn(async () => over.readsImages ?? true) } as unknown as LlmParseService,
{ canAccessTrip: vi.fn(() => ('trip' in over ? over.trip : { user_id: 5 })) } as unknown as DatabaseService,
{ checkPermission } as unknown as PermissionsService,
{ isAddonEnabled: vi.fn(() => over.addon ?? true) } as unknown as AddonsService,
);
return { c, startReceipt, checkPermission };
}
async function failure(fn: () => Promise<unknown>): Promise<[number, unknown]> {
try { await fn(); } catch (e) { expect(e).toBeInstanceOf(HttpException); return [(e as HttpException).getStatus(), (e as HttpException).getResponse()]; }
throw new Error('expected throw');
}
describe('ReceiptScanController', () => {
it('queues a photo and answers the job id', async () => {
const { c, startReceipt, checkPermission } = make();
await expect(c.scan(user, '9', photo())).resolves.toEqual({ jobId: 'job-1' });
expect(startReceipt).toHaveBeenCalledWith('9', expect.objectContaining({ originalname: 'r.jpg' }), 3);
expect(checkPermission).toHaveBeenCalledWith('budget_edit', 'user', 5, 3, true);
});
it('answers what the guards would: 404 for a trip out of reach or Costs off, 403 without budget_edit', async () => {
expect(await failure(() => make({ trip: undefined }).c.scan(user, '9', photo()))).toEqual([404, { error: 'Trip not found' }]);
expect(await failure(() => make({ addon: false }).c.scan(user, '9', photo()))).toEqual([404, { error: 'Costs addon is not enabled' }]);
const { c, startReceipt } = make({ allowed: false });
expect(await failure(() => c.scan(user, '9', photo()))).toEqual([403, { error: 'No permission' }]);
expect(startReceipt).not.toHaveBeenCalled();
});
it('refuses with 400 a missing file, a file that is not a photo, or a model that reads no images', async () => {
expect((await failure(() => make().c.scan(user, '9', undefined)))[0]).toBe(400);
expect((await failure(() => make().c.scan(user, '9', photo('r.pdf'))))[0]).toBe(400);
const { c, startReceipt } = make({ readsImages: false });
expect(await failure(() => c.scan(user, '9', photo()))).toEqual([400, { error: 'The configured AI model does not read photos' }]);
expect(startReceipt).not.toHaveBeenCalled();
});
});
+9
View File
@@ -1,4 +1,6 @@
import {
asLlmVision,
LLM_VISION_MODES,
adminUserCreateRequestSchema,
adminUserUpdateRequestSchema,
adminPermissionsRequestSchema,
@@ -106,6 +108,13 @@ describe('adminAddonUpdateRequestSchema', () => {
});
});
describe('asLlmVision', () => {
it('keeps the three modes and reads anything else as auto', () => {
for (const mode of LLM_VISION_MODES) expect(asLlmVision(mode)).toBe(mode);
for (const other of [undefined, null, '', 'ON', true, false, 1]) expect(asLlmVision(other)).toBe('auto');
});
});
describe('adminCollabFeaturesRequestSchema', () => {
it('accepts a single computed key', () => {
expect(adminCollabFeaturesRequestSchema.safeParse({ chat: false }).success).toBe(true);
+24
View File
@@ -80,6 +80,30 @@ export const adminAddonUpdateRequestSchema = z.object({
});
export type AdminAddonUpdateRequest = z.infer<typeof adminAddonUpdateRequestSchema>;
/**
* Whether the AI Parsing model is handed a photo, stored as `vision` on that
* addon's config. `auto` asks a local Ollama server what the model can do and
* means no for a cloud provider; `on` and `off` are the admin's word.
*/
export const LLM_VISION_MODES = ['auto', 'on', 'off'] as const;
export type LlmVision = (typeof LLM_VISION_MODES)[number];
/**
* The photos handed to a model that reads images. HEIC is not here: no provider
* TREK talks to reads it, and a browser picking from the camera roll hands over
* a JPEG anyway.
*/
export const LLM_PHOTO_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.webp'] as const;
/** GET /api/llm/capabilities: what the caller's AI model can be handed. */
export const llmCapabilitiesResponseSchema = z.object({ images: z.boolean() });
export type LlmCapabilitiesResponse = z.infer<typeof llmCapabilitiesResponseSchema>;
/** A stored or submitted value as a mode; anything else, a missing value included, is `auto`. */
export function asLlmVision(value: unknown): LlmVision {
return (LLM_VISION_MODES as readonly unknown[]).includes(value) ? (value as LlmVision) : 'auto';
}
// Fully partial — the client PUTs one computed key at a time.
export const adminCollabFeaturesRequestSchema = z.object({
chat: z.boolean().optional(),
+46
View File
@@ -0,0 +1,46 @@
import { z } from 'zod';
/**
* A photographed receipt, as the AI model read it for the Costs tab.
*
* It only pre-fills the expense editor: nothing is saved until the person saves
* the expense there, so every field may be missing and each is checked by eye.
* There is no category: which kind of spending it was is the person's call.
*/
export const receiptLineSchema = z.object({
name: z.string().min(1),
/** What the line cost in total, quantity included, in the receipt's currency. */
price: z.number(),
});
export type ReceiptLine = z.infer<typeof receiptLineSchema>;
export const receiptReadSchema = z.object({
/** The business, which becomes the expense name. */
merchant: z.string().min(1).nullable(),
/** YYYY-MM-DD. */
date: z
.string()
.regex(/^\d{4}-\d{2}-\d{2}$/)
.nullable(),
/** The grand total paid. */
total: z.number().positive().nullable(),
/** ISO 4217. */
currency: z
.string()
.regex(/^[A-Z]{3}$/)
.nullable(),
/** The itemized lines, which seed the Ticket split. */
items: z.array(receiptLineSchema),
});
export type ReceiptRead = z.infer<typeof receiptReadSchema>;
/** What a finished receipt scan job answers: the read, or null with the reason in `warnings`. */
export const receiptScanResultSchema = z.object({
receipt: receiptReadSchema.nullable(),
warnings: z.array(z.string()),
});
export type ReceiptScanResult = z.infer<typeof receiptScanResultSchema>;
/** POST /api/trips/:tripId/budget/receipt-scan: the background job reading the photo. */
export const receiptScanStartResponseSchema = z.object({ jobId: z.string().min(1) });
export type ReceiptScanStartResponse = z.infer<typeof receiptScanStartResponseSchema>;
+5
View File
@@ -626,6 +626,11 @@ const admin: TranslationStrings = {
'قراءة الزيارات والمسارات المسجَّلة من خادم Dawarich يربطه كل قارئ بنفسه',
'admin.addons.catalog.llm_parsing.name': 'التحليل بالذكاء الاصطناعي',
'admin.addons.catalog.llm_parsing.description': 'يقرأ الحجوزات التي يعجز المحلل المدمج عن فهمها، بنموذج ذكاء اصطناعي تختاره',
'admin.addons.llm.vision.auto': 'تلقائي',
'admin.addons.llm.vision.on': 'نعم',
'admin.addons.llm.vision.off': 'لا',
'admin.addons.llm.vision.hintLocal': 'يسأل الخيار «تلقائي» خادم Ollama عمّا إذا كان هذا النموذج يقرأ الصور.',
'admin.addons.llm.vision.hintCloud': 'يعني «تلقائي» «لا» للنموذج السحابي. اختر «نعم» إذا كان هذا النموذج يقرأ الصور.',
'admin.addons.enabled': 'مفعّل',
'admin.addons.disabled': 'معطّل',
'admin.addons.type.trip': 'رحلة',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'الإيصالات',
'costs.receiptsTitle': 'الإيصالات والفواتير',
'costs.attachReceipt': 'إرفاق إيصال / فاتورة',
'costs.scan.button': 'مسح إيصال',
'costs.scan.reading': 'جارٍ قراءة الإيصال…',
'costs.scan.review': 'مراجعة المصروف',
'costs.scan.nothingRead': 'تعذّرت قراءة أي إيصال من هذه الصورة.',
'costs.scan.failed': 'تعذّر بدء قراءة الإيصال.',
'costs.scan.title': 'مسح إيصال',
'costs.scan.accepted': 'صورة لإيصال واحد (JPG أو PNG أو WEBP، حتى 10 ميغابايت). يقرأها نموذج الذكاء الاصطناعي في الخلفية، ثم تراجع المصروف قبل حفظه.',
'costs.scan.dropHere': 'أفلت صورة الإيصال هنا، أو انقر لاختيار صورة أو التقاطها',
'costs.scan.dropActive': 'أفلت الصورة لمسحها',
'costs.scan.start': 'مسح',
'costs.scan.unsupported': 'يمكن مسح الصور فقط: JPG أو PNG أو WEBP.',
'costs.noReceipts': 'لا توجد إيصالات مرفقة',
'costs.deleteReceipt': 'حذف الإيصال',
'costs.viewReceipt': 'عرض الإيصال',
+1
View File
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropActive': 'أسقط الملفات للاستيراد',
'reservations.import.acceptedFormats':
'المقبول: EML، PDF، PKPass، HTML، TXT (بحد أقصى 10 ميغابايت لكل ملف، حتى 5 ملفات)',
'reservations.import.acceptedPhotos': 'يقرأ نموذج الذكاء الاصطناعي الصور (JPG وPNG وWEBP).',
'reservations.import.parsing': 'جارٍ معالجة الملفات…',
'reservations.import.previewHeading': 'تم العثور على {count} حجز/حجوزات',
'reservations.import.previewEmpty': 'تعذّر استخراج أي حجوزات من الملفات المُحمَّلة.',
+2 -2
View File
@@ -383,8 +383,8 @@ const settings: TranslationStrings = {
'settings.aiParsing.baseUrlHint': 'المكان الذي يعمل فيه النموذج — خادم Ollama محلي أو نقطة نهاية متوافقة مع OpenAI.',
'settings.aiParsing.apiKey': 'مفتاح API',
'settings.aiParsing.apiKeyHint': 'يُخزَّن مشفّرًا. اتركه فارغًا للإبقاء على المفتاح الحالي.',
'settings.aiParsing.multimodal': 'إرسال المستندات كصور',
'settings.aiParsing.multimodalHint': 'للنماذج القادرة على الرؤية — يرسل ملف PDF الأصلي بدلًا من النص المستخرج.',
'settings.aiParsing.multimodal': 'النموذج يقرأ الصور',
'settings.aiParsing.multimodalHint': 'فعّله لنموذج يقرأ الصور، ليمكن استيراد صورة أو مسحها.',
'settings.aiParsing.toast.saved': 'تم حفظ إعدادات الذكاء الاصطناعي',
'settings.aiParsing.toast.saveError': 'تعذّر حفظ إعدادات الذكاء الاصطناعي',
'settings.tabs.appearance': 'Appearance',
+5
View File
@@ -733,6 +733,11 @@ const admin: TranslationStrings = {
'admin.addons.catalog.llm_parsing.name': 'Süni intellektlə təhlil',
'admin.addons.catalog.llm_parsing.description':
'Daxili təhlilçinin oxuya bilmədiyi rezervasiyaları seçdiyiniz süni intellekt modeli ilə oxuyur',
'admin.addons.llm.vision.auto': 'Avtomatik',
'admin.addons.llm.vision.on': 'Bəli',
'admin.addons.llm.vision.off': 'Xeyr',
'admin.addons.llm.vision.hintLocal': 'Avtomatik rejim bu modelin şəkilləri oxuyub-oxumadığını Ollama serverindən soruşur.',
'admin.addons.llm.vision.hintCloud': 'Bulud modeli üçün Avtomatik "xeyr" deməkdir. Bu model şəkilləri oxuyursa, Bəli seç.',
'admin.addons.enabled': 'Aktivdir',
'admin.addons.disabled': 'Deaktivdir',
'admin.addons.type.trip': 'Səyahət',
+11
View File
@@ -186,6 +186,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Qəbzlər',
'costs.receiptsTitle': 'Qəbzlər və fakturalar',
'costs.attachReceipt': 'Qəbz / faktura əlavə et',
'costs.scan.button': 'Qəbzi skan et',
'costs.scan.reading': 'Qəbz oxunur…',
'costs.scan.review': 'Xərci yoxla',
'costs.scan.nothingRead': 'Bu fotodan qəbz oxunmadı.',
'costs.scan.failed': 'Qəbzin oxunmasına başlamaq mümkün olmadı.',
'costs.scan.title': 'Qəbzi skan et',
'costs.scan.accepted': 'Bir qəbzin fotosu (JPG, PNG və ya WEBP, 10 MB-a qədər). Süni intellekt modeli onu arxa planda oxuyur, sonra xərci saxlamazdan əvvəl yoxlayırsan.',
'costs.scan.dropHere': 'Qəbzin fotosunu bura burax və ya seçmək ya çəkmək üçün kliklə',
'costs.scan.dropActive': 'Skan etmək üçün fotonu burax',
'costs.scan.start': 'Skan et',
'costs.scan.unsupported': 'Yalnız foto skan edilə bilər: JPG, PNG və ya WEBP.',
'costs.noReceipts': 'Əlavə edilmiş qəbz yoxdur',
'costs.deleteReceipt': 'Qəbzi sil',
'costs.viewReceipt': 'Qəbzə bax',
+1
View File
@@ -179,6 +179,7 @@ const reservations: TranslationStrings = {
'İdxal etmək üçün faylları buraya buraxın',
'reservations.import.acceptedFormats':
'Qəbul edilən formatlar: EML, PDF, PKPass, HTML, TXT (hər biri maksimum 10 MB, 5 fayladək)',
'reservations.import.acceptedPhotos': 'Fotoları (JPG, PNG, WEBP) süni intellekt modeli oxuyur.',
'reservations.import.parsing': 'Fayllar təhlil edilir…',
'reservations.import.previewHeading':
'{count} rezervasiya tapıldı',
+2 -2
View File
@@ -497,9 +497,9 @@ const settings: TranslationStrings = {
'settings.aiParsing.apiKey': 'API açarı',
'settings.aiParsing.apiKeyHint':
'Şifrələnmiş şəkildə saxlanılır. Cari açarı saxlamaq üçün boş buraxın.',
'settings.aiParsing.multimodal': 'Sənədləri şəkil kimi göndər',
'settings.aiParsing.multimodal': 'Model şəkilləri oxuyur',
'settings.aiParsing.multimodalHint':
'Görmə qabiliyyətli modellər üçün — çıxarılmış mətn əvəzinə orijinal PDF-i göndərir.',
'Şəkilləri oxuyan model üçün aç: onda foto idxal oluna və ya skan edilə bilər.',
'settings.aiParsing.toast.saved': 'AI tənzimləmələri yadda saxlanıldı',
'settings.aiParsing.toast.saveError':
'AI tənzimləmələrini yadda saxlamaq mümkün olmadı',
+5
View File
@@ -618,6 +618,11 @@ const admin: TranslationStrings = {
'Leia visitas e rotas gravadas de uma instância do Dawarich que cada leitor conecta por conta própria',
'admin.addons.catalog.llm_parsing.name': 'Análise por IA',
'admin.addons.catalog.llm_parsing.description': 'Lê reservas que o analisador integrado não entende, usando um modelo de IA à sua escolha',
'admin.addons.llm.vision.auto': 'Automático',
'admin.addons.llm.vision.on': 'Sim',
'admin.addons.llm.vision.off': 'Não',
'admin.addons.llm.vision.hintLocal': 'Automático pergunta ao servidor Ollama se este modelo lê imagens.',
'admin.addons.llm.vision.hintCloud': 'Automático significa não para um modelo na nuvem. Escolha Sim se este modelo lê imagens.',
'admin.addons.enabled': 'Ativado',
'admin.addons.disabled': 'Desativado',
'admin.addons.type.trip': 'Viagem',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Comprovantes',
'costs.receiptsTitle': 'Comprovantes e notas',
'costs.attachReceipt': 'Anexar comprovante / nota',
'costs.scan.button': 'Escanear comprovante',
'costs.scan.reading': 'Lendo o comprovante…',
'costs.scan.review': 'Revisar a despesa',
'costs.scan.nothingRead': 'Não foi possível ler um comprovante nesta foto.',
'costs.scan.failed': 'Não foi possível começar a ler o comprovante.',
'costs.scan.title': 'Escanear comprovante',
'costs.scan.accepted': 'A foto de um único comprovante (JPG, PNG ou WEBP, até 10 MB). O modelo de IA lê em segundo plano e depois você confere a despesa antes de salvar.',
'costs.scan.dropHere': 'Solte aqui a foto do comprovante, ou clique para escolher ou tirar uma',
'costs.scan.dropActive': 'Solte a foto para escaneá-la',
'costs.scan.start': 'Escanear',
'costs.scan.unsupported': 'Só é possível escanear uma foto: JPG, PNG ou WEBP.',
'costs.noReceipts': 'Nenhum comprovante anexado',
'costs.deleteReceipt': 'Remover comprovante',
'costs.viewReceipt': 'Ver comprovante',
+1
View File
@@ -128,6 +128,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropHere': 'Solte os arquivos de confirmação de reserva aqui ou clique para selecionar',
'reservations.import.dropActive': 'Solte os arquivos para importar',
'reservations.import.acceptedFormats': 'Aceitos: EML, PDF, PKPass, HTML, TXT (máx. 10 MB cada, até 5 arquivos)',
'reservations.import.acceptedPhotos': 'As fotos (JPG, PNG, WEBP) são lidas pelo modelo de IA.',
'reservations.import.parsing': 'Analisando arquivos…',
'reservations.import.previewHeading': '{count} reserva(s) encontrada(s)',
'reservations.import.previewEmpty': 'Nenhuma reserva pôde ser extraída dos arquivos enviados.',
+2 -2
View File
@@ -398,9 +398,9 @@ const settings: TranslationStrings = {
'Onde o modelo é executado — um servidor Ollama local ou um endpoint compatível com OpenAI.',
'settings.aiParsing.apiKey': 'Chave de API',
'settings.aiParsing.apiKeyHint': 'Armazenada de forma criptografada. Deixe em branco para manter a chave atual.',
'settings.aiParsing.multimodal': 'Enviar documentos como imagens',
'settings.aiParsing.multimodal': 'O modelo lê imagens',
'settings.aiParsing.multimodalHint':
'Para modelos com capacidade de visão — envia o PDF original em vez do texto extraído.',
'Ative para um modelo que lê imagens, assim uma foto pode ser importada ou escaneada.',
'settings.aiParsing.toast.saved': 'Configurações de IA salvas',
'settings.aiParsing.toast.saveError': 'Não foi possível salvar as configurações de IA',
'settings.tabs.appearance': 'Appearance',
+5
View File
@@ -233,6 +233,11 @@ const admin: TranslationStrings = {
'Llegeix estades i recorreguts enregistrats d’una instància de Dawarich que cada lector connecta pel seu compte',
'admin.addons.catalog.llm_parsing.name': 'Anàlisi amb IA',
'admin.addons.catalog.llm_parsing.description': 'Llegeix reserves que l’analitzador integrat no entén, amb un model d’IA que triïs',
'admin.addons.llm.vision.auto': 'Automàtic',
'admin.addons.llm.vision.on': 'Sí',
'admin.addons.llm.vision.off': 'No',
'admin.addons.llm.vision.hintLocal': 'Automàtic pregunta al servidor Ollama si aquest model llegeix imatges.',
'admin.addons.llm.vision.hintCloud': 'Automàtic vol dir no per a un model al núvol. Tria Sí si aquest model llegeix imatges.',
'admin.addons.enabled': 'Actiu',
'admin.addons.disabled': 'Desactivat',
'admin.addons.type.trip': 'Viatge',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Rebuts',
'costs.receiptsTitle': 'Rebuts i factures',
'costs.attachReceipt': 'Adjuntar rebut / factura',
'costs.scan.button': 'Escanejar rebut',
'costs.scan.reading': 'Llegint el rebut…',
'costs.scan.review': 'Revisar la despesa',
'costs.scan.nothingRead': "No s'ha pogut llegir cap rebut d'aquesta foto.",
'costs.scan.failed': "No s'ha pogut començar a llegir el rebut.",
'costs.scan.title': 'Escanejar rebut',
'costs.scan.accepted': "Una foto d'un sol rebut (JPG, PNG o WEBP, fins a 10 MB). El model d'IA la llegeix en segon pla i després revises la despesa abans de desar-la.",
'costs.scan.dropHere': 'Deixa aquí la foto del rebut, o fes clic per triar-ne o fer-ne una',
'costs.scan.dropActive': 'Deixa anar la foto per escanejar-la',
'costs.scan.start': 'Escanejar',
'costs.scan.unsupported': 'Només es pot escanejar una foto: JPG, PNG o WEBP.',
'costs.noReceipts': 'Cap rebut adjunt',
'costs.deleteReceipt': 'Eliminar rebut',
'costs.viewReceipt': 'Veure rebut',
+1
View File
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropActive': 'Deixa anar els fitxers per importar',
'reservations.import.acceptedFormats':
'Acceptats: EML, PDF, PKPass, HTML, TXT (màx. 10 MB per fitxer, fins a 5 fitxers)',
'reservations.import.acceptedPhotos': "El model d'IA llegeix les fotos (JPG, PNG, WEBP).",
'reservations.import.parsing': 'Analitzant els fitxers…',
'reservations.import.previewHeading': '{count} reserva(es) trobada(es)',
'reservations.import.previewEmpty': "No s'han pogut extreure reserves dels fitxers pujats.",
+2 -2
View File
@@ -379,9 +379,9 @@ const settings: TranslationStrings = {
"On s'executa el model — un servidor local d'Ollama o un punt final compatible amb OpenAI.",
'settings.aiParsing.apiKey': "Clau de l'API",
'settings.aiParsing.apiKeyHint': "S'emmagatzema xifrada. Deixa-ho en blanc per mantenir la clau actual.",
'settings.aiParsing.multimodal': 'Enviar documents com a imatges',
'settings.aiParsing.multimodal': 'El model llegeix imatges',
'settings.aiParsing.multimodalHint':
'Per a models amb capacitats de visió — envia el PDF original en comptes del text extret.',
'Activa-ho per a un model que llegeix imatges, així es pot importar o escanejar una foto.',
'settings.aiParsing.toast.saved': "Ajustos d'IA desats",
'settings.aiParsing.toast.saveError': "No s'han pogut desar els ajustos d'IA",
'settings.tabs.appearance': 'Aparença',
+5
View File
@@ -608,6 +608,11 @@ const admin: TranslationStrings = {
'Načtěte návštěvy a zaznamenané trasy z instance Dawarich, kterou si každý čtenář připojí sám',
'admin.addons.catalog.llm_parsing.name': 'Analýza pomocí AI',
'admin.addons.catalog.llm_parsing.description': 'Přečte rezervace, které vestavěný analyzátor nezvládne, pomocí zvoleného modelu AI',
'admin.addons.llm.vision.auto': 'Automaticky',
'admin.addons.llm.vision.on': 'Ano',
'admin.addons.llm.vision.off': 'Ne',
'admin.addons.llm.vision.hintLocal': 'Automaticky se zeptá serveru Ollama, zda tento model čte obrázky.',
'admin.addons.llm.vision.hintCloud': 'Automaticky znamená u cloudového modelu ne. Zvolte Ano, pokud tento model čte obrázky.',
'admin.addons.catalog.roadtrip.name': 'Cesta autem',
'admin.addons.catalog.roadtrip.description': 'Plánujte jízdy se zastávkami – doby jízdy a časy příjezdu se přepočítají samy',
'admin.addons.enabled': 'Povoleno',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Účtenky',
'costs.receiptsTitle': 'Účtenky a faktury',
'costs.attachReceipt': 'Připojit účtenku / fakturu',
'costs.scan.button': 'Naskenovat účtenku',
'costs.scan.reading': 'Čtení účtenky…',
'costs.scan.review': 'Zkontrolovat výdaj',
'costs.scan.nothingRead': 'Z této fotografie se nepodařilo přečíst žádnou účtenku.',
'costs.scan.failed': 'Čtení účtenky se nepodařilo spustit.',
'costs.scan.title': 'Naskenovat účtenku',
'costs.scan.accepted': 'Fotografie jedné účtenky (JPG, PNG nebo WEBP, do 10 MB). Model AI ji přečte na pozadí a vy pak výdaj před uložením zkontrolujete.',
'costs.scan.dropHere': 'Přetáhněte sem fotografii účtenky, nebo klikněte a vyberte či pořiďte novou',
'costs.scan.dropActive': 'Pusťte fotografii pro naskenování',
'costs.scan.start': 'Skenovat',
'costs.scan.unsupported': 'Skenovat lze jen fotografii: JPG, PNG nebo WEBP.',
'costs.noReceipts': 'Žádné připojené účtenky',
'costs.deleteReceipt': 'Odebrat účtenku',
'costs.viewReceipt': 'Zobrazit účtenku',
+1
View File
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropActive': 'Pusťte soubory pro import',
'reservations.import.acceptedFormats':
'Přijímané formáty: EML, PDF, PKPass, HTML, TXT (max. 10 MB každý, až 5 souborů)',
'reservations.import.acceptedPhotos': 'Fotografie (JPG, PNG, WEBP) čte model AI.',
'reservations.import.parsing': 'Zpracování souborů…',
'reservations.import.previewHeading': 'Nalezeno {count} rezervace/í',
'reservations.import.previewEmpty': 'Z nahraných souborů se nepodařilo extrahovat žádné rezervace.',
+2 -2
View File
@@ -390,9 +390,9 @@ const settings: TranslationStrings = {
'settings.aiParsing.baseUrlHint': 'Kde model běží — místní server Ollama nebo koncový bod kompatibilní s OpenAI.',
'settings.aiParsing.apiKey': 'Klíč API',
'settings.aiParsing.apiKeyHint': 'Ukládá se šifrovaně. Ponechte prázdné pro zachování aktuálního klíče.',
'settings.aiParsing.multimodal': 'Odesílat dokumenty jako obrázky',
'settings.aiParsing.multimodal': 'Model čte obrázky',
'settings.aiParsing.multimodalHint':
'Pro modely se schopností zpracovat obraz — odešle původní PDF místo extrahovaného textu.',
'Zapněte pro model, který čte obrázky, aby šlo fotografii importovat nebo naskenovat.',
'settings.aiParsing.toast.saved': 'Nastavení AI uloženo',
'settings.aiParsing.toast.saveError': 'Nastavení AI se nepodařilo uložit',
'settings.tabs.appearance': 'Appearance',
+5
View File
@@ -624,6 +624,11 @@ const admin: TranslationStrings = {
'Aufenthalte und aufgezeichnete Routen aus einer Dawarich-Instanz lesen, die jeder selbst verbindet',
'admin.addons.catalog.llm_parsing.name': 'KI-Auswertung',
'admin.addons.catalog.llm_parsing.description': 'Liest Buchungen, an denen der eingebaute Parser scheitert, mit einem KI-Modell deiner Wahl',
'admin.addons.llm.vision.auto': 'Automatisch',
'admin.addons.llm.vision.on': 'Ja',
'admin.addons.llm.vision.off': 'Nein',
'admin.addons.llm.vision.hintLocal': 'Automatisch fragt den Ollama-Server, ob dieses Modell Bilder liest.',
'admin.addons.llm.vision.hintCloud': 'Automatisch bedeutet bei einem Cloud-Modell Nein. Wähle Ja, wenn dieses Modell Bilder liest.',
'admin.addons.enabled': 'Aktiviert',
'admin.addons.disabled': 'Deaktiviert',
'admin.addons.type.trip': 'Reise',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Belege',
'costs.receiptsTitle': 'Belege & Rechnungen',
'costs.attachReceipt': 'Beleg / Rechnung anhängen',
'costs.scan.button': 'Beleg scannen',
'costs.scan.reading': 'Beleg wird gelesen…',
'costs.scan.review': 'Ausgabe prüfen',
'costs.scan.nothingRead': 'Aus diesem Foto ließ sich kein Beleg lesen.',
'costs.scan.failed': 'Der Beleg konnte nicht eingelesen werden.',
'costs.scan.title': 'Beleg scannen',
'costs.scan.accepted': 'Ein Foto eines Belegs (JPG, PNG oder WEBP, bis 10 MB). Das KI-Modell liest es im Hintergrund, danach prüfst du die Ausgabe, bevor sie gespeichert wird.',
'costs.scan.dropHere': 'Belegfoto hier ablegen oder klicken, um eins auszuwählen oder aufzunehmen',
'costs.scan.dropActive': 'Foto ablegen, um es zu scannen',
'costs.scan.start': 'Scannen',
'costs.scan.unsupported': 'Nur Fotos lassen sich scannen: JPG, PNG oder WEBP.',
'costs.noReceipts': 'Keine Belege angehängt',
'costs.deleteReceipt': 'Beleg entfernen',
'costs.viewReceipt': 'Beleg ansehen',
+1
View File
@@ -132,6 +132,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropActive': 'Dateien zum Importieren ablegen',
'reservations.import.acceptedFormats':
'Akzeptiert: EML, PDF, PKPass, HTML, TXT (max. 10 MB pro Datei, bis zu 5 Dateien)',
'reservations.import.acceptedPhotos': 'Fotos (JPG, PNG, WEBP) liest das KI-Modell.',
'reservations.import.parsing': 'Dateien werden verarbeitet…',
'reservations.import.previewHeading': '{count} Reservierung(en) gefunden',
'reservations.import.previewEmpty': 'Aus den hochgeladenen Dateien konnten keine Reservierungen extrahiert werden.',
+2 -2
View File
@@ -401,9 +401,9 @@ const settings: TranslationStrings = {
'Wo das Modell läuft — ein lokaler Ollama-Server oder ein OpenAI-kompatibler Endpunkt.',
'settings.aiParsing.apiKey': 'API-Schlüssel',
'settings.aiParsing.apiKeyHint': 'Verschlüsselt gespeichert. Leer lassen, um den aktuellen Schlüssel zu behalten.',
'settings.aiParsing.multimodal': 'Dokumente als Bilder senden',
'settings.aiParsing.multimodal': 'Modell liest Bilder',
'settings.aiParsing.multimodalHint':
'Für Modelle mit Bildverständnis — sendet das Original-PDF statt extrahiertem Text.',
'Für ein Modell, das Bilder liest: Dann lässt sich ein Foto importieren oder scannen.',
'settings.aiParsing.toast.saved': 'KI-Einstellungen gespeichert',
'settings.aiParsing.toast.saveError': 'KI-Einstellungen konnten nicht gespeichert werden',
'settings.tabs.appearance': 'Erscheinungsbild',
+5
View File
@@ -663,6 +663,11 @@ const admin: TranslationStrings = {
'Read visits and recorded routes from a Dawarich instance each reader connects themselves',
'admin.addons.catalog.llm_parsing.name': 'AI Parsing',
'admin.addons.catalog.llm_parsing.description': 'Reads bookings the built-in parser cannot, using an AI model you choose',
'admin.addons.llm.vision.auto': 'Automatic',
'admin.addons.llm.vision.on': 'Yes',
'admin.addons.llm.vision.off': 'No',
'admin.addons.llm.vision.hintLocal': 'Automatic asks the Ollama server whether this model reads images.',
'admin.addons.llm.vision.hintCloud': 'Automatic means no for a cloud model. Choose Yes if this model reads images.',
'admin.addons.enabled': 'Enabled',
'admin.addons.disabled': 'Disabled',
'admin.addons.type.trip': 'Trip',
+11
View File
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
'costs.receipts': 'Receipts',
'costs.receiptsTitle': 'Receipts & Invoices',
'costs.attachReceipt': 'Attach receipt / invoice',
'costs.scan.button': 'Scan receipt',
'costs.scan.reading': 'Reading the receipt…',
'costs.scan.review': 'Review expense',
'costs.scan.nothingRead': 'No receipt could be read from this photo.',
'costs.scan.failed': 'Could not start reading the receipt.',
'costs.scan.title': 'Scan a receipt',
'costs.scan.accepted': 'A photo of one receipt (JPG, PNG or WEBP, up to 10 MB). The AI model reads it in the background, then you check the expense before it is saved.',
'costs.scan.dropHere': 'Drop the receipt photo here, or click to pick or take one',
'costs.scan.dropActive': 'Drop the photo to scan it',
'costs.scan.start': 'Scan',
'costs.scan.unsupported': 'Only a photo can be scanned: JPG, PNG or WEBP.',
'costs.noReceipts': 'No receipts attached',
'costs.deleteReceipt': 'Remove receipt',
'costs.viewReceipt': 'View receipt',
+1
View File
@@ -133,6 +133,7 @@ const reservations: TranslationStrings = {
'reservations.import.dropHere': 'Drop booking confirmation files here, or click to select',
'reservations.import.dropActive': 'Drop files to import',
'reservations.import.acceptedFormats': 'Accepted: EML, PDF, PKPass, HTML, TXT (max 10 MB each, up to 5 files)',
'reservations.import.acceptedPhotos': 'Photos (JPG, PNG, WEBP) are read by the AI model.',
'reservations.import.parsing': 'Parsing files…',
'reservations.import.previewHeading': '{count} reservation(s) found',
'reservations.import.previewEmpty': 'No reservations could be extracted from the uploaded files.',
+2 -2
View File
@@ -400,8 +400,8 @@ const settings: TranslationStrings = {
'settings.aiParsing.baseUrlHint': 'Where the model runs — a local Ollama server or an OpenAI-compatible endpoint.',
'settings.aiParsing.apiKey': 'API key',
'settings.aiParsing.apiKeyHint': 'Stored encrypted. Leave blank to keep the current key.',
'settings.aiParsing.multimodal': 'Send documents as images',
'settings.aiParsing.multimodalHint': 'For vision-capable models — sends the original PDF instead of extracted text.',
'settings.aiParsing.multimodal': 'Model reads images',
'settings.aiParsing.multimodalHint': 'Turn on for a model that reads images, so a photo can be imported or scanned.',
'settings.aiParsing.toast.saved': 'AI settings saved',
'settings.aiParsing.toast.saveError': 'Could not save AI settings',
'settings.tabs.appearance': 'Appearance',

Some files were not shown because too many files have changed in this diff Show More