mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
feat(llm-parse): read a photo, as a booking or as a receipt in Costs
Whether the AI Parsing model reads images is an admin setting now, and it lands with the two features that send a photo, so it has something to control. The setting is `vision: 'auto' | 'on' | 'off'` on the addon config. On Automatic a local Ollama is asked what the model can do (`/api/show`), and a cloud provider means no; Yes and No are applied as set. The old `multimodal` flag is not read: the admin screens wrote it false on every save, so a stored false meant nothing. The per-user toggle keeps its on/off meaning and is read by the extraction now. The lookup is server-side only: capped read, validated body, never throws, cached per server and model. GET /api/llm/capabilities answers it for the signed-in caller, and both photo pickers below ask it. The booking import takes JPG, PNG and WEBP when the model reads images. The photo goes as an image to every provider (image_url, an Anthropic image block, Ollama's native chat with `images`), capped at 1600px, and skips KDE Itinerary. A scanned PDF, which used to reach the model as empty text, has its first two pages drawn by pdf-parse and sent the same way. The rest is the import as it was: background job, review, the file attached to the booking, the linked cost. On a real qwen3.5:4b a train ticket came back as bare clock times until the router gave the model every type's field meanings, and its price as an object inside a string, which read as 4900; both are handled. Costs gets Scan receipt: a dialog laid out like the booking import's takes one photo, read on the import job queue, then reviewed in the existing expense editor pre-filled with merchant, total, currency, day and lines, the photo waiting in Receipts & Invoices to be uploaded on save. No table, no migration, no category guess, no document-type detection.
This commit is contained in:
@@ -54,6 +54,7 @@ import {
|
||||
type BookingImportPreviewResponse,
|
||||
type BookingImportConfirmResponse,
|
||||
type BookingImportMode,
|
||||
type ReceiptScanResult,
|
||||
type StorageAdminState,
|
||||
type StorageBackend,
|
||||
type StorageConfigPut,
|
||||
@@ -1475,7 +1476,7 @@ export const reservationsApi = {
|
||||
return postMultipart(`/trips/${tripId}/reservations/import/booking/async`, fd)
|
||||
},
|
||||
// Poll a background job — recovery path when a WebSocket push was missed.
|
||||
importJobStatus: (tripId: number | string, jobId: string): Promise<{ status: 'running' | 'done' | 'error'; done: number; total: number; result?: BookingImportPreviewResponse; error?: string }> =>
|
||||
importJobStatus: (tripId: number | string, jobId: string): Promise<{ status: 'running' | 'done' | 'error'; done: number; total: number; result?: BookingImportPreviewResponse | ReceiptScanResult; error?: string }> =>
|
||||
apiClient.get(`/trips/${tripId}/reservations/import/jobs/${jobId}`).then(r => r.data),
|
||||
}
|
||||
|
||||
|
||||
@@ -703,7 +703,7 @@ describe('AddonManager', () => {
|
||||
await user.click(screen.getByRole('button', { name: 'Save' }));
|
||||
await screen.findByText('Saved');
|
||||
expect(bodies[0]).toEqual({
|
||||
config: { provider: 'local', model: 'qwen3:8b', baseUrl: '', apiKey: '••••••••', multimodal: true },
|
||||
config: { provider: 'local', model: 'qwen3:8b', baseUrl: '', apiKey: '••••••••', vision: 'auto' },
|
||||
});
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Save' }));
|
||||
@@ -737,7 +737,7 @@ describe('AddonManager', () => {
|
||||
await screen.findByText('Saved');
|
||||
// The model is trimmed before it is stored, the key is sent verbatim
|
||||
expect(bodies[0]).toEqual({
|
||||
config: { provider: 'openai', model: 'mistral:7b', baseUrl: '', apiKey: 'sk-live', multimodal: false },
|
||||
config: { provider: 'openai', model: 'mistral:7b', baseUrl: '', apiKey: 'sk-live', vision: 'auto' },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -765,10 +765,40 @@ describe('AddonManager', () => {
|
||||
await screen.findByText('Saved');
|
||||
// The stale local base URL must not ride along to Anthropic — it would hijack the endpoint.
|
||||
expect(bodies[0]).toEqual({
|
||||
config: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001', baseUrl: '', apiKey: 'sk-ant-live', multimodal: false },
|
||||
config: { provider: 'anthropic', model: 'claude-haiku-4-5-20251001', baseUrl: '', apiKey: 'sk-ant-live', vision: 'auto' },
|
||||
});
|
||||
});
|
||||
|
||||
it('FE-ADMIN-ADDON-035: whether the model reads images is auto, on or off, kept as stored and saved as chosen', async () => {
|
||||
const user = userEvent.setup();
|
||||
const bodies: unknown[] = [];
|
||||
server.use(
|
||||
addonsRoute([llmAddon({ provider: 'openai', model: 'gpt-4.1-mini', baseUrl: '', apiKey: 'sk', vision: 'off' })]),
|
||||
http.put('/api/admin/addons/llm_parsing', async ({ request }) => {
|
||||
bodies.push(await request.json());
|
||||
return HttpResponse.json({ success: true });
|
||||
}),
|
||||
);
|
||||
render(<><ToastContainer /><AddonManager /></>);
|
||||
|
||||
// Stored off stays off; a cloud provider is told what Automatic means for it.
|
||||
await user.click(await screen.findByRole('button', { name: 'No' }));
|
||||
expect(screen.getByText('Automatic means no for a cloud model. Choose Yes if this model reads images.')).toBeInTheDocument();
|
||||
await user.click(screen.getByRole('button', { name: 'Yes' }));
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Save' }));
|
||||
await screen.findByText('Saved');
|
||||
expect(bodies[0]).toMatchObject({ config: { provider: 'openai', vision: 'on' } });
|
||||
expect(bodies[0]).not.toHaveProperty('config.multimodal');
|
||||
});
|
||||
|
||||
it('FE-ADMIN-ADDON-036: with the local provider the hint says Automatic asks the server', async () => {
|
||||
server.use(addonsRoute([llmAddon({ provider: 'local', model: 'qwen3.5:4b' })]), modelsRoute([]));
|
||||
render(<AddonManager />);
|
||||
expect(await screen.findByText('Automatic asks the Ollama server whether this model reads images.')).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: 'Automatic' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('FE-ADMIN-ADDON-027: an error frame in the pull stream aborts the pull and is reported', async () => {
|
||||
const user = userEvent.setup();
|
||||
server.use(
|
||||
|
||||
@@ -6,6 +6,7 @@ import { useIsDark } from '../../hooks/useIsDark'
|
||||
import { useToast } from '../shared/Toast'
|
||||
import { Puzzle, ListChecks, Wallet, FileText, CalendarDays, Globe, Briefcase, Image, Terminal, Link2, Compass, BookOpen, MessageCircle, StickyNote, BarChart3, Sparkles, Luggage, Plane, Server, Cloud, Bookmark, Users, Loader2 } from 'lucide-react'
|
||||
import CustomSelect from '../shared/CustomSelect'
|
||||
import { asLlmVision, LLM_VISION_MODES, type LlmVision } from '@trek/shared'
|
||||
import EmptyState from '../shared/EmptyState'
|
||||
import DawarichIcon from '../shared/DawarichIcon'
|
||||
import AirTrailIcon from '../shared/AirTrailIcon'
|
||||
@@ -371,12 +372,14 @@ const RECOMMENDED_MODELS: { id: string; label: string; note: string; recommended
|
||||
* provider, it also lists installed Ollama models and can pull NuExtract models.
|
||||
*/
|
||||
function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
const { t } = useTranslation()
|
||||
const toast = useToast()
|
||||
const cfg = (addon.config ?? {}) as Record<string, unknown>
|
||||
const [provider, setProvider] = useState<string>((cfg.provider as string) ?? 'local')
|
||||
const [model, setModel] = useState<string>((cfg.model as string) ?? '')
|
||||
const [baseUrl, setBaseUrl] = useState<string>((cfg.baseUrl as string) ?? '')
|
||||
const [apiKey, setApiKey] = useState<string>((cfg.apiKey as string) ?? '')
|
||||
const [vision, setVision] = useState<LlmVision>(asLlmVision(cfg.vision))
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
// Local-provider model management.
|
||||
@@ -438,7 +441,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
setSaving(true)
|
||||
try {
|
||||
// Send the masked sentinel unchanged so the server keeps the stored key.
|
||||
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: provider === 'anthropic' ? '' : baseUrl.trim(), apiKey, multimodal: cfg.multimodal === true } })
|
||||
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: provider === 'anthropic' ? '' : baseUrl.trim(), apiKey, vision } })
|
||||
toast.success('Saved')
|
||||
} catch {
|
||||
toast.error('Failed to save')
|
||||
@@ -455,6 +458,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
{ value: 'openai', label: 'OpenAI', icon: <Cloud size={14} /> },
|
||||
{ value: 'anthropic', label: 'Anthropic', icon: <Sparkles size={14} /> },
|
||||
]
|
||||
const visionOptions = LLM_VISION_MODES.map(value => ({ value, label: t(`admin.addons.llm.vision.${value}`) }))
|
||||
|
||||
/* Lives in the tile's shelf like the collab toggles, so everything is caption-
|
||||
sized and single-column — the band this used to be had a whole page width. */
|
||||
@@ -486,6 +490,14 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
<input autoComplete="off" className={fieldCls} value={model} onChange={e => setModel(e.target.value)} placeholder={provider === 'anthropic' ? 'claude-opus-4-8' : provider === 'openai' ? 'gpt-4o' : 'select or pull below'} />
|
||||
</label>
|
||||
|
||||
<div>
|
||||
<span className={labelCls}>{t('settings.aiParsing.multimodal')}</span>
|
||||
<CustomSelect value={vision} onChange={v => setVision(asLlmVision(v))} options={visionOptions} />
|
||||
<p className="mt-1 text-caption text-content-faint">
|
||||
{t(provider === 'local' ? 'admin.addons.llm.vision.hintLocal' : 'admin.addons.llm.vision.hintCloud')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Local model management (Ollama) */}
|
||||
{provider === 'local' && (
|
||||
<div className="space-y-2 rounded-lg border border-edge-secondary bg-surface p-2.5">
|
||||
|
||||
@@ -304,3 +304,44 @@ describe('BackgroundTasksWidget: the mark on a finished job (#2477)', () => {
|
||||
expect(baseElement.querySelector('svg.lucide-alert-triangle')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('BackgroundTasksWidget — a receipt scanned from Costs', () => {
|
||||
const RECEIPT = { merchant: 'Café', date: '2026-09-20', total: 12.5, currency: 'EUR', items: [] }
|
||||
|
||||
it('FE-W4BGT-025: says it is reading the receipt while the job runs', () => {
|
||||
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
|
||||
render(<BackgroundTasksWidget />)
|
||||
expect(screen.getByText('Reading the receipt…')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-W4BGT-026: import:done with a read receipt offers to review the expense, which opens the trip', () => {
|
||||
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
|
||||
render(<BackgroundTasksWidget />)
|
||||
const handler = vi.mocked(addListener).mock.calls[0][0] as WsHandler
|
||||
|
||||
act(() => { handler({ type: 'import:done', jobId: 'j1', tripId: 't1', result: { receipt: RECEIPT, warnings: [] } }) })
|
||||
expect(useBackgroundTasksStore.getState().tasks[0].receipt).toEqual(RECEIPT)
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Review expense' }))
|
||||
expect(useBackgroundTasksStore.getState().tasks[0].reviewRequested).toBe(true)
|
||||
expect(navigate).toHaveBeenCalledWith('/trips/t1')
|
||||
})
|
||||
|
||||
it('FE-W4BGT-027: a photo nothing could be read from says so, with its warning and no AI retry', async () => {
|
||||
vi.mocked(healthApi.features).mockResolvedValue({ bookingImport: true, aiParsing: true })
|
||||
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', receipt: null, warnings: ['bill.jpg: no receipt could be read'], sourceFiles: [photo] })] })
|
||||
render(<BackgroundTasksWidget />)
|
||||
expect(screen.getByText('No receipt could be read from this photo.')).toBeInTheDocument()
|
||||
expect(screen.getByText('bill.jpg: no receipt could be read')).toBeInTheDocument()
|
||||
await waitFor(() => expect(healthApi.features).toHaveBeenCalled())
|
||||
expect(screen.queryByRole('button', { name: 'Try AI parsing' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-W4BGT-028: the reload reconcile reads a receipt job\'s status too', async () => {
|
||||
vi.mocked(reservationsApi.importJobStatus).mockResolvedValue({ status: 'done', done: 1, total: 1, result: { receipt: RECEIPT, warnings: [] } as never })
|
||||
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', items: undefined })] })
|
||||
render(<BackgroundTasksWidget />)
|
||||
expect(await screen.findByRole('button', { name: 'Review expense' })).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -6,7 +6,18 @@ import { useTranslation } from '../../i18n'
|
||||
import { addListener, removeListener } from '../../api/websocket'
|
||||
import { reservationsApi, healthApi } from '../../api/client'
|
||||
import { saveImportFiles } from '../../db/offlineDb'
|
||||
import { useBackgroundTasksStore, type BackgroundImportTask } from '../../store/backgroundTasksStore'
|
||||
import { useBackgroundTasksStore, taskFoundSomething, type BackgroundImportTask } from '../../store/backgroundTasksStore'
|
||||
import type { ReceiptRead } from '@trek/shared'
|
||||
|
||||
/**
|
||||
* One reading of a finished job's result, wherever it arrives from (the push,
|
||||
* the reload reconcile, the poll): a booking parse answers `items`, a receipt
|
||||
* scan `receipt`.
|
||||
*/
|
||||
function settleJob(id: string, tripId: string, result: unknown): void {
|
||||
const r = result as { items?: unknown[]; warnings?: string[]; receipt?: ReceiptRead | null } | undefined
|
||||
useBackgroundTasksStore.getState().setDone(id, tripId, (r?.items ?? []) as never, r?.warnings ?? [], r?.receipt)
|
||||
}
|
||||
|
||||
/**
|
||||
* Global, route-independent widget (bottom-right) that tracks background booking
|
||||
@@ -20,7 +31,6 @@ export default function BackgroundTasksWidget() {
|
||||
const navigate = useNavigate()
|
||||
const tasks = useBackgroundTasksStore((s) => s.tasks)
|
||||
const setProgress = useBackgroundTasksStore((s) => s.setProgress)
|
||||
const setDone = useBackgroundTasksStore((s) => s.setDone)
|
||||
const setError = useBackgroundTasksStore((s) => s.setError)
|
||||
const requestReview = useBackgroundTasksStore((s) => s.requestReview)
|
||||
const dismiss = useBackgroundTasksStore((s) => s.dismiss)
|
||||
@@ -66,7 +76,7 @@ export default function BackgroundTasksWidget() {
|
||||
reservationsApi
|
||||
.importJobStatus(task.tripId, task.id)
|
||||
.then((s) => {
|
||||
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
|
||||
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
|
||||
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
|
||||
else setProgress(task.id, task.tripId, s.done, s.total)
|
||||
})
|
||||
@@ -87,14 +97,12 @@ export default function BackgroundTasksWidget() {
|
||||
const tripId = String(e.tripId ?? '')
|
||||
if (!id) return
|
||||
if (type === 'import:progress') setProgress(id, tripId, Number(e.done ?? 0), Number(e.total ?? 1))
|
||||
else if (type === 'import:done') {
|
||||
const result = e.result as { items?: unknown[]; warnings?: string[] } | undefined
|
||||
setDone(id, tripId, (result?.items ?? []) as never, result?.warnings ?? [])
|
||||
} else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
|
||||
else if (type === 'import:done') settleJob(id, tripId, e.result)
|
||||
else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
|
||||
}
|
||||
addListener(handler)
|
||||
return () => removeListener(handler)
|
||||
}, [setProgress, setDone, setError])
|
||||
}, [setProgress, setError])
|
||||
|
||||
// Backstop: poll jobs whose state we still need — running ones (in case a WebSocket push
|
||||
// was missed) and a restored 'done' task whose items haven't been re-fetched yet (so a
|
||||
@@ -107,7 +115,7 @@ export default function BackgroundTasksWidget() {
|
||||
reservationsApi
|
||||
.importJobStatus(task.tripId, task.id)
|
||||
.then((s) => {
|
||||
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
|
||||
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
|
||||
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
|
||||
else setProgress(task.id, task.tripId, s.done, s.total)
|
||||
})
|
||||
@@ -120,7 +128,7 @@ export default function BackgroundTasksWidget() {
|
||||
}
|
||||
}, 5000)
|
||||
return () => clearInterval(iv)
|
||||
}, [tasks, setProgress, setDone, setError, t])
|
||||
}, [tasks, setProgress, setError, t])
|
||||
|
||||
if (tasks.length === 0) return null
|
||||
|
||||
@@ -141,11 +149,11 @@ export default function BackgroundTasksWidget() {
|
||||
>
|
||||
<div style={{ flexShrink: 0, marginTop: 1 }}>
|
||||
{(task.status === 'running' || (task.status === 'done' && task.items === undefined)) && <Loader2 size={16} className="animate-spin" color="var(--accent)" />}
|
||||
{task.status === 'done' && task.items !== undefined && task.items.length > 0 && <CheckCircle2 size={16} color="var(--success)" />}
|
||||
{task.status === 'done' && task.items !== undefined && taskFoundSomething(task) && <CheckCircle2 size={16} color="var(--success)" />}
|
||||
{/* A parse that found nothing is not a success, whatever the job
|
||||
status says: the same card used to show a green tick next to
|
||||
"no reservations could be extracted" (#2477). */}
|
||||
{task.status === 'done' && task.items?.length === 0 && <AlertTriangle size={16} color="var(--warning)" />}
|
||||
{task.status === 'done' && task.items !== undefined && !taskFoundSomething(task) && <AlertTriangle size={16} color="var(--warning)" />}
|
||||
{task.status === 'error' && <AlertCircle size={16} color="var(--danger)" />}
|
||||
</div>
|
||||
|
||||
@@ -156,7 +164,7 @@ export default function BackgroundTasksWidget() {
|
||||
|
||||
{task.status === 'running' && (
|
||||
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
|
||||
{t('reservations.import.parsing')}
|
||||
{t(task.kind === 'costs' ? 'costs.scan.reading' : 'reservations.import.parsing')}
|
||||
{task.total > 1 ? ` · ${task.done}/${task.total}` : ''}
|
||||
</div>
|
||||
)}
|
||||
@@ -165,14 +173,14 @@ export default function BackgroundTasksWidget() {
|
||||
task.items === undefined ? (
|
||||
// Restored from a reload; items are being re-fetched (see the poll backstop).
|
||||
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>{t('reservations.import.parsing')}</div>
|
||||
) : task.items.length > 0 ? (
|
||||
) : taskFoundSomething(task) ? (
|
||||
<div>
|
||||
<button type="button"
|
||||
onClick={() => review(task)}
|
||||
className="bg-accent text-accent-text"
|
||||
style={{ marginTop: 4, border: 'none', borderRadius: 8, padding: '4px 12px', fontSize: 'calc(11.5px * var(--fs-scale-caption, 1))', fontWeight: 600, cursor: 'pointer', fontFamily: 'inherit' }}
|
||||
>
|
||||
{t('common.import')}
|
||||
{t(task.kind === 'costs' ? 'costs.scan.review' : 'common.import')}
|
||||
</button>
|
||||
{/* A partly-understood import warns too. Warnings used to be
|
||||
shown only when nothing at all was found, so anything the
|
||||
@@ -188,14 +196,14 @@ export default function BackgroundTasksWidget() {
|
||||
) : (
|
||||
<div>
|
||||
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
|
||||
{t('reservations.import.previewEmpty')}
|
||||
{t(task.kind === 'costs' ? 'costs.scan.nothingRead' : 'reservations.import.previewEmpty')}
|
||||
{(task.warnings?.length ?? 0) > 0 && (
|
||||
<div style={{ color: '#b45309', marginTop: 3, whiteSpace: 'pre-wrap', wordBreak: 'break-word', maxHeight: 96, overflowY: 'auto' }}>
|
||||
{task.warnings!.join('\n')}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{aiParsing && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
|
||||
{aiParsing && task.kind !== 'costs' && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
|
||||
<button type="button"
|
||||
onClick={() => retryWithAi(task)}
|
||||
disabled={retrying === task.id}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, paidByUser, payerSum, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitCents, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
|
||||
import { calculateTicketShares, expenseEditorFor, newExpenseSeed, receiptToPrefill, finalBudgetFor, finalBudgetSources, hasTicketSplit, paidByUser, payerSum, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitCents, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
|
||||
|
||||
describe('splitCents', () => {
|
||||
it('splits evenly when it divides cleanly', () => {
|
||||
@@ -310,3 +310,55 @@ describe('finalBudgetSources', () => {
|
||||
.toEqual({ fronted: [], moved: [], outstanding: [] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('newExpenseSeed', () => {
|
||||
it('starts a plain new expense on the base currency, today, with nothing to attach', () => {
|
||||
expect(newExpenseSeed(undefined, 'eur', [1, 2], '2026-09-24')).toEqual({
|
||||
currency: 'EUR', day: '2026-09-24', total: '', ticketItems: [], receiptFiles: [],
|
||||
})
|
||||
})
|
||||
|
||||
it('reads a booking price in the base currency, padded to its decimals', () => {
|
||||
expect(newExpenseSeed({ amount: 4.9 }, 'EUR', [], '2026-09-24').total).toBe('4.90')
|
||||
})
|
||||
|
||||
it('seeds a scanned receipt: its currency, day, lines shared by everyone, and the photo', () => {
|
||||
const photo = new File(['x'], 'bill.jpg')
|
||||
const seed = newExpenseSeed(
|
||||
{ amount: 1500, currency: 'jpy', date: '2026-09-20', lines: [{ name: 'Ramen', price: 1200 }, { name: 'Tea', price: 300 }], receiptFiles: [photo] },
|
||||
'EUR', [1, 2], '2026-09-24',
|
||||
)
|
||||
expect(seed).toMatchObject({ currency: 'JPY', day: '2026-09-20', total: '1500', receiptFiles: [photo] })
|
||||
expect(seed.ticketItems.map(i => [i.name, i.price, [...i.participants]])).toEqual([['Ramen', '1200', [1, 2]], ['Tea', '300', [1, 2]]])
|
||||
})
|
||||
})
|
||||
|
||||
describe('receiptToPrefill', () => {
|
||||
it('turns a read receipt into an expense prefill, leaving out what was not read', () => {
|
||||
const photo = new File(['x'], 'bill.jpg')
|
||||
expect(receiptToPrefill({ merchant: 'Café', date: null, total: 12.5, currency: 'EUR', items: [] }, [photo])).toEqual({
|
||||
name: 'Café', amount: 12.5, currency: 'EUR', date: undefined, lines: [], receiptFiles: [photo],
|
||||
})
|
||||
expect(receiptToPrefill({ merchant: null, date: '2026-09-01', total: null, currency: null, items: [] }, [])).toMatchObject({
|
||||
name: undefined, amount: undefined, currency: undefined, date: '2026-09-01',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('expenseEditorFor', () => {
|
||||
const closeBooking = () => {}
|
||||
const closeReceipt = () => {}
|
||||
|
||||
it('is nothing when neither opener asks', () => {
|
||||
expect(expenseEditorFor(null, closeBooking, null, closeReceipt)).toBeNull()
|
||||
})
|
||||
|
||||
it('opens a scanned receipt as a new expense', () => {
|
||||
expect(expenseEditorFor(null, closeBooking, { name: 'Café' }, closeReceipt)).toEqual({ key: 'receipt', editing: null, prefill: { name: 'Café' }, close: closeReceipt })
|
||||
})
|
||||
|
||||
it('lets the booking win, since it is what was just clicked', () => {
|
||||
expect(expenseEditorFor({ editing: null, prefill: { amount: 3 } }, closeBooking, { name: 'Café' }, closeReceipt))
|
||||
.toEqual({ key: 'booking', editing: null, prefill: { amount: 3 }, close: closeBooking })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -11,8 +11,10 @@
|
||||
* Amounts are the raw input strings, parsed on use (same as customAmounts).
|
||||
*/
|
||||
|
||||
import type { BudgetParticipantFinal } from '@trek/shared'
|
||||
import { currencyDecimals } from '../../utils/formatters'
|
||||
import type { BudgetParticipantFinal, ReceiptRead } from '@trek/shared'
|
||||
import { amountToInputString, currencyDecimals } from '../../utils/formatters'
|
||||
import type { ExpensePrefill } from './CostsPanel'
|
||||
import type { BudgetItem } from '../../types'
|
||||
|
||||
// The split and receipt fields guard their own precision on every keystroke, so the
|
||||
// guard has to follow the currency: a three-decimal one (KWD, BHD, …) seeds three
|
||||
@@ -264,3 +266,59 @@ export function calculateTicketShares(items: TicketItem[]): { shares: Record<num
|
||||
|
||||
return { shares: finalShares, total: totalCents / 100 }
|
||||
}
|
||||
|
||||
/**
|
||||
* Where a new expense's editor starts from a prefill: a booking's price, or a
|
||||
* scanned receipt, which also brings its currency, day, lines and photo. Both
|
||||
* editors (the desktop modal and the phone sheet) seed from this one place.
|
||||
*
|
||||
* A prefill without a currency is in `base`, which is also what the currency
|
||||
* field starts on. Receipt lines seed the Ticket split, shared by everyone like
|
||||
* a line added by hand, while the split itself stays Equally until the person
|
||||
* switches.
|
||||
*/
|
||||
export function newExpenseSeed(prefill: ExpensePrefill | undefined, base: string, peopleIds: number[], today: string) {
|
||||
const currency = (prefill?.currency || base).toUpperCase()
|
||||
return {
|
||||
currency,
|
||||
day: prefill?.date || today,
|
||||
total: prefill?.amount != null ? amountToInputString(prefill.amount, currency) : '',
|
||||
ticketItems: (prefill?.lines ?? []).map((line, i): TicketItem => ({
|
||||
id: `receipt-${i}`,
|
||||
name: line.name,
|
||||
price: amountToInputString(line.price, currency),
|
||||
participants: new Set(peopleIds),
|
||||
})),
|
||||
receiptFiles: prefill?.receiptFiles ?? [],
|
||||
}
|
||||
}
|
||||
|
||||
/** A scanned receipt as the prefill of a new expense, with its photo to attach on save. */
|
||||
export function receiptToPrefill(receipt: ReceiptRead, photos: File[]): ExpensePrefill {
|
||||
return {
|
||||
name: receipt.merchant ?? undefined,
|
||||
amount: receipt.total ?? undefined,
|
||||
currency: receipt.currency ?? undefined,
|
||||
date: receipt.date ?? undefined,
|
||||
lines: receipt.items,
|
||||
receiptFiles: photos,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The expense editor a trip page shows, if any. Two things open it: a booking's
|
||||
* Costs block (edit that expense, or start one from the booking's price) and a
|
||||
* receipt scanned from Costs. The booking wins, since it is what the person just
|
||||
* clicked; `key` remounts the editor when one hands over to the other, so each
|
||||
* starts from its own seed.
|
||||
*/
|
||||
export function expenseEditorFor(
|
||||
booking: { editing: BudgetItem | null; prefill?: ExpensePrefill } | null,
|
||||
closeBooking: () => void,
|
||||
receipt: ExpensePrefill | null,
|
||||
closeReceipt: () => void,
|
||||
): { key: string; editing: BudgetItem | null; prefill?: ExpensePrefill; close: () => void } | null {
|
||||
if (booking) return { key: 'booking', editing: booking.editing, prefill: booking.prefill, close: closeBooking }
|
||||
if (receipt) return { key: 'receipt', editing: null, prefill: receipt, close: closeReceipt }
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -1625,6 +1625,33 @@ describe('CostsPanel — expense modal', () => {
|
||||
expect(onSaved).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('FE-W5COSTS-081: offers Scan receipt beside Add expense only when the AI model reads images', async () => {
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })))
|
||||
const { unmount } = render(<CostsPanel tripId={1} tripMembers={tripMembers} />)
|
||||
fireEvent.click((await screen.findAllByRole('button', { name: 'Scan receipt' }))[0])
|
||||
expect(screen.getByRole('dialog', { name: 'Scan a receipt' })).toBeInTheDocument()
|
||||
unmount()
|
||||
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
|
||||
render(<CostsPanel tripId={1} tripMembers={tripMembers} />)
|
||||
await screen.findAllByRole('button', { name: 'Add expense' })
|
||||
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-W5COSTS-080: a scanned receipt opens in its own currency and day, with its photo waiting to be attached', () => {
|
||||
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
render(
|
||||
<ExpenseModal tripId={1} base="EUR" people={tripMembers} me={1} editing={null}
|
||||
prefill={{ name: 'Sushi Dai', amount: 4200, currency: 'JPY', date: '2026-09-20', lines: [{ name: 'Omakase', price: 4200 }], receiptFiles: [photo] }}
|
||||
onClose={() => {}} onSaved={() => {}} />
|
||||
)
|
||||
|
||||
expect(screen.getByDisplayValue('Sushi Dai')).toBeInTheDocument()
|
||||
// Yen has no minor unit, so the amount is not padded.
|
||||
expect(screen.getByDisplayValue('4200')).toBeInTheDocument()
|
||||
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
// The mobile sheet already dates a new expense by the traveller's own clock;
|
||||
// the desktop modal filed it under the UTC day, so the same expense entered
|
||||
// late in Tokyo or early in Los Angeles landed on a different day per surface.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Fragment, useState, useEffect, useMemo, useCallback } from 'react'
|
||||
import { useSearchParams } from 'react-router'
|
||||
import { ArrowDown, ArrowUp, BarChart3, Plus, Search, ArrowRight, ArrowLeftRight, Check, RotateCcw, Pencil, Trash2, AlertCircle, Download, StickyNote, ChevronDown, Receipt, Paperclip } from 'lucide-react'
|
||||
import { ArrowDown, ArrowUp, BarChart3, Plus, Search, ArrowRight, ArrowLeftRight, Check, RotateCcw, Pencil, Trash2, AlertCircle, Download, StickyNote, ChevronDown, Receipt, Paperclip, ScanLine } from 'lucide-react'
|
||||
import { useTripStore } from '../../store/tripStore'
|
||||
import { useAuthStore } from '../../store/authStore'
|
||||
import { useSettingsStore } from '../../store/settingsStore'
|
||||
@@ -19,11 +19,13 @@ import Modal from '../shared/Modal'
|
||||
import CustomSelect from '../shared/CustomSelect'
|
||||
import { CustomDatePicker } from '../shared/CustomDateTimePicker'
|
||||
import { localToday } from '../Planner/today'
|
||||
import { useReceiptScan } from './useReceiptScan'
|
||||
import { ReceiptScanModal } from './ReceiptScanModal'
|
||||
import { SYMBOLS, currenciesWith, SPLIT_COLORS } from './BudgetPanel.constants'
|
||||
import { amountPattern, calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, NOTE_MAX, paidByUser, payersBalanced, readTicketItems, readUserNote, rebalancePayers, settlementDate, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
|
||||
import { amountPattern, calculateTicketShares, finalBudgetFor, finalBudgetSources, hasTicketSplit, NOTE_MAX, paidByUser, payersBalanced, readTicketItems, newExpenseSeed, readUserNote, rebalancePayers, settlementDate, splitEqualShares, writeTicketItems, type TicketItem } from './CostsPanel.helpers'
|
||||
import { COST_CATEGORY_LIST, catMeta } from './costsCategories'
|
||||
import { ReceiptPreviewModal } from './ReceiptPreviewModal'
|
||||
import type { BudgetParticipantFinal, BudgetUnconverted } from '@trek/shared'
|
||||
import type { BudgetParticipantFinal, BudgetUnconverted, ReceiptLine } from '@trek/shared'
|
||||
import type { BudgetItem, BudgetItemReceipt } from '../../types'
|
||||
import type { TripMember } from './BudgetPanelMemberChips'
|
||||
import GuestBadge from '../shared/GuestBadge'
|
||||
@@ -83,6 +85,7 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
|
||||
const me = useAuthStore(s => s.user?.id ?? -1)
|
||||
const can = useCanDo()
|
||||
const canEdit = can('budget_edit', trip)
|
||||
const receiptScan = useReceiptScan(tripId, canEdit)
|
||||
const toast = useToast()
|
||||
const { t, locale } = useTranslation()
|
||||
const isMobile = useIsMobile()
|
||||
@@ -387,7 +390,8 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
|
||||
<div style={{ maxWidth: '100%', margin: '0 auto' }}>
|
||||
<CostsToolbar dateMeta={dateMeta} people={people} me={me} colorFor={colorFor}
|
||||
canEdit={canEdit} canSettle={(settlement?.flows || []).length > 0}
|
||||
onSettleAll={settleAll} onAddExpense={() => { setEditing(null); setModalOpen(true) }} />
|
||||
onSettleAll={settleAll} onAddExpense={() => { setEditing(null); setModalOpen(true) }}
|
||||
onScanReceipt={receiptScan.offered ? receiptScan.open : undefined} />
|
||||
|
||||
{/* ── Summary cards ── */}
|
||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(4, 1fr)', gap: 16, marginBottom: 36 }} className="costs-summary">
|
||||
@@ -509,6 +513,8 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
|
||||
</div>
|
||||
</div>)}
|
||||
|
||||
<ReceiptScanModal scan={receiptScan} />
|
||||
|
||||
{modalOpen && (
|
||||
<ExpenseModal tripId={tripId} base={base} people={people} me={me} editing={editing}
|
||||
onClose={() => setModalOpen(false)}
|
||||
@@ -634,6 +640,13 @@ export default function CostsPanel({ tripId, tripMembers = [] }: CostsPanelProps
|
||||
<Plus size={17} /> {t('costs.addExpense')}
|
||||
</button>
|
||||
)}
|
||||
{receiptScan.offered && (
|
||||
<button type="button" onClick={receiptScan.open}
|
||||
style={{ marginTop: 8, width: '100%', display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 8, background: 'transparent', border: '1px solid rgba(255,255,255,0.16)', color: '#fff', padding: 11, borderRadius: 14, fontSize: 'calc(13px * var(--fs-scale-body, 1))', fontWeight: 500, cursor: 'pointer', fontFamily: 'inherit' }} // theme-lint-disable — on the dark total card, drawn like Add expense above it
|
||||
>
|
||||
<ScanLine size={16} /> {t('costs.scan.button')}
|
||||
</button>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{/* Owe / Owed */}
|
||||
@@ -1237,6 +1250,11 @@ export interface ExpensePrefill {
|
||||
reservationId?: number
|
||||
/** Set when the expense is being created from a place (#1298). */
|
||||
placeId?: number
|
||||
/** The rest comes from a scanned receipt: its currency, day and lines, and the photo, attached on save. */
|
||||
currency?: string
|
||||
date?: string
|
||||
lines?: ReceiptLine[]
|
||||
receiptFiles?: File[]
|
||||
}
|
||||
|
||||
export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClose, onSaved }: {
|
||||
@@ -1252,17 +1270,16 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
|
||||
|
||||
const [name, setName] = useState(editing?.name || prefill?.name || '')
|
||||
const [cat, setCat] = useState<string>(editing ? catMeta(editing.category).key : (prefill?.category || 'food'))
|
||||
const [currency, setCurrency] = useState(editingCurrency)
|
||||
const [day, setDay] = useState(editing?.expense_date || localToday())
|
||||
const [seed] = useState(() => newExpenseSeed(prefill, base, people.map(p => p.id), localToday()))
|
||||
const [currency, setCurrency] = useState(editing ? editingCurrency : seed.currency)
|
||||
const [day, setDay] = useState(editing ? (editing.expense_date || localToday()) : seed.day)
|
||||
const [note, setNote] = useState(() => readUserNote(editing))
|
||||
// Edit and prefill seeds are padded to the currency's decimals (#2175): the DB
|
||||
// returns numbers, so a saved 4,90 would otherwise reopen as "4,9" and a saved
|
||||
// 5,00 as "5". A prefill has no currency of its own — it is read as `base`,
|
||||
// which is also what the currency field starts on.
|
||||
// 5,00 as "5".
|
||||
const [total, setTotal] = useState<string>(() => {
|
||||
if (editing) return editing.total_price ? amountToInputString(editing.total_price, editingCurrency) : ''
|
||||
if (prefill?.amount != null) return amountToInputString(prefill.amount, base)
|
||||
return ''
|
||||
return seed.total
|
||||
})
|
||||
const [participants, setParticipants] = useState<Set<number>>(() =>
|
||||
editing ? new Set((editing.members || []).map(m => m.user_id)) : new Set(people.map(p => p.id)))
|
||||
@@ -1303,7 +1320,7 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
|
||||
return 'equally'
|
||||
})
|
||||
|
||||
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => readTicketItems(editing))
|
||||
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => editing ? readTicketItems(editing) : seed.ticketItems)
|
||||
|
||||
const [customAmounts, setCustomAmounts] = useState<Record<number, string>>(() => {
|
||||
const m: Record<number, string> = {}
|
||||
@@ -1318,7 +1335,7 @@ export function ExpenseModal({ tripId, base, people, me, editing, prefill, onClo
|
||||
})
|
||||
|
||||
const [receipts, setReceipts] = useState<BudgetItemReceipt[]>(() => editing?.receipts || [])
|
||||
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>([])
|
||||
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>(() => editing ? [] : seed.receiptFiles)
|
||||
const [uploadingReceipt, setUploadingReceipt] = useState(false)
|
||||
const [modalPreviewReceipts, setModalPreviewReceipts] = useState<{ receipts: BudgetItemReceipt[]; initialIndex: number } | null>(null)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// FE-COSTSBAR-001 to FE-COSTSBAR-007
|
||||
// FE-COSTSBAR-001 to FE-COSTSBAR-008
|
||||
import { describe, it, expect, vi } from 'vitest'
|
||||
import { render, screen, fireEvent } from '../../../tests/helpers/render'
|
||||
import CostsToolbar from './CostsToolbar'
|
||||
@@ -66,4 +66,18 @@ describe('CostsToolbar', () => {
|
||||
expect(screen.queryByRole('button')).toBeNull()
|
||||
expect(screen.getByText('8 days')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-COSTSBAR-008: offers Scan receipt only when handed a way to open it', () => {
|
||||
const onScanReceipt = vi.fn()
|
||||
const { unmount } = render(
|
||||
<CostsToolbar dateMeta={null} people={PEOPLE} me={1} colorFor={() => '#000'} canEdit canSettle
|
||||
onSettleAll={vi.fn()} onAddExpense={vi.fn()} onScanReceipt={onScanReceipt} />,
|
||||
)
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Scan receipt' }))
|
||||
expect(onScanReceipt).toHaveBeenCalledTimes(1)
|
||||
unmount()
|
||||
|
||||
setup()
|
||||
expect(screen.queryByRole('button', { name: 'Scan receipt' })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Check, Plus } from 'lucide-react'
|
||||
import { Check, Plus, ScanLine } from 'lucide-react'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import type { TripMember } from './BudgetPanelMemberChips'
|
||||
|
||||
@@ -13,6 +13,8 @@ interface CostsToolbarProps {
|
||||
canSettle: boolean
|
||||
onSettleAll: () => void
|
||||
onAddExpense: () => void
|
||||
/** Opens the receipt scan; left out when the AI model reads no images. */
|
||||
onScanReceipt?: () => void
|
||||
}
|
||||
|
||||
const BODY_SIZE = 'calc(13px * var(--fs-scale-body, 1))'
|
||||
@@ -22,7 +24,7 @@ const BODY_SIZE = 'calc(13px * var(--fs-scale-body, 1))'
|
||||
* Bookings, Lists and Files open with: the tab's name, then what the numbers
|
||||
* cover (the trip's days and who travels), and the two actions on the right.
|
||||
*/
|
||||
export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit, canSettle, onSettleAll, onAddExpense }: CostsToolbarProps) {
|
||||
export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit, canSettle, onSettleAll, onAddExpense, onScanReceipt }: CostsToolbarProps) {
|
||||
const { t } = useTranslation()
|
||||
const chip = 'inline-flex items-center whitespace-nowrap rounded-full bg-surface-card px-3 py-1.5 font-medium text-content-muted shadow-sm'
|
||||
const button = 'inline-flex items-center gap-1.5 rounded-[10px] border-0 px-[14px] py-[9px] font-medium hover:opacity-[0.88]'
|
||||
@@ -64,6 +66,12 @@ export default function CostsToolbar({ dateMeta, people, me, colorFor, canEdit,
|
||||
<Check size={14} strokeWidth={2.5} />
|
||||
{t('costs.settleUp')}
|
||||
</button>
|
||||
{onScanReceipt && (
|
||||
<button type="button" onClick={onScanReceipt} className={`${button} bg-surface-card text-content`}>
|
||||
<ScanLine size={14} strokeWidth={2.5} />
|
||||
{t('costs.scan.button')}
|
||||
</button>
|
||||
)}
|
||||
<button type="button" onClick={onAddExpense} className={`${button} bg-accent text-accent-text`}>
|
||||
<Plus size={14} strokeWidth={2.5} />
|
||||
{t('costs.addExpense')}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
// FE-BUDGET-SCAN-001 to FE-BUDGET-SCAN-011
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest'
|
||||
import { http, HttpResponse } from 'msw'
|
||||
import { render, screen, fireEvent, waitFor } from '../../../tests/helpers/render'
|
||||
import { server } from '../../../tests/helpers/msw/server'
|
||||
import { saveImportFiles } from '../../db/offlineDb'
|
||||
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
|
||||
import { useReceiptScan } from './useReceiptScan'
|
||||
import { ReceiptScanModal } from './ReceiptScanModal'
|
||||
import MReceiptScanButton from '../../mobile/screens/trip/tabs/MReceiptScanButton'
|
||||
|
||||
vi.mock('../../db/offlineDb', () => ({ saveImportFiles: vi.fn(async () => {}) }))
|
||||
|
||||
function Harness({ canEdit = true }: { canEdit?: boolean }) {
|
||||
const scan = useReceiptScan(7, canEdit)
|
||||
return (
|
||||
<>
|
||||
{scan.offered && <button type="button" onClick={scan.open}>Scan receipt</button>}
|
||||
<ReceiptScanModal scan={scan} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
const input = () => screen.getByTestId('receipt-scan-input') as HTMLInputElement
|
||||
const photo = (name = 'bill.jpg', size = 4) => {
|
||||
const f = new File(['x'.repeat(Math.min(size, 16))], name, { type: 'image/jpeg' })
|
||||
if (size > 16) Object.defineProperty(f, 'size', { value: size })
|
||||
return f
|
||||
}
|
||||
async function openDialog() {
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Scan receipt' }))
|
||||
return screen.getByRole('dialog', { name: 'Scan a receipt' })
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(saveImportFiles).mockClear()
|
||||
useBackgroundTasksStore.setState({ tasks: [] })
|
||||
server.use(
|
||||
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })),
|
||||
http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({ jobId: 'job-5' })),
|
||||
)
|
||||
})
|
||||
|
||||
describe('Scan receipt', () => {
|
||||
it('FE-BUDGET-SCAN-001: offers the scan when the model reads images, and opens a dialog saying what it takes', async () => {
|
||||
render(<Harness />)
|
||||
await openDialog()
|
||||
expect(screen.getByText(/A photo of one receipt \(JPG, PNG or WEBP, up to 10 MB\)/)).toBeInTheDocument()
|
||||
expect(input().accept).toContain('image/jpeg')
|
||||
expect(screen.getByRole('button', { name: 'Scan' })).toBeDisabled()
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-002: offers nothing, and asks nothing, without budget_edit', async () => {
|
||||
let asked = false
|
||||
server.use(http.get('/api/llm/capabilities', () => { asked = true; return HttpResponse.json({ images: true }) }))
|
||||
render(<Harness canEdit={false} />)
|
||||
await new Promise(r => setTimeout(r, 20))
|
||||
expect(asked).toBe(false)
|
||||
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-003: offers nothing when the model reads no images', async () => {
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
|
||||
render(<Harness />)
|
||||
await new Promise(r => setTimeout(r, 20))
|
||||
expect(screen.queryByRole('button', { name: 'Scan receipt' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-004: a picked photo is named, then Scan starts a costs job the widget follows and closes the dialog', async () => {
|
||||
render(<Harness />)
|
||||
await openDialog()
|
||||
const picked = photo()
|
||||
fireEvent.change(input(), { target: { files: [picked] } })
|
||||
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Scan' }))
|
||||
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks[0]).toMatchObject({ id: 'job-5', tripId: '7', label: 'bill.jpg', kind: 'costs', status: 'running' }))
|
||||
expect(saveImportFiles).toHaveBeenCalledWith('job-5', [picked])
|
||||
await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument())
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-005: a dropped photo is taken like a picked one', async () => {
|
||||
render(<Harness />)
|
||||
await openDialog()
|
||||
const zone = screen.getByText('Drop the receipt photo here, or click to pick or take one').closest('button')!
|
||||
fireEvent.dragOver(zone)
|
||||
expect(screen.getByText('Drop the photo to scan it')).toBeInTheDocument()
|
||||
fireEvent.drop(zone, { dataTransfer: { files: [photo('drop.png')] } })
|
||||
expect(screen.getByText('drop.png')).toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: 'Scan' })).toBeEnabled()
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-006: refuses what is not a photo, and a photo over 10 MB, before anything is sent', async () => {
|
||||
let sent = false
|
||||
server.use(http.post('/api/trips/7/budget/receipt-scan', () => { sent = true; return HttpResponse.json({ jobId: 'x' }) }))
|
||||
render(<Harness />)
|
||||
await openDialog()
|
||||
fireEvent.change(input(), { target: { files: [new File(['%PDF'], 'bill.pdf', { type: 'application/pdf' })] } })
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('Only a photo can be scanned: JPG, PNG or WEBP.')
|
||||
fireEvent.change(input(), { target: { files: [photo('big.jpg', 11 * 1024 * 1024)] } })
|
||||
expect(screen.getByRole('alert')).toHaveTextContent('File "big.jpg" exceeds 10 MB limit.')
|
||||
expect(screen.getByRole('button', { name: 'Scan' })).toBeDisabled()
|
||||
expect(sent).toBe(false)
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-007: says why when the server refuses, and keeps the dialog open', async () => {
|
||||
server.use(http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({ error: 'The configured AI model does not read photos' }, { status: 400 })))
|
||||
render(<Harness />)
|
||||
await openDialog()
|
||||
fireEvent.change(input(), { target: { files: [photo()] } })
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Scan' }))
|
||||
expect(await screen.findByRole('alert')).toHaveTextContent('The configured AI model does not read photos')
|
||||
expect(screen.getByRole('dialog')).toBeInTheDocument()
|
||||
expect(useBackgroundTasksStore.getState().tasks).toEqual([])
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-008: Cancel, the close button and the backdrop close it; reopening starts empty', async () => {
|
||||
render(<Harness />)
|
||||
const dialog = await openDialog()
|
||||
fireEvent.change(input(), { target: { files: [photo()] } })
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }))
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
|
||||
|
||||
await openDialog()
|
||||
expect(screen.queryByText('bill.jpg')).not.toBeInTheDocument()
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Close' }))
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
|
||||
|
||||
await openDialog()
|
||||
fireEvent.click(screen.getByRole('dialog').parentElement!)
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
|
||||
expect(dialog).not.toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
|
||||
describe('MReceiptScanButton', () => {
|
||||
it('FE-BUDGET-SCAN-009: the phone header button opens the same dialog', async () => {
|
||||
render(<MReceiptScanButton tripId={7} canEdit />)
|
||||
fireEvent.click(await screen.findByRole('button', { name: 'Scan receipt' }))
|
||||
expect(screen.getByRole('dialog', { name: 'Scan a receipt' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-BUDGET-SCAN-010: draws nothing when the scan is not offered', async () => {
|
||||
const { container } = render(<MReceiptScanButton tripId={7} canEdit={false} />)
|
||||
await new Promise(r => setTimeout(r, 20))
|
||||
expect(container).toBeEmptyDOMElement()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,67 @@
|
||||
import { createPortal } from 'react-dom'
|
||||
import { useRef, useState } from 'react'
|
||||
import { ScanLine, X } from 'lucide-react'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import { RECEIPT_PHOTO_ACCEPT, type ReceiptScan } from './useReceiptScan'
|
||||
|
||||
/**
|
||||
* The dialog behind "Scan receipt", laid out like the booking import's: what it
|
||||
* takes, a drop zone that is also the picker (on a phone the picker offers the
|
||||
* camera), then Scan, which hands the photo to the background read and closes.
|
||||
*/
|
||||
export function ReceiptScanModal({ scan }: { scan: ReceiptScan }) {
|
||||
const { t } = useTranslation()
|
||||
const inputRef = useRef<HTMLInputElement>(null)
|
||||
const [dragOver, setDragOver] = useState(false)
|
||||
if (!scan.isOpen) return null
|
||||
|
||||
const onDrop = (e: React.DragEvent) => {
|
||||
e.preventDefault()
|
||||
setDragOver(false)
|
||||
scan.choose(Array.from(e.dataTransfer.files))
|
||||
}
|
||||
|
||||
return createPortal(
|
||||
<div role="presentation" className="fixed inset-0 z-[99999] flex items-center justify-center bg-black/40 p-4" onClick={(e) => { if (e.target === e.currentTarget) scan.close() }}>
|
||||
<div role="dialog" aria-modal="true" aria-label={t('costs.scan.title')} className="flex max-h-[90vh] w-full max-w-[540px] flex-col rounded-2xl bg-surface-card p-6 shadow-2xl">
|
||||
<div className="mb-3.5 flex items-center gap-2">
|
||||
<h2 className="flex-1 text-subtitle font-bold text-content">{t('costs.scan.title')}</h2>
|
||||
<button type="button" onClick={scan.close} aria-label={t('common.close')} className="flex rounded-md p-1 text-content-faint">
|
||||
<X size={16} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto">
|
||||
<p className="mb-3.5 text-caption leading-snug text-content-faint">{t('costs.scan.accepted')}</p>
|
||||
<input ref={inputRef} type="file" accept={RECEIPT_PHOTO_ACCEPT} hidden data-testid="receipt-scan-input"
|
||||
onChange={(e) => { scan.choose(e.target.files ? Array.from(e.target.files) : []); e.target.value = '' }} />
|
||||
<button type="button" onClick={() => inputRef.current?.click()}
|
||||
onDragOver={(e) => { e.preventDefault(); setDragOver(true) }}
|
||||
onDragEnter={(e) => { e.preventDefault(); setDragOver(true) }}
|
||||
onDragLeave={(e) => { if (e.target === e.currentTarget) setDragOver(false) }}
|
||||
onDrop={onDrop}
|
||||
className={`mb-3 flex min-h-[100px] w-full flex-col items-center justify-center gap-1.5 rounded-xl border-2 border-dashed p-4 text-body font-medium transition-colors ${dragOver ? 'border-accent bg-surface-tertiary' : 'border-edge bg-transparent'}`}>
|
||||
<ScanLine size={18} strokeWidth={1.8} className={`pointer-events-none ${dragOver ? 'text-accent' : 'text-content-faint'}`} />
|
||||
<span className={`pointer-events-none break-all text-center ${scan.photo ? 'text-content' : 'text-content-faint'}`}>
|
||||
{dragOver ? t('costs.scan.dropActive') : scan.photo ? scan.photo.name : t('costs.scan.dropHere')}
|
||||
</span>
|
||||
</button>
|
||||
{scan.error && (
|
||||
<div role="alert" className="mt-2 rounded-lg border border-danger bg-danger-soft px-2.5 py-2 text-caption text-danger">{scan.error}</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="mt-3.5 flex justify-end gap-2 border-t border-edge-secondary pt-3.5">
|
||||
<button type="button" onClick={scan.close} className="rounded-lg border border-edge px-4 py-2 text-body font-medium text-content">
|
||||
{t('common.cancel')}
|
||||
</button>
|
||||
<button type="button" onClick={scan.start} disabled={!scan.photo || scan.starting}
|
||||
className={`rounded-lg px-4 py-2 text-body font-medium ${scan.photo && !scan.starting ? 'bg-accent text-accent-text' : 'bg-surface-tertiary text-content-faint'}`}>
|
||||
{scan.starting ? t('costs.scan.reading') : t('costs.scan.start')}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>,
|
||||
document.body,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import { llmRepo } from '../../repo/llmRepo'
|
||||
import { saveImportFiles } from '../../db/offlineDb'
|
||||
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
|
||||
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'
|
||||
|
||||
/** The picker's filter: the extensions, and the MIME types a phone's camera roll matches on. */
|
||||
export const RECEIPT_PHOTO_ACCEPT = [...LLM_PHOTO_EXTENSIONS, 'image/jpeg', 'image/png', 'image/webp'].join(',')
|
||||
const MAX_PHOTO_BYTES = 10 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* "Scan a receipt" in Costs, for both shells. The button is only offered when
|
||||
* the person may add expenses and their AI model reads images. It opens a
|
||||
* dialog like the booking import's; the picked photo is read in the background
|
||||
* like a booking import too: the background tasks widget follows it, and its
|
||||
* review opens the expense editor pre-filled, with the photo waiting to be
|
||||
* attached on save. The photo is kept in IndexedDB so a reload mid-read still
|
||||
* has it for that review.
|
||||
*/
|
||||
export function useReceiptScan(tripId: number, canEdit: boolean) {
|
||||
const { t } = useTranslation()
|
||||
const addTask = useBackgroundTasksStore((s) => s.addTask)
|
||||
const [readsPhotos, setReadsPhotos] = useState(false)
|
||||
const [isOpen, setIsOpen] = useState(false)
|
||||
const [photo, setPhoto] = useState<File | null>(null)
|
||||
const [error, setError] = useState('')
|
||||
const [starting, setStarting] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (!canEdit) return
|
||||
let live = true
|
||||
llmRepo.readsPhotos().then((v) => { if (live) setReadsPhotos(v) }).catch(() => { if (live) setReadsPhotos(false) })
|
||||
return () => { live = false }
|
||||
}, [canEdit])
|
||||
|
||||
const open = () => { setPhoto(null); setError(''); setIsOpen(true) }
|
||||
const close = () => { if (!starting) setIsOpen(false) }
|
||||
|
||||
/** Take the first of the picked or dropped files, if it is a photo the model can read. */
|
||||
const choose = (files: File[]) => {
|
||||
const picked = files[0]
|
||||
if (!picked) return
|
||||
const ext = '.' + (picked.name.toLowerCase().split('.').pop() ?? '')
|
||||
if (!(LLM_PHOTO_EXTENSIONS as readonly string[]).includes(ext)) { setError(t('costs.scan.unsupported')); return }
|
||||
if (picked.size > MAX_PHOTO_BYTES) { setError(t('reservations.import.fileTooLarge', { name: picked.name })); return }
|
||||
setPhoto(picked)
|
||||
setError('')
|
||||
}
|
||||
|
||||
const start = async () => {
|
||||
if (!photo || starting) return
|
||||
setStarting(true)
|
||||
setError('')
|
||||
try {
|
||||
const jobId = await llmRepo.scanReceipt(tripId, photo)
|
||||
await saveImportFiles(jobId, [photo])
|
||||
addTask({ id: jobId, tripId: String(tripId), label: photo.name, total: 1, files: [photo], kind: 'costs' })
|
||||
setIsOpen(false)
|
||||
} catch (err) {
|
||||
const message = (err as { response?: { data?: { error?: string } } })?.response?.data?.error
|
||||
setError(message ?? t('costs.scan.failed'))
|
||||
} finally {
|
||||
setStarting(false)
|
||||
}
|
||||
}
|
||||
|
||||
return { offered: canEdit && readsPhotos, isOpen, open, close, photo, choose, error, starting, start }
|
||||
}
|
||||
|
||||
export type ReceiptScan = ReturnType<typeof useReceiptScan>
|
||||
@@ -197,3 +197,38 @@ describe('BookingImportModal', () => {
|
||||
expect(onClose).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('BookingImportModal with a model that reads images', () => {
|
||||
const jpg = () => new File(['jpeg'], 'ticket.jpg', { type: 'image/jpeg' });
|
||||
|
||||
it('FE-PLANNER-BOOKIMP-020: offers photos, and imports one with AI, when the model reads images', async () => {
|
||||
let mode: string | null = null;
|
||||
server.use(
|
||||
http.get('/api/health/features', () => HttpResponse.json({ bookingImport: true, aiParsing: true })),
|
||||
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })),
|
||||
http.post('/api/trips/4/reservations/import/booking/async', async ({ request }) => {
|
||||
mode = (await readMultipart(request)).fields.mode ?? null;
|
||||
return HttpResponse.json({ jobId: 'job-2' });
|
||||
}),
|
||||
);
|
||||
render(<BookingImportModal {...defaultProps} kind="transports" />);
|
||||
|
||||
expect(await screen.findByText(/Photos \(JPG, PNG, WEBP\) are read by the AI model\./)).toBeInTheDocument();
|
||||
expect(fileInput().accept).toContain('.jpg');
|
||||
fireEvent.change(fileInput(), { target: { files: [jpg()] } });
|
||||
expect(screen.getByText('ticket.jpg')).toBeInTheDocument();
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: 'Import' }));
|
||||
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks.find(t => t.id === 'job-2')).toMatchObject({ kind: 'transports' }));
|
||||
expect(mode).toBe('fallback-on-empty');
|
||||
});
|
||||
|
||||
it('FE-PLANNER-BOOKIMP-021: refuses a photo while the model reads no images', async () => {
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })));
|
||||
render(<BookingImportModal {...defaultProps} />);
|
||||
await waitFor(() => expect(fileInput().accept).not.toContain('.jpg'));
|
||||
fireEvent.change(fileInput(), { target: { files: [jpg()] } });
|
||||
expect(screen.getByText('Unsupported file format. Use EML, PDF, PKPass, HTML, or TXT.')).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Photos \(JPG/)).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,8 @@ import { useState, useRef, useEffect } from 'react'
|
||||
import { Upload, X } from 'lucide-react'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import { reservationsApi, healthApi } from '../../api/client'
|
||||
import { llmRepo } from '../../repo/llmRepo'
|
||||
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'
|
||||
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
|
||||
import { saveImportFiles } from '../../db/offlineDb'
|
||||
|
||||
@@ -36,6 +38,9 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [error, setError] = useState('')
|
||||
const [aiParsing, setAiParsing] = useState(false)
|
||||
const [readsPhotos, setReadsPhotos] = useState(false)
|
||||
// Only a model that reads images can do anything with a photo, so photos are offered only then.
|
||||
const acceptedExts: readonly string[] = readsPhotos ? [...ACCEPTED_EXTS, ...LLM_PHOTO_EXTENSIONS] : ACCEPTED_EXTS
|
||||
|
||||
const reset = () => {
|
||||
setFiles([])
|
||||
@@ -53,13 +58,14 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
|
||||
useEffect(() => {
|
||||
if (!isOpen) return
|
||||
healthApi.features().then((f) => setAiParsing(!!f.aiParsing)).catch(() => setAiParsing(false))
|
||||
llmRepo.readsPhotos().then(setReadsPhotos).catch(() => setReadsPhotos(false))
|
||||
}, [isOpen])
|
||||
|
||||
const handleClose = () => { reset(); onClose() }
|
||||
|
||||
const validateFile = (f: File): string | null => {
|
||||
const ext = ('.' + f.name.toLowerCase().split('.').pop()) as string
|
||||
if (!ACCEPTED_EXTS.includes(ext)) return t('reservations.import.unsupportedFormat')
|
||||
if (!acceptedExts.includes(ext)) return t('reservations.import.unsupportedFormat')
|
||||
if (f.size > MAX_FILE_BYTES) return t('reservations.import.fileTooLarge', { name: f.name })
|
||||
return null
|
||||
}
|
||||
@@ -143,12 +149,13 @@ export default function BookingImportModal({ isOpen, onClose, tripId, kind }: Bo
|
||||
<div style={{ flex: 1, overflowY: 'auto', minHeight: 0 }}>
|
||||
<div style={{ fontSize: 'calc(12px * var(--fs-scale-body, 1))', color: 'var(--text-faint)', marginBottom: 14, lineHeight: 1.45 }}>
|
||||
{t('reservations.import.acceptedFormats')}
|
||||
{readsPhotos && <> {t('reservations.import.acceptedPhotos')}</>}
|
||||
</div>
|
||||
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept={ACCEPTED_EXTS.join(',')}
|
||||
accept={acceptedExts.join(',')}
|
||||
multiple
|
||||
style={{ display: 'none' }}
|
||||
onChange={handleInputChange}
|
||||
|
||||
@@ -40,7 +40,7 @@ function renderSection() {
|
||||
}
|
||||
|
||||
function multimodalToggle(): HTMLElement {
|
||||
const row = screen.getByText('Send documents as images').parentElement as HTMLElement;
|
||||
const row = screen.getByText('Model reads images').parentElement as HTMLElement;
|
||||
return row.querySelector('button') as HTMLElement;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import DawarichIcon from '../../../components/shared/DawarichIcon'
|
||||
import AirTrailIcon from '../../../components/shared/AirTrailIcon'
|
||||
import { DOCUMENT_PROVIDER_ICONS } from '../../../components/shared/DocumentProviderIcons'
|
||||
import MToggle from '../../components/MToggle'
|
||||
import { asLlmVision, LLM_VISION_MODES, type LlmVision } from '@trek/shared'
|
||||
import { MAdminButton, MAdminCard, MAdminField, MAdminInput, MAdminSecretInput } from './MAdminUi'
|
||||
|
||||
const ICON_MAP = {
|
||||
@@ -391,12 +392,14 @@ const RECOMMENDED_MODELS: { id: string; label: string; note: string; recommended
|
||||
* provider, it also lists installed Ollama models and can pull NuExtract models.
|
||||
*/
|
||||
function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
const { t } = useTranslation()
|
||||
const toast = useToast()
|
||||
const cfg = (addon.config ?? {}) as Record<string, unknown>
|
||||
const [provider, setProvider] = useState<string>((cfg.provider as string) ?? 'local')
|
||||
const [model, setModel] = useState<string>((cfg.model as string) ?? '')
|
||||
const [baseUrl, setBaseUrl] = useState<string>((cfg.baseUrl as string) ?? '')
|
||||
const [apiKey, setApiKey] = useState<string>((cfg.apiKey as string) ?? '')
|
||||
const [vision, setVision] = useState<LlmVision>(asLlmVision(cfg.vision))
|
||||
const [saving, setSaving] = useState(false)
|
||||
|
||||
// Local-provider model management.
|
||||
@@ -458,7 +461,7 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
setSaving(true)
|
||||
try {
|
||||
// Send the masked sentinel unchanged so the server keeps the stored key.
|
||||
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: baseUrl.trim(), apiKey, multimodal: cfg.multimodal === true } })
|
||||
await adminApi.updateAddon(addon.id, { config: { provider, model: model.trim(), baseUrl: baseUrl.trim(), apiKey, vision } })
|
||||
toast.success('Saved')
|
||||
} catch {
|
||||
toast.error('Failed to save')
|
||||
@@ -546,6 +549,31 @@ function LlmParsingConfig({ addon }: { addon: Addon }) {
|
||||
placeholder={provider === 'anthropic' ? 'claude-opus-4-8' : provider === 'openai' ? 'gpt-4o' : 'select or pull below'}
|
||||
/>
|
||||
|
||||
<MAdminField label={t('settings.aiParsing.multimodal')}>
|
||||
<div role="radiogroup" aria-label={t('settings.aiParsing.multimodal')} className="flex gap-[6px]">
|
||||
{LLM_VISION_MODES.map((mode) => {
|
||||
const active = vision === mode
|
||||
return (
|
||||
<button
|
||||
key={mode}
|
||||
type="button"
|
||||
role="radio"
|
||||
aria-checked={active}
|
||||
onClick={() => setVision(mode)}
|
||||
className={`flex-1 rounded-xl border px-3 py-[10px] text-[0.8125rem] font-semibold text-m-ink ${
|
||||
active ? 'border-[color:var(--m-act)] bg-[color:var(--m-ic)]' : 'border-[color:var(--m-rowbr)]'
|
||||
}`}
|
||||
>
|
||||
{t(`admin.addons.llm.vision.${mode}`)}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</MAdminField>
|
||||
<p className="font-geist text-[0.625rem] leading-relaxed text-m-faint">
|
||||
{t(provider === 'local' ? 'admin.addons.llm.vision.hintLocal' : 'admin.addons.llm.vision.hintCloud')}
|
||||
</p>
|
||||
|
||||
{/* Local model management (Ollama) */}
|
||||
{provider === 'local' && (
|
||||
<div className="space-y-3 rounded-xl border border-[color:var(--m-rowbr)] bg-[color:var(--m-ic)] p-3">
|
||||
|
||||
@@ -13,6 +13,7 @@ import MMapArea from './map/MMapArea'
|
||||
import MPlacesBrowser from './places/MPlacesBrowser'
|
||||
import MTripTabPanel from './tabs/MTripTabPanel'
|
||||
import MTripSheets from './sheets/MTripSheets'
|
||||
import MReceiptScanButton from './tabs/MReceiptScanButton'
|
||||
import MTripLoadingSplash from './MTripLoadingSplash'
|
||||
import { usePluginDayTints, dayTintBackground } from '../../../components/Plugins/PluginDaySchedule'
|
||||
import { stageOf } from '../../../components/Roadtrip/roadtripRowModel'
|
||||
@@ -593,6 +594,7 @@ export default function MTripShell({
|
||||
{trTab === 'finanzplan' && (
|
||||
<div className="absolute left-1/2 top-1/2 flex -translate-x-1/2 -translate-y-1/2 items-center gap-[7px]">
|
||||
<PrimaryPill label={t('costs.addExpense')} onClick={() => setAddExpenseSignal(s => s + 1)} />
|
||||
<MReceiptScanButton tripId={tripId} canEdit={planner.can('budget_edit', trip)} />
|
||||
<MIconBtn ariaLabel={t('budget.exportCsv')} onClick={() => setExportCostsCsvSignal(s => s + 1)} size={40} className="text-m-muted backdrop-blur-[24px] backdrop-saturate-[1.7]">
|
||||
<FileDown size={15} strokeWidth={2} />
|
||||
</MIconBtn>
|
||||
|
||||
@@ -15,7 +15,7 @@ import { splitShareLabel, useExpenseFx } from '../../../../components/Budget/exp
|
||||
import { SYMBOLS, SPLIT_COLORS, currenciesWith } from '../../../../components/Budget/BudgetPanel.constants'
|
||||
import { COST_CATEGORY_LIST, catMeta } from '../../../../components/Budget/costsCategories'
|
||||
import { localToday } from '../../../../components/Planner/today'
|
||||
import { amountPattern, calculateTicketShares, hasTicketSplit, NOTE_MAX, readTicketItems, readUserNote, splitEqualShares, writeTicketItems, type TicketItem } from '../../../../components/Budget/CostsPanel.helpers'
|
||||
import { amountPattern, calculateTicketShares, hasTicketSplit, newExpenseSeed, NOTE_MAX, readTicketItems, readUserNote, splitEqualShares, writeTicketItems, type TicketItem } from '../../../../components/Budget/CostsPanel.helpers'
|
||||
import type { ExpensePrefill } from '../../../../components/Budget/CostsPanel'
|
||||
import { payersBalanced, rebalancePayers } from '../../../../components/Budget/CostsPanel.helpers'
|
||||
import GuestBadge from '../../../../components/shared/GuestBadge'
|
||||
@@ -76,15 +76,14 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
|
||||
const [cat, setCat] = useState<string>(editing ? catMeta(editing.category).key : (prefill?.category || 'food'))
|
||||
const [catOpen, setCatOpen] = useState(false)
|
||||
const [note, setNote] = useState(() => readUserNote(editing))
|
||||
const [currency, setCurrency] = useState(editingCurrency)
|
||||
const [day, setDay] = useState(editing?.expense_date || localToday())
|
||||
const [seed] = useState(() => newExpenseSeed(prefill, base, people.map(p => p.id), localToday()))
|
||||
const [currency, setCurrency] = useState(editing ? editingCurrency : seed.currency)
|
||||
const [day, setDay] = useState(editing ? (editing.expense_date || localToday()) : seed.day)
|
||||
// Edit and prefill seeds are padded to the currency's decimals (#2175), same
|
||||
// as the desktop modal: a saved 4,90 must reopen as "4,90", not "4,9". A
|
||||
// prefill has no currency of its own and is read as `base`.
|
||||
// as the desktop modal: a saved 4,90 must reopen as "4,90", not "4,9".
|
||||
const [total, setTotal] = useState<string>(() => {
|
||||
if (editing) return editing.total_price ? amountToInputString(editing.total_price, editingCurrency) : ''
|
||||
if (prefill?.amount != null) return amountToInputString(prefill.amount, base)
|
||||
return ''
|
||||
return seed.total
|
||||
})
|
||||
const [participants, setParticipants] = useState<Set<number>>(() =>
|
||||
editing ? new Set((editing.members || []).map(m => m.user_id)) : new Set(people.map(p => p.id)))
|
||||
@@ -115,7 +114,7 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
|
||||
return 'equally'
|
||||
})
|
||||
|
||||
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => readTicketItems(editing))
|
||||
const [ticketItems, setTicketItems] = useState<TicketItem[]>(() => editing ? readTicketItems(editing) : seed.ticketItems)
|
||||
|
||||
const [customAmounts, setCustomAmounts] = useState<Record<number, string>>(() => {
|
||||
const m: Record<number, string> = {}
|
||||
@@ -128,7 +127,7 @@ export default function MCostSheet({ tripId, base, people, me, editing, prefill,
|
||||
})
|
||||
|
||||
const [receipts, setReceipts] = useState<BudgetItemReceipt[]>(() => editing?.receipts || [])
|
||||
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>([])
|
||||
const [pendingReceiptFiles, setPendingReceiptFiles] = useState<File[]>(() => editing ? [] : seed.receiptFiles)
|
||||
const [uploadingReceipt, setUploadingReceipt] = useState(false)
|
||||
const [previewReceipts, setPreviewReceipts] = useState<{ receipts: BudgetItemReceipt[]; initialIndex: number } | null>(null)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import AirTrailImportModal from '../../../../components/Planner/AirTrailImportMo
|
||||
import TripFormModal from '../../../../components/Trips/TripFormModal'
|
||||
import TripMembersModal from '../../../../components/Trips/TripMembersModal'
|
||||
import type { ExpensePrefill } from '../../../../components/Budget/CostsPanel'
|
||||
import { expenseEditorFor } from '../../../../components/Budget/CostsPanel.helpers'
|
||||
import { useAuthStore } from '../../../../store/authStore'
|
||||
import { useSettingsStore } from '../../../../store/settingsStore'
|
||||
import { useTripStore } from '../../../../store/tripStore'
|
||||
@@ -59,6 +60,7 @@ export default function MTripSheets({ planner, shell }: MTripSheetsProps) {
|
||||
else if (req.prefill) setBookingExpense({ editing: null, prefill: req.prefill })
|
||||
}
|
||||
const costsBase = (displayCurrency || trip?.currency || 'EUR').toUpperCase()
|
||||
const expenseEditor = expenseEditorFor(bookingExpense, () => setBookingExpense(null), planner.receiptExpense, planner.clearReceiptExpense)
|
||||
|
||||
return (
|
||||
<>
|
||||
@@ -142,16 +144,17 @@ export default function MTripSheets({ planner, shell }: MTripSheetsProps) {
|
||||
/>
|
||||
)}
|
||||
|
||||
{bookingExpense && (
|
||||
{expenseEditor && (
|
||||
<MCostSheet
|
||||
key={expenseEditor.key}
|
||||
tripId={tripId}
|
||||
base={costsBase}
|
||||
people={planner.tripMembers}
|
||||
me={meId}
|
||||
editing={bookingExpense.editing}
|
||||
prefill={bookingExpense.prefill}
|
||||
onClose={() => setBookingExpense(null)}
|
||||
onSaved={() => { setBookingExpense(null); loadBudgetItems(tripId) }}
|
||||
editing={expenseEditor.editing}
|
||||
prefill={expenseEditor.prefill}
|
||||
onClose={expenseEditor.close}
|
||||
onSaved={() => { expenseEditor.close(); loadBudgetItems(tripId) }}
|
||||
/>
|
||||
)}
|
||||
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { ScanLine } from 'lucide-react'
|
||||
import MIconBtn from '../../../components/MIconBtn'
|
||||
import { useReceiptScan } from '../../../../components/Budget/useReceiptScan'
|
||||
import { ReceiptScanModal } from '../../../../components/Budget/ReceiptScanModal'
|
||||
import { useTranslation } from '../../../../i18n'
|
||||
|
||||
/** The Costs header's "Scan receipt" button on a phone; the dialog and the scan are the shared ones. */
|
||||
export default function MReceiptScanButton({ tripId, canEdit }: { tripId: number; canEdit: boolean }) {
|
||||
const { t } = useTranslation()
|
||||
const scan = useReceiptScan(tripId, canEdit)
|
||||
if (!scan.offered) return null
|
||||
return (
|
||||
<>
|
||||
<MIconBtn ariaLabel={t('costs.scan.button')} onClick={scan.open} size={40} className="text-m-muted backdrop-blur-[24px] backdrop-saturate-[1.7]">
|
||||
<ScanLine size={15} strokeWidth={2} />
|
||||
</MIconBtn>
|
||||
<ReceiptScanModal scan={scan} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import AirTrailImportModal from '../components/Planner/AirTrailImportModal'
|
||||
import ApplyTemplateButton from '../components/Packing/ApplyTemplateButton'
|
||||
import PackingExportMenu from '../components/Packing/PackingExportMenu'
|
||||
import type { ExpensePrefill } from '../components/Budget/CostsPanel'
|
||||
import { expenseEditorFor } from '../components/Budget/CostsPanel.helpers'
|
||||
import type { BookingExpenseRequest } from '../components/Planner/BookingCostsSection.types'
|
||||
import type { BudgetItem } from '../types'
|
||||
import PluginFrame from '../components/Plugins/PluginFrame'
|
||||
@@ -309,6 +310,7 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
|
||||
transportModalDayId, setTransportModalDayId,
|
||||
transportModalAutomated, setTransportModalAutomated, transitPrefill, setTransitPrefill, transitJourney, setTransitJourney,
|
||||
reservationPrefill, transportPrefill, importReviewActive, advanceImportReview,
|
||||
receiptExpense, clearReceiptExpense,
|
||||
routeShown, setRouteShown, transitRoutesShown, routeProfile, setRouteProfile, routeVias, fitKey, setFitKey,
|
||||
mobileSidebarOpen, setMobileSidebarOpen, mobilePlanScrollTopRef, mobilePlacesScrollTopRef,
|
||||
deletePlaceId, setDeletePlaceId, deletePlaceIds, setDeletePlaceIds, deletePlaceNote, deletePlacesNote,
|
||||
@@ -375,6 +377,9 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
|
||||
if (req.editItem) setBookingExpense({ editing: req.editItem })
|
||||
else if (req.prefill) setBookingExpense({ editing: null, prefill: req.prefill })
|
||||
}
|
||||
// One expense editor for both openers: a booking's Costs block, and a scanned
|
||||
// receipt sent here from the background tasks widget.
|
||||
const expenseEditor = expenseEditorFor(bookingExpense, () => setBookingExpense(null), receiptExpense, clearReceiptExpense)
|
||||
|
||||
if (isLoading || !splashDone) {
|
||||
return (
|
||||
@@ -1174,18 +1179,19 @@ function TripPlannerPageDesktop(): React.ReactElement | null {
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{bookingExpense && (
|
||||
{expenseEditor && (
|
||||
<ErrorBoundary boundaryId="planner-panel:expense" fallback={null}>
|
||||
<Suspense fallback={null}>
|
||||
<ExpenseModal
|
||||
key={expenseEditor.key}
|
||||
tripId={tripId}
|
||||
base={costsBase}
|
||||
people={tripMembers}
|
||||
me={meId}
|
||||
editing={bookingExpense.editing}
|
||||
prefill={bookingExpense.prefill}
|
||||
onClose={() => setBookingExpense(null)}
|
||||
onSaved={() => { setBookingExpense(null); loadBudgetItems(tripId) }}
|
||||
editing={expenseEditor.editing}
|
||||
prefill={expenseEditor.prefill}
|
||||
onClose={expenseEditor.close}
|
||||
onSaved={() => { expenseEditor.close(); loadBudgetItems(tripId) }}
|
||||
/>
|
||||
</Suspense>
|
||||
</ErrorBoundary>
|
||||
|
||||
@@ -17,7 +17,7 @@ import {
|
||||
healthApi, airtrailApi, mapsApi,
|
||||
} from '../../api/client'
|
||||
import { accommodationRepo } from '../../repo/accommodationRepo'
|
||||
import { offlineDb } from '../../db/offlineDb'
|
||||
import { offlineDb, saveImportFiles, getImportFiles } from '../../db/offlineDb'
|
||||
import { getCached, fetchPhoto } from '../../services/photoService'
|
||||
import type { Accommodation, Place, Reservation, Settings } from '../../types'
|
||||
|
||||
@@ -2104,6 +2104,66 @@ describe('useTripPlanner — booking import review', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('useTripPlanner — a receipt scanned from Costs', () => {
|
||||
const RECEIPT = { merchant: 'Café', date: '2026-09-20', total: 12.5, currency: 'EUR', items: [{ name: 'Tart', price: 12.5 }] }
|
||||
|
||||
it('FE-TP-HOOK-160: a read receipt opens the expense editor pre-filled, with the photo to attach, and clears the widget', async () => {
|
||||
seedTrip()
|
||||
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
useBackgroundTasksStore.setState({
|
||||
tasks: [{
|
||||
id: 'job-r', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
|
||||
reviewRequested: true, items: [], receipt: RECEIPT, sourceFiles: [photo],
|
||||
}] as never,
|
||||
})
|
||||
|
||||
const { result } = await renderPlanner()
|
||||
|
||||
await waitFor(() => expect(result.current.receiptExpense).not.toBeNull())
|
||||
expect(result.current.receiptExpense).toEqual({
|
||||
name: 'Café', amount: 12.5, currency: 'EUR', date: '2026-09-20', lines: [{ name: 'Tart', price: 12.5 }], receiptFiles: [photo],
|
||||
})
|
||||
expect(result.current.showReservationModal).toBe(false)
|
||||
expect(useBackgroundTasksStore.getState().tasks).toHaveLength(0)
|
||||
|
||||
act(() => { result.current.clearReceiptExpense() })
|
||||
expect(result.current.receiptExpense).toBeNull()
|
||||
})
|
||||
|
||||
it('FE-TP-HOOK-162: after a reload the photo comes back from IndexedDB for the review', async () => {
|
||||
seedTrip()
|
||||
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
await saveImportFiles('job-db', [photo])
|
||||
useBackgroundTasksStore.setState({
|
||||
tasks: [{
|
||||
id: 'job-db', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
|
||||
reviewRequested: true, items: [], receipt: RECEIPT,
|
||||
}] as never,
|
||||
})
|
||||
|
||||
const { result } = await renderPlanner()
|
||||
|
||||
await waitFor(() => expect(result.current.receiptExpense).not.toBeNull())
|
||||
expect(result.current.receiptExpense?.receiptFiles?.map(f => f.name)).toEqual(['bill.jpg'])
|
||||
await waitFor(async () => expect(await getImportFiles('job-db')).toEqual([]))
|
||||
})
|
||||
|
||||
it('FE-TP-HOOK-161: a scan that read nothing just clears the widget', async () => {
|
||||
seedTrip()
|
||||
useBackgroundTasksStore.setState({
|
||||
tasks: [{
|
||||
id: 'job-n', tripId: '42', label: 'bill.jpg', status: 'done', done: 1, total: 1, kind: 'costs',
|
||||
reviewRequested: true, items: [], receipt: null,
|
||||
}] as never,
|
||||
})
|
||||
|
||||
const { result } = await renderPlanner()
|
||||
|
||||
await waitFor(() => expect(useBackgroundTasksStore.getState().tasks).toHaveLength(0))
|
||||
expect(result.current.receiptExpense).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('useTripPlanner — misc state', () => {
|
||||
it('FE-TP-HOOK-095: the map tile url falls back to the default basemap', async () => {
|
||||
seedTrip()
|
||||
|
||||
@@ -22,6 +22,8 @@ import { accommodationRepo } from '../../repo/accommodationRepo'
|
||||
import { offlineDb, getImportFiles, deleteImportFiles } from '../../db/offlineDb'
|
||||
import { isEffectivelyOffline } from '../../sync/networkMode'
|
||||
import { useBackgroundTasksStore } from '../../store/backgroundTasksStore'
|
||||
import { receiptToPrefill } from '../../components/Budget/CostsPanel.helpers'
|
||||
import type { ExpensePrefill } from '../../components/Budget/CostsPanel'
|
||||
import { useAuthStore } from '../../store/authStore'
|
||||
import { useResizablePanels } from '../../hooks/useResizablePanels'
|
||||
import { useTripWebSocket } from '../../hooks/useTripWebSocket'
|
||||
@@ -435,6 +437,8 @@ export function useTripPlanner() {
|
||||
const [reservationPrefill, setReservationPrefill] = useState<BookingReviewDraft | null>(null)
|
||||
const [transportPrefill, setTransportPrefill] = useState<BookingReviewDraft | null>(null)
|
||||
const [importReviewActive, setImportReviewActive] = useState(false)
|
||||
// The expense a scanned receipt pre-fills, opened by the page's expense editor.
|
||||
const [receiptExpense, setReceiptExpense] = useState<ExpensePrefill | null>(null)
|
||||
const importQueueRef = useRef<BookingImportPreviewItem[]>([])
|
||||
// The files this import was parsed from, so each reviewed booking can attach its source doc.
|
||||
const importSourceFilesRef = useRef<File[]>([])
|
||||
@@ -2727,13 +2731,26 @@ export function useTripPlanner() {
|
||||
const task = bgTasks.find(
|
||||
(tk) => tk.tripId === String(tripId) && tk.status === 'done' && tk.reviewRequested && !tk.consumed,
|
||||
)
|
||||
if (task && task.items && task.items.length > 0) {
|
||||
if (task && task.kind === 'costs') {
|
||||
// A scanned receipt is reviewed in the expense editor, pre-filled with what
|
||||
// was read and with the photo waiting to be attached when it is saved.
|
||||
const receipt = task.receipt
|
||||
const jobId = task.id
|
||||
const inMemory = task.sourceFiles
|
||||
dismissBgTask(jobId)
|
||||
if (!receipt) return
|
||||
void (async () => {
|
||||
const files = inMemory && inMemory.length ? inMemory : await getImportFiles(jobId)
|
||||
deleteImportFiles(jobId)
|
||||
setReceiptExpense(receiptToPrefill(receipt, files))
|
||||
})()
|
||||
} else if (task && task.items && task.items.length > 0) {
|
||||
// Hand the items (and the source files, to attach to each booking) to the review flow
|
||||
// and clear the widget entry — once the user hit "review", the background card is done.
|
||||
const items = task.items
|
||||
const jobId = task.id
|
||||
const inMemory = task.sourceFiles
|
||||
const kind = task.kind ?? 'bookings'
|
||||
const kind = task.kind === 'transports' ? 'transports' : 'bookings'
|
||||
dismissBgTask(jobId)
|
||||
// Prefer the in-memory files (immediate path); after a reload they live in IndexedDB.
|
||||
void (async () => {
|
||||
@@ -2866,6 +2883,7 @@ export function useTripPlanner() {
|
||||
transportModalDayId, setTransportModalDayId,
|
||||
transportModalAutomated, setTransportModalAutomated, transitPrefill, setTransitPrefill, transitJourney, setTransitJourney,
|
||||
reservationPrefill, transportPrefill, importReviewActive, startImportReview, advanceImportReview,
|
||||
receiptExpense, clearReceiptExpense: () => setReceiptExpense(null),
|
||||
routeShown, setRouteShown, autoShowRoute, transitRoutesShown, routeProfile, setRouteProfile, routeVias, fitKey, setFitKey,
|
||||
mobileSidebarOpen, setMobileSidebarOpen, mobilePlanScrollTopRef, mobilePlacesScrollTopRef,
|
||||
deletePlaceId, setDeletePlaceId, deletePlaceIds, setDeletePlaceIds, deletePlaceNote, deletePlacesNote,
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
// FE-REPO-LLM-001 to FE-REPO-LLM-004
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { http, HttpResponse } from 'msw'
|
||||
import { server } from '../../tests/helpers/msw/server'
|
||||
import { readMultipart } from '../../tests/helpers/multipart'
|
||||
import { llmRepo } from './llmRepo'
|
||||
|
||||
function setOnline(v: boolean): void {
|
||||
Object.defineProperty(navigator, 'onLine', { value: v, writable: true, configurable: true })
|
||||
}
|
||||
|
||||
beforeEach(() => setOnline(true))
|
||||
|
||||
describe('llmRepo', () => {
|
||||
it('FE-REPO-LLM-001: readsPhotos answers what the server says the model reads', async () => {
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: true })))
|
||||
await expect(llmRepo.readsPhotos()).resolves.toBe(true)
|
||||
server.use(http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })))
|
||||
await expect(llmRepo.readsPhotos()).resolves.toBe(false)
|
||||
})
|
||||
|
||||
it('FE-REPO-LLM-002: readsPhotos is false offline, without asking', async () => {
|
||||
let asked = false
|
||||
server.use(http.get('/api/llm/capabilities', () => { asked = true; return HttpResponse.json({ images: true }) }))
|
||||
setOnline(false)
|
||||
await expect(llmRepo.readsPhotos()).resolves.toBe(false)
|
||||
expect(asked).toBe(false)
|
||||
})
|
||||
|
||||
it('FE-REPO-LLM-003: scanReceipt uploads the photo as `file` and answers the job id', async () => {
|
||||
let sent: string[] = []
|
||||
server.use(http.post('/api/trips/7/budget/receipt-scan', async ({ request }) => {
|
||||
sent = (await readMultipart(request)).filenames
|
||||
return HttpResponse.json({ jobId: 'job-9' })
|
||||
}))
|
||||
const photo = new File(['jpeg'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
await expect(llmRepo.scanReceipt(7, photo)).resolves.toBe('job-9')
|
||||
// One file part; jsdom names every File part 'blob' on the wire, so the name is not checked.
|
||||
expect(sent).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('FE-REPO-LLM-004: scanReceipt refuses offline and rejects an answer without a job id', async () => {
|
||||
setOnline(false)
|
||||
await expect(llmRepo.scanReceipt(7, new File(['x'], 'a.jpg'))).rejects.toThrow(/connection/)
|
||||
setOnline(true)
|
||||
server.use(http.post('/api/trips/7/budget/receipt-scan', () => HttpResponse.json({})))
|
||||
await expect(llmRepo.scanReceipt(7, new File(['x'], 'a.jpg'))).rejects.toThrow()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,25 @@
|
||||
import { llmCapabilitiesResponseSchema, receiptScanStartResponseSchema } from '@trek/shared'
|
||||
import { apiClient, postMultipart } from '../api/client'
|
||||
import { isEffectivelyOffline } from '../sync/networkMode'
|
||||
|
||||
/**
|
||||
* What the AI model can read, and the receipt scan that depends on it. Both are
|
||||
* online-only: the model runs on the server, so offline a photo is simply not
|
||||
* offered.
|
||||
*/
|
||||
export const llmRepo = {
|
||||
/** Whether a photo is worth offering to this user: their AI model reads images. */
|
||||
async readsPhotos(): Promise<boolean> {
|
||||
if (isEffectivelyOffline()) return false
|
||||
const reply = await apiClient.get('/llm/capabilities')
|
||||
return llmCapabilitiesResponseSchema.parse(reply.data).images
|
||||
},
|
||||
|
||||
/** Start reading one receipt photo in the background; answers the import job id. */
|
||||
async scanReceipt(tripId: number | string, photo: File): Promise<string> {
|
||||
if (isEffectivelyOffline()) throw new Error('Scanning a receipt requires a connection')
|
||||
const form = new FormData()
|
||||
form.append('file', photo)
|
||||
return receiptScanStartResponseSchema.parse(await postMultipart(`/trips/${tripId}/budget/receipt-scan`, form)).jobId
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { useBackgroundTasksStore, taskFoundSomething, type BackgroundImportTask } from './backgroundTasksStore'
|
||||
|
||||
const RECEIPT = { merchant: 'Café', date: null, total: 4, currency: 'EUR', items: [] }
|
||||
|
||||
beforeEach(() => useBackgroundTasksStore.setState({ tasks: [] }))
|
||||
|
||||
describe('backgroundTasksStore with a receipt scan', () => {
|
||||
it('keeps what the scan read on the finished task', () => {
|
||||
const s = useBackgroundTasksStore.getState()
|
||||
s.addTask({ id: 'j1', tripId: '7', label: 'bill.jpg', total: 1, kind: 'costs' })
|
||||
s.setDone('j1', '7', [], [], RECEIPT)
|
||||
expect(useBackgroundTasksStore.getState().tasks[0]).toMatchObject({ status: 'done', kind: 'costs', receipt: RECEIPT, items: [] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('taskFoundSomething', () => {
|
||||
const task = (over: Partial<BackgroundImportTask>): BackgroundImportTask => ({ id: 'j', tripId: '7', label: 'x', status: 'done', done: 1, total: 1, ...over })
|
||||
|
||||
it('asks a receipt scan for its receipt', () => {
|
||||
expect(taskFoundSomething(task({ kind: 'costs', items: [], receipt: RECEIPT }))).toBe(true)
|
||||
expect(taskFoundSomething(task({ kind: 'costs', items: [], receipt: null }))).toBe(false)
|
||||
})
|
||||
|
||||
it('asks a booking parse for its items', () => {
|
||||
expect(taskFoundSomething(task({ kind: 'bookings', items: [{} as never] }))).toBe(true)
|
||||
expect(taskFoundSomething(task({ items: [] }))).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,6 @@
|
||||
import { create } from 'zustand'
|
||||
import { persist } from 'zustand/middleware'
|
||||
import type { BookingImportPreviewItem, BookingImportMode } from '@trek/shared'
|
||||
import type { BookingImportPreviewItem, BookingImportMode, ReceiptRead } from '@trek/shared'
|
||||
|
||||
/**
|
||||
* Tracks booking-import parses that run in the BACKGROUND (the async endpoint).
|
||||
@@ -15,6 +15,8 @@ import type { BookingImportPreviewItem, BookingImportMode } from '@trek/shared'
|
||||
* on mount. We deliberately persist neither the parsed `items` (re-fetched) nor the
|
||||
* transient review flags (so a reload never auto-reopens the review flow).
|
||||
*/
|
||||
export type BackgroundTaskKind = 'transports' | 'bookings' | 'costs'
|
||||
|
||||
export interface BackgroundImportTask {
|
||||
id: string // server job id
|
||||
tripId: string
|
||||
@@ -23,6 +25,8 @@ export interface BackgroundImportTask {
|
||||
done: number
|
||||
total: number
|
||||
items?: BookingImportPreviewItem[]
|
||||
/** A receipt scanned from Costs (`kind: 'costs'`): what the model read, or null when it read nothing. */
|
||||
receipt?: ReceiptRead | null
|
||||
warnings?: string[]
|
||||
error?: string
|
||||
reviewRequested?: boolean // user clicked "review" — the trip page consumes it
|
||||
@@ -37,16 +41,17 @@ export interface BackgroundImportTask {
|
||||
* when its type is guessed (#2076), and unlike the other transient flags it IS
|
||||
* persisted: the parse deliberately outlives navigation and reload, and the
|
||||
* review is triggered by the global widget on any page, so a component-local
|
||||
* value is gone by the time it is needed.
|
||||
* value is gone by the time it is needed. `costs` is a receipt scan, reviewed
|
||||
* in the expense editor instead of the booking forms.
|
||||
*/
|
||||
kind?: 'transports' | 'bookings'
|
||||
kind?: BackgroundTaskKind
|
||||
}
|
||||
|
||||
interface BackgroundTasksState {
|
||||
tasks: BackgroundImportTask[]
|
||||
addTask: (task: { id: string; tripId: string; label: string; total: number; files?: File[]; mode?: BookingImportMode; kind?: 'transports' | 'bookings' }) => void
|
||||
addTask: (task: { id: string; tripId: string; label: string; total: number; files?: File[]; mode?: BookingImportMode; kind?: BackgroundTaskKind }) => void
|
||||
setProgress: (id: string, tripId: string, done: number, total: number) => void
|
||||
setDone: (id: string, tripId: string, items: BookingImportPreviewItem[], warnings: string[]) => void
|
||||
setDone: (id: string, tripId: string, items: BookingImportPreviewItem[], warnings: string[], receipt?: ReceiptRead | null) => void
|
||||
setError: (id: string, tripId: string, error: string) => void
|
||||
requestReview: (id: string) => void
|
||||
markConsumed: (id: string) => void
|
||||
@@ -73,7 +78,7 @@ export const useBackgroundTasksStore = create<BackgroundTasksState>()(
|
||||
tasks: [],
|
||||
addTask: ({ id, tripId, label, total, files, mode, kind }) => upsert(id, tripId, { label, total, status: 'running', done: 0, sourceFiles: files, mode, kind }),
|
||||
setProgress: (id, tripId, done, total) => upsert(id, tripId, { done, total, status: 'running' }),
|
||||
setDone: (id, tripId, items, warnings) => upsert(id, tripId, { status: 'done', items, warnings, done: items?.length ?? 0 }),
|
||||
setDone: (id, tripId, items, warnings, receipt) => upsert(id, tripId, { status: 'done', items, warnings, receipt, done: items?.length ?? 0 }),
|
||||
setError: (id, tripId, error) => upsert(id, tripId, { status: 'error', error }),
|
||||
requestReview: (id) => set((s) => ({ tasks: s.tasks.map((t) => (t.id === id ? { ...t, reviewRequested: true } : t)) })),
|
||||
markConsumed: (id) => set((s) => ({ tasks: s.tasks.map((t) => (t.id === id ? { ...t, consumed: true, reviewRequested: false } : t)) })),
|
||||
@@ -96,3 +101,8 @@ export const useBackgroundTasksStore = create<BackgroundTasksState>()(
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
/** Whether a finished task has something to review: a read receipt, or at least one booking. */
|
||||
export function taskFoundSomething(task: BackgroundImportTask): boolean {
|
||||
return task.kind === 'costs' ? Boolean(task.receipt) : (task.items?.length ?? 0) > 0
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { http, HttpResponse } from 'msw';
|
||||
|
||||
export const addonHandlers = [
|
||||
// Whose AI model reads images: off unless a test says otherwise, so no photo picker shows by default.
|
||||
http.get('/api/llm/capabilities', () => HttpResponse.json({ images: false })),
|
||||
http.get('/api/addons', () => {
|
||||
return HttpResponse.json({
|
||||
bagTracking: false,
|
||||
|
||||
@@ -649,7 +649,7 @@ describe('MAdminAddonManager', () => {
|
||||
model: 'gpt-4o',
|
||||
baseUrl: 'https://api.openai.com/v1',
|
||||
apiKey: '••••••••',
|
||||
multimodal: true,
|
||||
vision: 'auto',
|
||||
},
|
||||
});
|
||||
|
||||
@@ -657,6 +657,30 @@ describe('MAdminAddonManager', () => {
|
||||
await screen.findByText('Failed to save');
|
||||
});
|
||||
|
||||
it('FE-MOB-AADD-030: whether the model reads images is a three-way choice, saved as picked', async () => {
|
||||
const user = userEvent.setup();
|
||||
const bodies: unknown[] = [];
|
||||
server.use(
|
||||
addonsRoute([llmAddon({ provider: 'local', model: 'llava:7b', baseUrl: '', apiKey: '', vision: 'on' })]),
|
||||
http.get('/api/admin/llm/local/models', () => HttpResponse.json({ models: [] })),
|
||||
http.put('/api/admin/addons/llm_parsing', async ({ request }) => {
|
||||
bodies.push(await request.json());
|
||||
return HttpResponse.json({ success: true });
|
||||
}),
|
||||
);
|
||||
render(<><ToastContainer /><MAdminAddonManager /></>);
|
||||
|
||||
const group = await screen.findByRole('radiogroup', { name: 'Model reads images' });
|
||||
expect(within(group).getByRole('radio', { name: 'Yes' })).toHaveAttribute('aria-checked', 'true');
|
||||
expect(screen.getByText('Automatic asks the Ollama server whether this model reads images.')).toBeInTheDocument();
|
||||
await user.click(within(group).getByRole('radio', { name: 'No' }));
|
||||
expect(within(group).getByRole('radio', { name: 'No' })).toHaveAttribute('aria-checked', 'true');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Save' }));
|
||||
await screen.findByText('Saved');
|
||||
expect(bodies[0]).toMatchObject({ config: { vision: 'off' } });
|
||||
});
|
||||
|
||||
it('FE-MOB-AADD-024: the API key field can be revealed', async () => {
|
||||
const user = userEvent.setup();
|
||||
server.use(addonsRoute([llmAddon({ provider: 'anthropic', apiKey: 'sk-secret' })]));
|
||||
|
||||
@@ -66,7 +66,7 @@ describe('MLlmConnectionSection', () => {
|
||||
|
||||
expect(screen.getByRole('button', { name: /OpenAI/ })).toBeInTheDocument();
|
||||
expect(screen.getByDisplayValue('gpt-4o-mini')).toBeInTheDocument();
|
||||
expect(screen.getByRole('switch', { name: 'Send documents as images' })).toHaveAttribute('aria-checked', 'true');
|
||||
expect(screen.getByRole('switch', { name: 'Model reads images' })).toHaveAttribute('aria-checked', 'true');
|
||||
});
|
||||
|
||||
it('FE-MOB-SETLLM-003: nothing is hydrated while the settings are still loading', () => {
|
||||
@@ -158,7 +158,7 @@ describe('MLlmConnectionSection', () => {
|
||||
const updateSettings = seedLlm();
|
||||
renderSection();
|
||||
|
||||
await user.click(screen.getByRole('switch', { name: 'Send documents as images' }));
|
||||
await user.click(screen.getByRole('switch', { name: 'Model reads images' }));
|
||||
await user.click(screen.getByRole('button', { name: /Save/ }));
|
||||
|
||||
expect(updateSettings).toHaveBeenCalledWith(expect.objectContaining({ llm_multimodal: true }));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { ExpensePrefill } from '../../../../src/components/Budget/CostsPanel'
|
||||
import MCostSheet from '../../../../src/mobile/screens/trip/sheets/MCostSheet'
|
||||
import type { TripMember } from '../../../../src/components/Budget/BudgetPanelMemberChips'
|
||||
import { clearExchangeRateCache } from '../../../../src/hooks/useExchangeRates'
|
||||
@@ -35,7 +36,7 @@ interface SheetOverrides {
|
||||
me?: number
|
||||
base?: string
|
||||
editing?: BudgetItem | null
|
||||
prefill?: { name?: string; category?: string; amount?: number; reservationId?: number; placeId?: number }
|
||||
prefill?: ExpensePrefill
|
||||
}
|
||||
|
||||
function renderSheet(overrides: SheetOverrides = {}) {
|
||||
@@ -181,6 +182,14 @@ describe('MCostSheet', () => {
|
||||
}))
|
||||
})
|
||||
|
||||
it('FE-MOB-COSTSH-003c: a scanned receipt opens with its currency and its photo waiting to be attached', () => {
|
||||
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
|
||||
renderSheet({ prefill: { name: 'Sushi Dai', amount: 4200, currency: 'JPY', date: '2026-09-20', receiptFiles: [photo] } })
|
||||
expect(nameField()).toHaveValue('Sushi Dai')
|
||||
expect(screen.getByDisplayValue('4200')).toBeInTheDocument()
|
||||
expect(screen.getByText('bill.jpg')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('FE-MOB-COSTSH-003b: a place prefill carries the place link into the payload (#1298)', async () => {
|
||||
renderSheet({ prefill: { name: 'Louvre', category: 'activities', placeId: 12 } })
|
||||
expect(nameField()).toHaveValue('Louvre')
|
||||
|
||||
@@ -57,6 +57,7 @@ import { StorageModule } from './storage/storage.module';
|
||||
import { BackupModule } from './backup/backup.module';
|
||||
import { BookingImportModule } from './booking-import/booking-import.module';
|
||||
import { ReservationImportModule } from './reservation-import/reservation-import.module';
|
||||
import { ReceiptScanModule } from './receipt-scan/receipt-scan.module';
|
||||
import { LlmParseModule } from './llm-parse/llm-parse.module';
|
||||
import { AuthModule } from './auth/auth.module';
|
||||
import { OidcModule } from './oidc/oidc.module';
|
||||
@@ -82,7 +83,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
|
||||
* migrated.
|
||||
*/
|
||||
@Module({
|
||||
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
|
||||
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, ReceiptScanModule, LlmParseModule, ManagedExtModule],
|
||||
providers: [
|
||||
// Default-deny: a route is authenticated unless it carries @Public() or
|
||||
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
|
||||
|
||||
@@ -4,6 +4,7 @@ import { PermissionsService } from '../permissions/permissions.service';
|
||||
import { ReservationsService } from '../reservations/reservations.service';
|
||||
import { PlacesService } from '../places/places.service';
|
||||
import { BudgetService } from '../budget/budget.service';
|
||||
import { imageMimeType } from '../llm-parse/image-input';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { MapsService } from '../maps/maps.service';
|
||||
@@ -56,6 +57,11 @@ export class BookingImportService {
|
||||
return this.llmParse.isAvailable(userId);
|
||||
}
|
||||
|
||||
/** Whether a photo this user imports can be read (see LlmParseService.readsImages). */
|
||||
readsImages(userId: number): Promise<boolean> {
|
||||
return this.llmParse.readsImages(userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse uploaded files and return a preview list. Does NOT persist anything.
|
||||
* Runs kitinerary first; depending on `mode`, falls back to the LLM:
|
||||
@@ -155,8 +161,10 @@ export class BookingImportService {
|
||||
let kiItems: KiReservation[] = [];
|
||||
let aiUsed = false;
|
||||
|
||||
// Stage 1: kitinerary (skipped entirely when forcing AI).
|
||||
if (mode !== 'force-ai' && kitineraryAvailable) {
|
||||
// Stage 1: kitinerary (skipped entirely when forcing AI, and for a photo,
|
||||
// which only a model can read).
|
||||
const photo = imageMimeType(file.originalname) !== null;
|
||||
if (mode !== 'force-ai' && kitineraryAvailable && !photo) {
|
||||
try {
|
||||
kiItems = await this.extractor.extract(file.buffer, file.originalname);
|
||||
} catch (err) {
|
||||
|
||||
@@ -2,7 +2,8 @@ import { Injectable } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import { BookingImportService } from './booking-import.service';
|
||||
import type { BookingImportMode, BookingImportPreviewResponse, TrekWsPayload } from '@trek/shared';
|
||||
import { LlmParseService } from '../llm-parse/llm-parse.service';
|
||||
import type { BookingImportMode, BookingImportPreviewResponse, ReceiptScanResult, TrekWsPayload } from '@trek/shared';
|
||||
|
||||
type JobStatus = 'running' | 'done' | 'error';
|
||||
|
||||
@@ -13,7 +14,7 @@ interface ImportJob {
|
||||
status: JobStatus;
|
||||
done: number;
|
||||
total: number;
|
||||
result?: BookingImportPreviewResponse;
|
||||
result?: BookingImportPreviewResponse | ReceiptScanResult;
|
||||
error?: string;
|
||||
createdAt: number;
|
||||
}
|
||||
@@ -28,7 +29,9 @@ const JOB_TTL_MS = 10 * 60_000;
|
||||
* user's sockets via `broadcastToUser` (which reaches them on ANY page, not just the
|
||||
* trip room). This is what lets the upload modal close at once and a background widget
|
||||
* track the work while the user keeps navigating. The actual parsing is the same
|
||||
* `BookingImportService.preview` the synchronous endpoint uses.
|
||||
* `BookingImportService.preview` the synchronous endpoint uses. A receipt scanned
|
||||
* from the Costs tab rides the same queue, with `LlmParseService.readReceipt` as
|
||||
* its work: it is the same slow model on the same server.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImportJobsService {
|
||||
@@ -36,17 +39,44 @@ export class ImportJobsService {
|
||||
/** Tail of each user's job chain — parses run one at a time per user, not all at once. */
|
||||
private readonly chains = new Map<number, Promise<void>>();
|
||||
|
||||
constructor(private readonly bookingImport: BookingImportService, private readonly realtime: RealtimeService) {}
|
||||
constructor(
|
||||
private readonly bookingImport: BookingImportService,
|
||||
private readonly realtime: RealtimeService,
|
||||
private readonly llmParse: LlmParseService,
|
||||
) {}
|
||||
|
||||
/** Create a job and queue it behind the user's other parses; returns the job id at once. */
|
||||
start(tripId: string, files: Express.Multer.File[], mode: BookingImportMode, userId: number): string {
|
||||
return this.enqueue(tripId, userId, files.length, (job) =>
|
||||
this.bookingImport.preview(files, mode, job.userId, (done, total, fileName) => {
|
||||
job.done = done;
|
||||
this.push(job, 'import:progress', { status: 'running', done, total, fileName });
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** Queue the read of one receipt photo, behind the same user's other parses. */
|
||||
startReceipt(tripId: string, file: Express.Multer.File, userId: number): string {
|
||||
return this.enqueue(tripId, userId, 1, async (job) => {
|
||||
const result = await this.llmParse.readReceipt({ buffer: file.buffer, originalName: file.originalname }, job.userId);
|
||||
job.done = 1;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
private enqueue(
|
||||
tripId: string,
|
||||
userId: number,
|
||||
total: number,
|
||||
work: (job: ImportJob) => Promise<BookingImportPreviewResponse | ReceiptScanResult>,
|
||||
): string {
|
||||
const id = randomUUID();
|
||||
const job: ImportJob = { id, tripId, userId, status: 'running', done: 0, total: files.length, createdAt: Date.now() };
|
||||
const job: ImportJob = { id, tripId, userId, status: 'running', done: 0, total, createdAt: Date.now() };
|
||||
this.jobs.set(id, job);
|
||||
// Chain onto the user's previous parse so they run sequentially (one CPU-heavy
|
||||
// inference at a time), while the request returns immediately.
|
||||
const prev = this.chains.get(userId) ?? Promise.resolve();
|
||||
const next = prev.then(() => this.run(job, files, mode)).catch(() => {});
|
||||
const next = prev.then(() => this.run(job, work)).catch(() => {});
|
||||
this.chains.set(userId, next);
|
||||
void next.finally(() => {
|
||||
if (this.chains.get(userId) === next) this.chains.delete(userId);
|
||||
@@ -59,13 +89,10 @@ export class ImportJobsService {
|
||||
return job && job.userId === userId ? job : undefined;
|
||||
}
|
||||
|
||||
private async run(job: ImportJob, files: Express.Multer.File[], mode: BookingImportMode): Promise<void> {
|
||||
private async run(job: ImportJob, work: (job: ImportJob) => Promise<BookingImportPreviewResponse | ReceiptScanResult>): Promise<void> {
|
||||
this.push(job, 'import:progress', { status: 'running', done: 0, total: job.total });
|
||||
try {
|
||||
const result = await this.bookingImport.preview(files, mode, job.userId, (done, total, fileName) => {
|
||||
job.done = done;
|
||||
this.push(job, 'import:progress', { status: 'running', done, total, fileName });
|
||||
});
|
||||
const result = await work(job);
|
||||
job.status = 'done';
|
||||
job.result = result;
|
||||
this.push(job, 'import:done', { result });
|
||||
|
||||
@@ -6,13 +6,14 @@ import { UnreadableLlmResponse } from './openai-compatible.client';
|
||||
|
||||
const MAX_TOKENS = 8192;
|
||||
const ANTHROPIC_VERSION = '2023-06-01';
|
||||
const TOOL_NAME = 'emit_reservations';
|
||||
|
||||
/**
|
||||
* Anthropic Messages API client. Structured output via forced tool-use: a single
|
||||
* `emit_reservations` tool whose `input_schema` is the reservations schema, with
|
||||
* `tool_choice` forcing it — the documented, reliable way to get structured JSON.
|
||||
* PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs).
|
||||
* `emit_<rootKey>` tool (`emit_reservations` by default) whose `input_schema` is
|
||||
* the caller's schema, with `tool_choice` forcing it — the documented, reliable
|
||||
* way to get structured JSON.
|
||||
* PDFs go as native base64 `document` blocks (Anthropic reads scanned PDFs), photos
|
||||
* as `image` blocks.
|
||||
* Raw fetch (no SDK) to match the codebase's HTTP style.
|
||||
*/
|
||||
export class AnthropicClient implements LlmExtractionClient {
|
||||
@@ -24,16 +25,21 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
const base = (input.baseUrl ?? 'https://api.anthropic.com').replace(/(?<!\/)\/+$/, '');
|
||||
const url = `${base}/v1/messages`;
|
||||
|
||||
const rootKey = input.rootKey ?? 'reservations';
|
||||
const toolName = `emit_${rootKey}`;
|
||||
const userText = input.userText ?? USER_TEXT;
|
||||
|
||||
const content: unknown[] = [];
|
||||
if (input.file) {
|
||||
for (const file of input.file ? [input.file, ...(input.pageImages ?? [])] : []) {
|
||||
content.push({
|
||||
type: 'document',
|
||||
source: { type: 'base64', media_type: input.file.mimeType, data: input.file.data.toString('base64') },
|
||||
// A photo is an image block; the document block only takes a PDF.
|
||||
type: file.mimeType.startsWith('image/') ? 'image' : 'document',
|
||||
source: { type: 'base64', media_type: file.mimeType, data: file.data.toString('base64') },
|
||||
});
|
||||
}
|
||||
content.push({
|
||||
type: 'text',
|
||||
text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT,
|
||||
text: input.text ? `${userText}\n\n${input.text}` : userText,
|
||||
});
|
||||
|
||||
const body = {
|
||||
@@ -42,12 +48,12 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
system: input.prompt,
|
||||
tools: [
|
||||
{
|
||||
name: TOOL_NAME,
|
||||
description: 'Return the travel reservations extracted from the document.',
|
||||
name: toolName,
|
||||
description: `Return the ${rootKey} extracted from the document.`,
|
||||
input_schema: input.jsonSchema,
|
||||
},
|
||||
],
|
||||
tool_choice: { type: 'tool', name: TOOL_NAME },
|
||||
tool_choice: { type: 'tool', name: toolName },
|
||||
messages: [{ role: 'user', content }],
|
||||
};
|
||||
|
||||
@@ -78,7 +84,7 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
|
||||
const data = (await res.json()) as {
|
||||
stop_reason?: string;
|
||||
content?: { type: string; name?: string; input?: { reservations?: unknown } }[];
|
||||
content?: { type: string; name?: string; input?: Record<string, unknown> }[];
|
||||
};
|
||||
|
||||
if (data.stop_reason === 'refusal') {
|
||||
@@ -95,9 +101,9 @@ export class AnthropicClient implements LlmExtractionClient {
|
||||
);
|
||||
}
|
||||
|
||||
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === TOOL_NAME);
|
||||
if (!toolUse) throw new UnreadableLlmResponse(`the model answered without calling ${TOOL_NAME}`);
|
||||
return toReservationList(toolUse.input?.reservations);
|
||||
const toolUse = data.content?.find(b => b.type === 'tool_use' && b.name === toolName);
|
||||
if (!toolUse) throw new UnreadableLlmResponse(`the model answered without calling ${toolName}`);
|
||||
return toReservationList(toolUse.input?.[rootKey], rootKey);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -110,7 +110,7 @@ function clean(value: unknown): unknown {
|
||||
* treating the right-most separator as the decimal point. Returns null when there
|
||||
* is no parseable amount.
|
||||
*/
|
||||
function parseAmount(raw: unknown): number | null {
|
||||
export function parseAmount(raw: unknown): number | null {
|
||||
if (typeof raw === 'number') return Number.isFinite(raw) ? raw : null;
|
||||
if (typeof raw !== 'string') return null;
|
||||
let s = raw.replace(/[^\d.,]/g, '');
|
||||
|
||||
@@ -84,18 +84,21 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
const base = (input.baseUrl ?? 'https://api.openai.com/v1').replace(/(?<!\/)\/+$/, '');
|
||||
const url = `${base}/chat/completions`;
|
||||
const nuextract = isNuExtractModel(input.model);
|
||||
const rootKey = input.rootKey ?? 'reservations';
|
||||
const userText = input.userText ?? USER_TEXT;
|
||||
|
||||
const userContent: unknown[] = nuextract
|
||||
? [{ type: 'text', text: buildNuExtractUserText(input.text ?? '') }]
|
||||
: [{ type: 'text', text: input.text ? `${USER_TEXT}\n\n${input.text}` : USER_TEXT }];
|
||||
: [{ type: 'text', text: input.text ? `${userText}\n\n${input.text}` : userText }];
|
||||
// Only genuine images go natively (as image_url) — OpenAI-compatible servers
|
||||
// (notably Ollama) reject `file`/PDF content parts. PDFs reach this client as
|
||||
// pre-extracted text (see llm-parse.service.ts), never as bytes.
|
||||
if (!nuextract && input.file && input.file.mimeType.startsWith('image/')) {
|
||||
const b64 = input.file.data.toString('base64');
|
||||
// pre-extracted text (see llm-parse.service.ts), or a scan as its pages drawn
|
||||
// as images, never as PDF bytes.
|
||||
for (const file of !nuextract && input.file ? [input.file, ...(input.pageImages ?? [])] : []) {
|
||||
if (!file.mimeType.startsWith('image/')) continue;
|
||||
userContent.push({
|
||||
type: 'image_url',
|
||||
image_url: { url: `data:${input.file.mimeType};base64,${b64}` },
|
||||
image_url: { url: `data:${file.mimeType};base64,${file.data.toString('base64')}` },
|
||||
});
|
||||
}
|
||||
|
||||
@@ -130,7 +133,7 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
...baseBody,
|
||||
response_format: shape.jsonObject
|
||||
? { type: 'json_object' as const }
|
||||
: { type: 'json_schema' as const, json_schema: { name: 'reservations', schema: input.jsonSchema, strict: false } },
|
||||
: { type: 'json_schema' as const, json_schema: { name: rootKey, schema: input.jsonSchema, strict: false } },
|
||||
};
|
||||
};
|
||||
|
||||
@@ -185,7 +188,7 @@ export class OpenAiCompatibleClient implements LlmExtractionClient {
|
||||
choices?: { message?: { content?: string } }[];
|
||||
};
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
return nuextract ? parseNuExtract(content) : parseReservations(content);
|
||||
return nuextract ? parseNuExtract(content) : parseReservations(content, rootKey);
|
||||
}
|
||||
|
||||
private async send(url: string, body: unknown, apiKey?: string): Promise<Response> {
|
||||
@@ -216,9 +219,9 @@ function parseNuExtract(content: string | undefined | null): Record<string, unkn
|
||||
|
||||
const USER_TEXT = 'Extract every travel reservation from the following document as schema.org JSON-LD.';
|
||||
|
||||
/** Tolerant parse: strip code fences, JSON(5).parse, pull `reservations`. */
|
||||
function parseReservations(content: string | undefined | null): Record<string, unknown>[] {
|
||||
return toReservationList(readAnswer(content));
|
||||
/** Tolerant parse: strip code fences, JSON(5).parse, pull the `rootKey` array. */
|
||||
function parseReservations(content: string | undefined | null, rootKey: string): Record<string, unknown>[] {
|
||||
return toReservationList(readAnswer(content), rootKey);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { extname } from 'node:path';
|
||||
import { Jimp } from 'jimp';
|
||||
import { PDFParse } from 'pdf-parse';
|
||||
import { LLM_PHOTO_EXTENSIONS } from '@trek/shared';
|
||||
|
||||
/** A photographed document on its way to a vision model, by the extensions shared/ lists. */
|
||||
const IMAGE_MIME_BY_EXT: Record<(typeof LLM_PHOTO_EXTENSIONS)[number], string> = {
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.webp': 'image/webp',
|
||||
};
|
||||
|
||||
export const IMAGE_EXTENSIONS: readonly string[] = LLM_PHOTO_EXTENSIONS;
|
||||
|
||||
/** The image MIME type for a file name, or null when it is not a photo TREK sends. */
|
||||
export function imageMimeType(fileName: string): string | null {
|
||||
return (IMAGE_MIME_BY_EXT as Record<string, string>)[extname(fileName).toLowerCase()] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Long edge in pixels. A phone photo is around twelve megapixels, which a model
|
||||
* pays for in prompt tokens without reading the print any better: the same
|
||||
* receipt on a local qwen3.5:4b took 252s whole and 37-124s at this size.
|
||||
*/
|
||||
export const IMAGE_MAX_EDGE = 1600;
|
||||
const IMAGE_JPEG_QUALITY = 82;
|
||||
|
||||
/**
|
||||
* The image, no larger than the model needs. Returns the bytes unchanged when
|
||||
* they are already small enough, or when they cannot be decoded (Jimp has no
|
||||
* WebP): a photo that might be read beats one refused over a failed resize.
|
||||
*/
|
||||
export async function capImage(data: Buffer, mimeType: string): Promise<{ data: Buffer; mimeType: string }> {
|
||||
try {
|
||||
const image = await Jimp.read(data);
|
||||
if (Math.max(image.width, image.height) <= IMAGE_MAX_EDGE) return { data, mimeType };
|
||||
if (image.width >= image.height) image.resize({ w: IMAGE_MAX_EDGE });
|
||||
else image.resize({ h: IMAGE_MAX_EDGE });
|
||||
const resized = await image.getBuffer('image/jpeg', { quality: IMAGE_JPEG_QUALITY });
|
||||
return { data: Buffer.from(resized), mimeType: 'image/jpeg' };
|
||||
} catch {
|
||||
return { data, mimeType };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pages of a PDF read the only way a scan can be: as pictures. The booking
|
||||
* details sit on the first pages, and every page costs the model a photo's
|
||||
* worth of prompt, so only the first two are drawn.
|
||||
*/
|
||||
export const PDF_PAGES_AS_IMAGES = 2;
|
||||
|
||||
/** The first pages of a PDF drawn as images, capped like a photo; empty when none could be drawn. */
|
||||
export async function renderPdfPages(data: Buffer): Promise<{ data: Buffer; mimeType: string }[]> {
|
||||
const parser = new PDFParse({ data: new Uint8Array(data) });
|
||||
try {
|
||||
const shot = await parser.getScreenshot({ first: PDF_PAGES_AS_IMAGES, desiredWidth: IMAGE_MAX_EDGE, imageBuffer: true, imageDataUrl: false });
|
||||
const pages: { data: Buffer; mimeType: string }[] = [];
|
||||
for (const page of shot.pages) {
|
||||
if (page.data?.length) pages.push(await capImage(Buffer.from(page.data), 'image/png'));
|
||||
}
|
||||
return pages;
|
||||
} finally {
|
||||
await parser.destroy().catch(() => {});
|
||||
}
|
||||
}
|
||||
@@ -60,13 +60,16 @@ export function parseLenientJson(content: string | undefined | null): unknown {
|
||||
* would have been accepted inside an array is accepted alone rather than thrown
|
||||
* away without a word (#2375). It is still not guessing: an object without that
|
||||
* key stays refused.
|
||||
*
|
||||
* `rootKey` names the wrapper for a caller that asks for another list than
|
||||
* reservations (the receipt read asks for `receipts`).
|
||||
*/
|
||||
export function toReservationList(value: unknown): Record<string, unknown>[] {
|
||||
export function toReservationList(value: unknown, rootKey = 'reservations'): Record<string, unknown>[] {
|
||||
const list = (v: unknown): Record<string, unknown>[] | null => {
|
||||
if (Array.isArray(v)) return v as Record<string, unknown>[];
|
||||
if (v && typeof v === 'object') {
|
||||
const o = v as { reservations?: unknown; '@type'?: unknown };
|
||||
if (Array.isArray(o.reservations)) return o.reservations as Record<string, unknown>[];
|
||||
const o = v as Record<string, unknown>;
|
||||
if (Array.isArray(o[rootKey])) return o[rootKey] as Record<string, unknown>[];
|
||||
if (typeof o['@type'] === 'string') return [v as Record<string, unknown>];
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Controller, Get, UseGuards } from '@nestjs/common';
|
||||
import type { LlmCapabilitiesResponse } from '@trek/shared';
|
||||
import type { User } from '../../types';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { LlmParseService } from './llm-parse.service';
|
||||
|
||||
/**
|
||||
* What the caller's AI model can be handed. Per user, unlike the public
|
||||
* /api/health/features: whose model applies depends on who is asking. The
|
||||
* photo pickers read it to decide whether a photo is offered at all.
|
||||
*/
|
||||
@Controller('api/llm')
|
||||
@UseGuards(JwtAuthGuard)
|
||||
export class LlmCapabilitiesController {
|
||||
constructor(private readonly llmParse: LlmParseService) {}
|
||||
|
||||
@Get('capabilities')
|
||||
async capabilities(@CurrentUser() user: User): Promise<LlmCapabilitiesResponse> {
|
||||
return { images: await this.llmParse.readsImages(user.id) };
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { asLlmVision } from '@trek/shared';
|
||||
import { decryptLlmApiKey, LLM_PROVIDERS, type LlmProvider, type ResolvedLlmConfig } from './llm-config';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { SettingsService } from '../settings/settings.service';
|
||||
@@ -53,7 +54,7 @@ export class LlmConfigResolver {
|
||||
model,
|
||||
baseUrl: typeof cfg.baseUrl === 'string' && cfg.baseUrl.trim() ? cfg.baseUrl.trim() : undefined,
|
||||
apiKey: decryptLlmApiKey(cfg.apiKey),
|
||||
multimodal: cfg.multimodal === true,
|
||||
vision: asLlmVision(cfg.vision),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -88,7 +89,9 @@ export class LlmConfigResolver {
|
||||
model,
|
||||
baseUrl,
|
||||
apiKey,
|
||||
multimodal: settings.llm_multimodal === true,
|
||||
// The personal switch has only ever been on or off, and was saved as the
|
||||
// person set it, so it is read as exactly that.
|
||||
vision: settings.llm_multimodal === true ? 'on' : 'off',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { asLlmVision, type LlmVision } from '@trek/shared';
|
||||
import { maybe_encrypt_api_key, decrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
|
||||
/**
|
||||
@@ -21,7 +22,7 @@ export interface ResolvedLlmConfig {
|
||||
model: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
multimodal: boolean;
|
||||
vision: LlmVision;
|
||||
}
|
||||
|
||||
/** Shape of the admin instance config stored in `addons.config` (apiKey encrypted). */
|
||||
@@ -30,7 +31,11 @@ export interface LlmAddonConfig {
|
||||
model?: string;
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
multimodal?: boolean;
|
||||
/**
|
||||
* Replaces `multimodal`, which the admin screens wrote as `false` on every
|
||||
* save whatever the model: a stored `false` means nothing, so it is not read.
|
||||
*/
|
||||
vision?: LlmVision;
|
||||
}
|
||||
|
||||
export const LLM_PROVIDERS: LlmProvider[] = ['local', 'openai', 'anthropic'];
|
||||
@@ -46,6 +51,7 @@ export function prepareLlmAddonConfigForWrite(
|
||||
existingStored: Record<string, unknown> | undefined,
|
||||
): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = { ...incoming };
|
||||
if ('vision' in incoming) out.vision = asLlmVision(incoming.vision);
|
||||
const key = incoming.apiKey;
|
||||
if (key === undefined || key === null || key === '' || key === MASKED_VALUE) {
|
||||
// Keep the existing encrypted key untouched (mask echoed or no key supplied).
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
import { Injectable, HttpException } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
import { safeFetchLlm } from '../../utils/ssrfGuard';
|
||||
import { readCappedJson } from '../../utils/cappedFetch';
|
||||
|
||||
/** `/api/show` carries the licence and the template: tens of KB, never megabytes. */
|
||||
const MAX_SHOW_BYTES = 1024 * 1024;
|
||||
const SHOW_TIMEOUT_MS = 5_000;
|
||||
/** A model's capabilities only change when it is re-pulled; ten minutes of staleness is harmless. */
|
||||
const CAPABILITIES_TTL_MS = 10 * 60_000;
|
||||
/** An unreachable server is asked again soon, so a restart does not cost ten minutes. */
|
||||
const CAPABILITIES_MISS_TTL_MS = 60_000;
|
||||
|
||||
const showResponseSchema = z.object({ capabilities: z.array(z.string()).optional() });
|
||||
|
||||
/**
|
||||
* Admin helpers for managing a local OpenAI-compatible LLM server (Ollama).
|
||||
@@ -11,6 +23,8 @@ import { safeFetchLlm } from '../../utils/ssrfGuard';
|
||||
*/
|
||||
@Injectable()
|
||||
export class LlmLocalService {
|
||||
private readonly capabilityCache = new Map<string, { capabilities: string[] | null; expires: number }>();
|
||||
|
||||
/** Derive the Ollama root from a configured base URL (strip a trailing /v1). */
|
||||
ollamaRoot(baseUrl: string | undefined): string {
|
||||
const raw = (baseUrl ?? 'http://localhost:11434').trim();
|
||||
@@ -65,4 +79,42 @@ export class LlmLocalService {
|
||||
if (!res.ok || !res.body) throw new HttpException({ error: `Pull failed (${res.status})` }, 502);
|
||||
return res.body;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the local server says the model can do (`['completion', 'vision', …]`),
|
||||
* or null when it cannot say: a bad URL, a server that is down, a model it
|
||||
* does not have, an answer that is not what Ollama sends. Never throws, since
|
||||
* every caller treats "unknown" as "no", and remembers the answer per server
|
||||
* and model so a page load does not become a request to the model server.
|
||||
*/
|
||||
async modelCapabilities(baseUrl: string | undefined, model: string): Promise<string[] | null> {
|
||||
const now = Date.now();
|
||||
const key = `${baseUrl ?? ''}\n${model}`;
|
||||
const cached = this.capabilityCache.get(key);
|
||||
if (cached && cached.expires > now) return cached.capabilities;
|
||||
|
||||
let capabilities: string[] | null = null;
|
||||
try {
|
||||
const root = this.ollamaRoot(baseUrl);
|
||||
const res = await safeFetchLlm(`${root}/api/show`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model }),
|
||||
signal: AbortSignal.timeout(SHOW_TIMEOUT_MS),
|
||||
});
|
||||
if (res.ok) {
|
||||
const parsed = showResponseSchema.safeParse(await readCappedJson<unknown>(res, MAX_SHOW_BYTES));
|
||||
if (parsed.success) capabilities = parsed.data.capabilities ?? [];
|
||||
} else {
|
||||
void res.body?.cancel().catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
capabilities = null;
|
||||
}
|
||||
this.capabilityCache.set(key, {
|
||||
capabilities,
|
||||
expires: now + (capabilities ? CAPABILITIES_TTL_MS : CAPABILITIES_MISS_TTL_MS),
|
||||
});
|
||||
return capabilities;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { Module } from '@nestjs/common';
|
||||
import { LlmParseService } from './llm-parse.service';
|
||||
import { LlmLocalService } from './llm-local.service';
|
||||
import { LlmLocalController } from './llm-local.controller';
|
||||
import { LlmCapabilitiesController } from './llm-capabilities.controller';
|
||||
import { LlmConfigResolver } from './llm-config.resolver';
|
||||
import { SettingsModule } from '../settings/settings.module';
|
||||
import { AddonsModule } from '../addons/addons.module';
|
||||
@@ -13,7 +14,7 @@ import { AppConfigModule } from '../app-config/app-config.module';
|
||||
*/
|
||||
@Module({
|
||||
imports: [AppConfigModule, SettingsModule, AddonsModule],
|
||||
controllers: [LlmLocalController],
|
||||
controllers: [LlmLocalController, LlmCapabilitiesController],
|
||||
providers: [LlmParseService, LlmLocalService, LlmConfigResolver],
|
||||
exports: [LlmParseService, LlmConfigResolver],
|
||||
})
|
||||
|
||||
@@ -1,13 +1,25 @@
|
||||
import type { KiReservation } from '../booking-import/kitinerary.types';
|
||||
import { createLlmClient } from './llm-client.factory';
|
||||
import { LlmConfigResolver } from './llm-config.resolver';
|
||||
import type { ResolvedLlmConfig } from './llm-config';
|
||||
import { LlmLocalService } from './llm-local.service';
|
||||
import { capImage, imageMimeType, renderPdfPages } from './image-input';
|
||||
import { extractEnforced } from './router/ollama-format.client';
|
||||
import {
|
||||
buildReceiptPrompt,
|
||||
RECEIPT_JSON_SCHEMA,
|
||||
RECEIPT_LIST_JSON_SCHEMA,
|
||||
RECEIPT_ROOT_KEY,
|
||||
RECEIPT_USER_TEXT,
|
||||
toReceiptRead,
|
||||
} from './receipt-read';
|
||||
import { buildSystemPrompt, KI_RESERVATION_JSON_SCHEMA } from './llm-prompt';
|
||||
import type { LlmExtractionInput } from './llm-provider.interface';
|
||||
import { isPdf, extractText } from './text-extract';
|
||||
import { routeExtraction, detectFlightNumbers, extractTotalPrice } from './router/extraction-router';
|
||||
import { routeExtraction, routeImageExtraction, detectFlightNumbers, extractTotalPrice } from './router/extraction-router';
|
||||
import { toIsoCurrency } from './currency-code';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { kiReservationSchema } from '@trek/shared';
|
||||
import { kiReservationSchema, type ReceiptScanResult } from '@trek/shared';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
|
||||
const MIME_BY_EXT: Record<string, string> = {
|
||||
@@ -21,7 +33,8 @@ export interface LlmParseResult {
|
||||
|
||||
/**
|
||||
* Orchestrates the LLM fallback: resolve config → pick client → build input
|
||||
* (native bytes vs extracted text by the `multimodal` flag) → call provider →
|
||||
* (a photo as an image when the model reads one, a PDF natively for Anthropic,
|
||||
* extracted text otherwise) → call provider →
|
||||
* validate the response → return schema.org `KiReservation[]` for the shared
|
||||
* mapper. Never throws for content/provider reasons — degrades to `[]` + a
|
||||
* warning, mirroring the kitinerary extractor's tolerance.
|
||||
@@ -31,6 +44,7 @@ export class LlmParseService {
|
||||
constructor(
|
||||
private readonly llmConfig: LlmConfigResolver,
|
||||
private readonly env: RuntimeEnvService,
|
||||
private readonly local: LlmLocalService,
|
||||
) {}
|
||||
|
||||
/** True when the addon is enabled AND a usable config resolves for this user. */
|
||||
@@ -38,6 +52,77 @@ export class LlmParseService {
|
||||
return this.llmConfig.resolve(userId) !== null;
|
||||
}
|
||||
|
||||
/** Whether a photo this user uploads can be read: AI parsing is set up and its model takes images. */
|
||||
async readsImages(userId: number): Promise<boolean> {
|
||||
const config = this.llmConfig.resolve(userId);
|
||||
return config ? this.visionFor(config) : false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a photographed receipt for the Costs tab. Like parse(), it never throws
|
||||
* for content or provider reasons: a refusal comes back as a null receipt and
|
||||
* a warning naming the file.
|
||||
*/
|
||||
async readReceipt(file: { buffer: Buffer; originalName: string }, userId: number): Promise<ReceiptScanResult> {
|
||||
const config = this.llmConfig.resolve(userId);
|
||||
if (!config) return { receipt: null, warnings: ['AI parsing is not configured'] };
|
||||
const imageType = imageMimeType(file.originalName);
|
||||
if (!imageType) return { receipt: null, warnings: [`${file.originalName}: not a photo`] };
|
||||
if (!(await this.visionFor(config))) {
|
||||
return { receipt: null, warnings: [`${file.originalName}: the configured model does not read images`] };
|
||||
}
|
||||
|
||||
const image = await capImage(file.buffer, imageType);
|
||||
const prompt = buildReceiptPrompt();
|
||||
let raw: unknown;
|
||||
try {
|
||||
if (config.provider === 'local') {
|
||||
// Ollama's own chat API, as the booking router uses: `think: false` and
|
||||
// `num_ctx` exist there and not on /v1, and a hybrid model that is left
|
||||
// to reason spends the whole token budget on it and answers nothing.
|
||||
raw = await extractEnforced({
|
||||
baseUrl: config.baseUrl ?? 'http://localhost:11434/v1',
|
||||
model: config.model,
|
||||
apiKey: config.apiKey,
|
||||
system: prompt,
|
||||
user: RECEIPT_USER_TEXT,
|
||||
schema: RECEIPT_JSON_SCHEMA,
|
||||
images: [image.data.toString('base64')],
|
||||
numCtx: 16384,
|
||||
// A supermarket receipt is a long list of lines.
|
||||
numPredict: 2048,
|
||||
});
|
||||
} else {
|
||||
const list = await createLlmClient(config).extract({
|
||||
prompt,
|
||||
jsonSchema: RECEIPT_LIST_JSON_SCHEMA,
|
||||
rootKey: RECEIPT_ROOT_KEY,
|
||||
userText: RECEIPT_USER_TEXT,
|
||||
model: config.model,
|
||||
baseUrl: config.baseUrl,
|
||||
apiKey: config.apiKey,
|
||||
file: image,
|
||||
});
|
||||
raw = list[0];
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[llm-parse] Receipt read failed for "${file.originalName}" (provider=${config.provider}):`, err instanceof Error ? err.message : err);
|
||||
return { receipt: null, warnings: [`${file.originalName}: AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
|
||||
}
|
||||
|
||||
const receipt = toReceiptRead(raw);
|
||||
return receipt ? { receipt, warnings: [] } : { receipt: null, warnings: [`${file.originalName}: no receipt could be read`] };
|
||||
}
|
||||
|
||||
private async visionFor(config: ResolvedLlmConfig): Promise<boolean> {
|
||||
if (config.vision !== 'auto') return config.vision === 'on';
|
||||
// Only a local Ollama can be asked. A cloud model is not assumed to read
|
||||
// images: an admin who knows it does says so.
|
||||
if (config.provider !== 'local') return false;
|
||||
const capabilities = await this.local.modelCapabilities(config.baseUrl, config.model);
|
||||
return capabilities?.includes('vision') ?? false;
|
||||
}
|
||||
|
||||
async parse(file: { buffer: Buffer; originalName: string }, userId: number): Promise<LlmParseResult> {
|
||||
const config = this.llmConfig.resolve(userId);
|
||||
if (!config) return { kiItems: [], warnings: ['AI parsing is not configured'] };
|
||||
@@ -51,11 +136,22 @@ export class LlmParseService {
|
||||
apiKey: config.apiKey,
|
||||
};
|
||||
|
||||
// Native PDF only for Anthropic (its document block reads text AND scans).
|
||||
// OpenAI-compatible servers (incl. Ollama/NuExtract) can't ingest PDFs/`file`
|
||||
// parts, so every other provider gets extracted text.
|
||||
const imageType = imageMimeType(file.originalName);
|
||||
if (imageType && !(await this.visionFor(config))) {
|
||||
return { kiItems: [], warnings: [`${file.originalName}: the configured model does not read images`] };
|
||||
}
|
||||
|
||||
// A photo goes as an image to every provider. Native PDF only for Anthropic
|
||||
// (its document block reads text AND scans). OpenAI-compatible servers
|
||||
// (incl. Ollama/NuExtract) can't ingest PDFs/`file` parts, so every other
|
||||
// provider gets extracted text.
|
||||
try {
|
||||
if (config.provider === 'anthropic' && isPdf(file.originalName)) {
|
||||
if (imageType) {
|
||||
input.file = await capImage(file.buffer, imageType);
|
||||
console.debug(
|
||||
`[DEBUG] Photo (${input.file.data.length} bytes) sent to ${config.provider}: ${file.originalName}`,
|
||||
);
|
||||
} else if (config.provider === 'anthropic' && isPdf(file.originalName)) {
|
||||
input.file = { mimeType: MIME_BY_EXT['.pdf'], data: file.buffer };
|
||||
console.debug(
|
||||
`[DEBUG] Extracted (native PDF, ${file.buffer.length} bytes) sent to ${config.provider}: ${file.originalName}`,
|
||||
@@ -78,10 +174,18 @@ export class LlmParseService {
|
||||
console.debug(`[DEBUG] Extracted text from ${file.originalName} (${input.text.length} chars):\n`, input.text);
|
||||
}
|
||||
if (!input.text.trim()) {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: no readable text found (a scanned PDF needs a cloud/vision provider)`],
|
||||
};
|
||||
// A scan has no text layer, but a model that reads images can read its pages.
|
||||
const pages = isPdf(file.originalName) && (await this.visionFor(config)) ? await renderPdfPages(file.buffer) : [];
|
||||
if (pages.length === 0) {
|
||||
return {
|
||||
kiItems: [],
|
||||
warnings: [`${file.originalName}: no readable text found (a scanned PDF needs a model that reads images)`],
|
||||
};
|
||||
}
|
||||
input.text = undefined;
|
||||
input.file = pages[0];
|
||||
input.pageImages = pages.slice(1);
|
||||
console.debug(`[DEBUG] Scanned PDF sent to ${config.provider} as ${pages.length} page image(s): ${file.originalName}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -96,13 +200,16 @@ export class LlmParseService {
|
||||
// templates → decompose + grammar-enforced per-reservation extraction → validate
|
||||
// + repair. Far more reliable on small CPU models than the single-shot path below
|
||||
// (which stays for cloud providers, whose strong models handle one-shot well).
|
||||
if (config.provider === 'local' && input.text) {
|
||||
if (config.provider === 'local' && (input.text || input.file)) {
|
||||
try {
|
||||
const routed = await routeExtraction(input.text, {
|
||||
const ctx = {
|
||||
baseUrl: config.baseUrl ?? 'http://localhost:11434/v1',
|
||||
model: config.model,
|
||||
apiKey: config.apiKey,
|
||||
});
|
||||
};
|
||||
const routed = input.file
|
||||
? await routeImageExtraction([input.file, ...(input.pageImages ?? [])].map((f) => f.data), ctx)
|
||||
: await routeExtraction(input.text ?? '', ctx);
|
||||
return { kiItems: routed.kiItems, warnings: [...warnings, ...routed.warnings] };
|
||||
} catch (err) {
|
||||
console.error(`[llm-parse] AI parsing failed for "${file.originalName}" (provider=${config.provider}):`, err instanceof Error ? err.message : err);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/** A single binary file (e.g. a PDF) sent natively to a multimodal provider. */
|
||||
/** A single binary file (a PDF, or a photo) sent natively to a provider that reads it. */
|
||||
export interface LlmExtractionFile {
|
||||
mimeType: string;
|
||||
data: Buffer;
|
||||
@@ -15,8 +15,17 @@ export interface LlmExtractionInput {
|
||||
apiKey?: string;
|
||||
/** Pre-extracted text (text-like files, or text-only-model mode). */
|
||||
text?: string;
|
||||
/** Native binary (PDF) for multimodal providers. */
|
||||
/** Native binary: a PDF for Anthropic, a photo for a model that reads images. */
|
||||
file?: LlmExtractionFile;
|
||||
/** The following pages of a scanned PDF, as images after `file`, which holds the first. */
|
||||
pageImages?: LlmExtractionFile[];
|
||||
/**
|
||||
* The key the answer's array sits under, `jsonSchema` being `{ [rootKey]: [...] }`.
|
||||
* Defaults to `reservations`; the receipt read asks for `receipts`.
|
||||
*/
|
||||
rootKey?: string;
|
||||
/** The user-turn instruction. Defaults to the booking import's. */
|
||||
userText?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { receiptReadSchema, type ReceiptLine, type ReceiptRead } from '@trek/shared';
|
||||
import { toIsoCurrency } from './currency-code';
|
||||
import { parseAmount } from './clients/nuextract';
|
||||
|
||||
/**
|
||||
* Reading a photographed receipt for the Costs tab: the prompt, the schema the
|
||||
* answer is held to, and the step that turns the answer into a `ReceiptRead`.
|
||||
* Pure, so every piece is testable without a model.
|
||||
*
|
||||
* The field list is flat and short on purpose. A till roll is printed small and
|
||||
* photographed under bad light, and a small local model fills a handful of named
|
||||
* fields far more reliably than the nested schema.org shape bookings use.
|
||||
*/
|
||||
|
||||
/** Key the cloud clients read the answer's array under (`{ receipts: [...] }`). */
|
||||
export const RECEIPT_ROOT_KEY = 'receipts';
|
||||
|
||||
export const RECEIPT_USER_TEXT = 'Read the receipt in the attached photo.';
|
||||
|
||||
/** One receipt, flat. Ollama compiles it to a grammar, so the answer has this shape or none. */
|
||||
export const RECEIPT_JSON_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
merchant: { type: 'string' },
|
||||
date: { type: 'string' },
|
||||
total: { type: 'number' },
|
||||
currency: { type: 'string' },
|
||||
items: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: { name: { type: 'string' }, price: { type: 'number' } },
|
||||
required: ['name', 'price'],
|
||||
},
|
||||
},
|
||||
},
|
||||
// Every field is required. Ollama's grammar lets a small model skip an
|
||||
// optional key, and it does: qwen3.5:4b answered the total and the lines of a
|
||||
// receipt printing both its name and its date, and nothing else. A field the
|
||||
// receipt does not show comes back empty, which toReceiptRead reads as absent.
|
||||
required: ['merchant', 'date', 'total', 'currency', 'items'],
|
||||
} as const;
|
||||
|
||||
/** The same receipt wrapped in the list the cloud clients' structured output returns. */
|
||||
export const RECEIPT_LIST_JSON_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: { [RECEIPT_ROOT_KEY]: { type: 'array', items: RECEIPT_JSON_SCHEMA } },
|
||||
required: [RECEIPT_ROOT_KEY],
|
||||
} as const;
|
||||
|
||||
export function buildReceiptPrompt(today: Date = new Date()): string {
|
||||
const todayIso = today.toISOString().slice(0, 10);
|
||||
return [
|
||||
'You read a photographed receipt, bill or invoice: often a paper till roll, possibly crumpled, skewed or in a foreign language.',
|
||||
'Fill "total" with the grand total actually paid, after tax and tip: the final TOTAL or AMOUNT DUE line, not a subtotal.',
|
||||
'Fill "currency" with its ISO 4217 code, from the symbol and the country when the code is not printed.',
|
||||
'Fill "merchant" with the business name printed at the top, and "date" as YYYY-MM-DD.',
|
||||
// A till roll prints "26-08-23" and means one of three dates. The merchant's
|
||||
// country settles day/month order; nothing on the paper settles which pair is
|
||||
// the year, and a receipt is scanned soon after it is paid.
|
||||
`Today is ${todayIso}. When a date could be read several ways, take the reading nearest to today and never one in the future.`,
|
||||
'Fill "items" with the purchased lines as printed, each with its line total as "price". Leave out subtotal, tax, discount, tip and total lines.',
|
||||
'Never invent a value: when the receipt does not show a field, answer an empty string for it (0 for "total", [] for "items"). Numbers are plain numbers: 12.5, not "12,50 €".',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function toText(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
||||
}
|
||||
|
||||
/** A real calendar date as YYYY-MM-DD, or null. */
|
||||
function toDate(value: unknown): string | null {
|
||||
const s = toText(value);
|
||||
if (!s || !/^\d{4}-\d{2}-\d{2}$/.test(s)) return null;
|
||||
const d = new Date(`${s}T00:00:00Z`);
|
||||
return !Number.isNaN(d.getTime()) && d.toISOString().slice(0, 10) === s ? s : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The model's answer as a `ReceiptRead`, or null when it holds nothing a person
|
||||
* could use: no total, no merchant and no line. Each field is kept only when it
|
||||
* reads as what it claims to be, so a garbled one is left for the person to fill
|
||||
* rather than saved wrong.
|
||||
*/
|
||||
export function toReceiptRead(raw: unknown): ReceiptRead | null {
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
|
||||
const node = raw as Record<string, unknown>;
|
||||
const total = parseAmount(node.total);
|
||||
const items: ReceiptLine[] = [];
|
||||
if (Array.isArray(node.items)) {
|
||||
for (const line of node.items) {
|
||||
if (!line || typeof line !== 'object') continue;
|
||||
const name = toText((line as Record<string, unknown>).name);
|
||||
const price = parseAmount((line as Record<string, unknown>).price);
|
||||
if (name && price !== null) items.push({ name, price });
|
||||
}
|
||||
}
|
||||
const read = {
|
||||
merchant: toText(node.merchant),
|
||||
date: toDate(node.date),
|
||||
total: total !== null && total > 0 ? total : null,
|
||||
currency: toIsoCurrency(node.currency) ?? null,
|
||||
items,
|
||||
};
|
||||
if (read.total === null && read.merchant === null && items.length === 0) return null;
|
||||
const parsed = receiptReadSchema.safeParse(read);
|
||||
return parsed.success ? parsed.data : null;
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import type { KiReservation } from '../../booking-import/kitinerary.types';
|
||||
import { nuExtractToKiReservations } from '../clients/nuextract';
|
||||
import { FLAT_SCHEMA_BY_TYPE, FLAT_TYPES, FLIGHTS_ARRAY_SCHEMA, UNION_SINGLE_SCHEMA, type FlatType, type FlatLike } from './flat-schemas';
|
||||
import { extractEnforced } from './ollama-format.client';
|
||||
import { parseLenientJson } from '../lenient-json';
|
||||
|
||||
export interface RouterContext {
|
||||
baseUrl: string;
|
||||
@@ -45,6 +46,24 @@ const TYPE_HINT: Record<FlatType, string> = {
|
||||
event: 'event/attraction. name = the event/ticket, address = the venue, start_time/end_time = full ISO, price/currency = ticket price.',
|
||||
};
|
||||
|
||||
/**
|
||||
* The union prompt for a photo. Text has keywords that pick a type-specific
|
||||
* schema and hint before the call; a photo has none, so the model gets every
|
||||
* type's field meanings at once. Without them qwen3.5:4b read a train ticket
|
||||
* as bare clock times with no stations, and its price as a JSON object inside
|
||||
* the string.
|
||||
*/
|
||||
const PHOTO_UNION_HINT = [
|
||||
'Pick the correct "type", then fill its fields:',
|
||||
'- flight: vehicle_number = flight number, from_code/to_code = IATA codes.',
|
||||
'- train, bus, ferry: from_name/to_name = stations, stops or ports, vehicle_number = train or line number.',
|
||||
'- car: operator = rental company, from_name = pick-up location, to_name = return location.',
|
||||
'- hotel: name, address, checkin_time/checkout_time.',
|
||||
'- restaurant, event: name, address = the venue, start_time (and end_time).',
|
||||
"Every time is full ISO 'YYYY-MM-DDTHH:MM:00' (a transport uses departure_time/arrival_time).",
|
||||
'price = the total paid as a plain number like 49.00, currency = its ISO 4217 code.',
|
||||
].join('\n');
|
||||
|
||||
/**
|
||||
* Keyword → reservation type, so an obvious document skips the costlier union/strong path.
|
||||
*
|
||||
@@ -213,21 +232,23 @@ async function extractFlights(text: string, ctx: RouterContext): Promise<FlatLik
|
||||
|
||||
/** One enforced call for a single reservation — a type-specific schema when the type is
|
||||
* obvious from keywords, else a union schema the model fills with the type it picks. */
|
||||
async function extractSingle(text: string, ctx: RouterContext): Promise<FlatLike> {
|
||||
async function extractSingle(text: string, ctx: RouterContext, images?: Buffer[]): Promise<FlatLike> {
|
||||
const known = detectType(text);
|
||||
const call = (schema: Record<string, unknown>, hint: string) =>
|
||||
extractEnforced({
|
||||
baseUrl: ctx.baseUrl, model: ctx.model, apiKey: ctx.apiKey,
|
||||
system: `Extract the single reservation from the document into the flat fields. ${hint} Omit any field that is truly absent.`,
|
||||
user: `Document:\n${text}`,
|
||||
user: images?.length ? 'The document is the attached photo.' : `Document:\n${text}`,
|
||||
schema,
|
||||
// A photo costs far more prompt tokens than the text of the same page.
|
||||
...(images?.length ? { images: images.map((image) => image.toString('base64')), numCtx: 16384 } : {}),
|
||||
});
|
||||
|
||||
if (known) {
|
||||
const out = (await call(FLAT_SCHEMA_BY_TYPE[known], `It is a ${TYPE_HINT[known]}`)) ?? {};
|
||||
return fixArrivalDate(normalizeDates({ ...out, type: known }));
|
||||
}
|
||||
const out = (await call(UNION_SINGLE_SCHEMA, 'Pick the correct "type".')) ?? {};
|
||||
const out = (await call(UNION_SINGLE_SCHEMA, images?.length ? PHOTO_UNION_HINT : 'Pick the correct "type".')) ?? {};
|
||||
// "I could not tell" used to be written down as "hotel", and a hotel is a
|
||||
// booking, so a ferry voucher imported from the Transport tab opened the
|
||||
// booking form with no transport type to pick and stayed 'other' (#2076).
|
||||
@@ -298,3 +319,38 @@ export async function routeExtraction(text: string, ctx: RouterContext): Promise
|
||||
const kiItems = nuExtractToKiReservations(flats as unknown as Record<string, unknown>[]) as unknown as KiReservation[];
|
||||
return { kiItems, warnings };
|
||||
}
|
||||
|
||||
/**
|
||||
* The router for a photographed document: one enforced call with the photo
|
||||
* attached, or the pages of a scanned PDF, one image each. There is no text to
|
||||
* spot a flight number or a vendor in, so the model picks the type, and the
|
||||
* booking-wide fields come from its answer alone.
|
||||
*/
|
||||
export async function routeImageExtraction(images: Buffer[], ctx: RouterContext): Promise<{ kiItems: KiReservation[]; warnings: string[] }> {
|
||||
let flat: FlatLike;
|
||||
try {
|
||||
flat = unwrapPrice(await extractSingle('', ctx, images));
|
||||
} catch (err) {
|
||||
return { kiItems: [], warnings: [`AI parsing failed — ${err instanceof Error ? err.message : String(err)}`] };
|
||||
}
|
||||
const kiItems = nuExtractToKiReservations([flat] as unknown as Record<string, unknown>[]) as unknown as KiReservation[];
|
||||
return { kiItems, warnings: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* The price as the amount alone. With no document text to take the total from,
|
||||
* the model's `price` is all there is, and qwen3.5:4b writes it as an object
|
||||
* inside the string (`{"amount": 49.00, "currency": "EUR"}`), which the amount
|
||||
* parser then reads as 4900.
|
||||
*/
|
||||
function unwrapPrice(flat: FlatLike): FlatLike {
|
||||
if (typeof flat.price !== 'string' || !flat.price.trim().startsWith('{')) return flat;
|
||||
const inner = parseLenientJson(flat.price);
|
||||
if (!inner || typeof inner !== 'object' || Array.isArray(inner)) return flat;
|
||||
const { amount, currency } = inner as { amount?: unknown; currency?: unknown };
|
||||
const out: FlatLike = { ...flat };
|
||||
if (typeof amount === 'number' || typeof amount === 'string') out.price = String(amount);
|
||||
else delete out.price;
|
||||
if (!out.currency && typeof currency === 'string') out.currency = currency;
|
||||
return out;
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ export interface EnforcedExtractInput {
|
||||
numPredict?: number;
|
||||
/** Context window. 8192 fits a typical multi-section booking; raise for long itineraries. */
|
||||
numCtx?: number;
|
||||
/** Base64 images attached to the user turn, for a vision model reading a photo. */
|
||||
images?: string[];
|
||||
}
|
||||
|
||||
/** Resolve the native API base from a config base URL that may end in `/v1`. */
|
||||
@@ -59,7 +61,7 @@ export async function extractEnforced(input: EnforcedExtractInput): Promise<Reco
|
||||
options: { temperature: 0, num_predict: input.numPredict ?? 512, num_ctx: input.numCtx ?? 8192 },
|
||||
messages: [
|
||||
{ role: 'system', content: input.system },
|
||||
{ role: 'user', content: input.user },
|
||||
{ role: 'user', content: input.user, ...(input.images?.length ? { images: input.images } : {}) },
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Controller, HttpException, Param, Post, UploadedFile, UseGuards, UseInterceptors } from '@nestjs/common';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { memoryStorage } from 'multer';
|
||||
import type { ReceiptScanStartResponse } from '@trek/shared';
|
||||
import type { User } from '../../types';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { PermissionsService } from '../permissions/permissions.service';
|
||||
import { ImportJobsService } from '../booking-import/import-jobs.service';
|
||||
import { LlmParseService } from '../llm-parse/llm-parse.service';
|
||||
import { imageMimeType } from '../llm-parse/image-input';
|
||||
|
||||
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Scan a receipt photo into an expense draft.
|
||||
*
|
||||
* It only reads: the job answers a `ReceiptRead` the client pre-fills the
|
||||
* expense editor with, and the expense (and the photo, attached to it) is saved
|
||||
* by that editor's own path once the person has checked it. The job is a
|
||||
* booking-import job like any other, so the background tasks widget follows it
|
||||
* and `GET …/reservations/import/jobs/:jobId` answers for it.
|
||||
*
|
||||
* Trip access, the addon and the right are checked in the handler, not by
|
||||
* guards: a guard answers before multer has read the body, and a response sent
|
||||
* while the client is still uploading reaches it as ECONNRESET, not as the 404
|
||||
* or 403 (see files.controller.ts). The answers are the guards' own.
|
||||
*
|
||||
* There is no MCP twin, for the reason the booking-file import has none: the
|
||||
* route takes an uploaded photo, and a tool caller has no file to hand it.
|
||||
*/
|
||||
@Controller('api/trips/:tripId/budget/receipt-scan')
|
||||
@UseGuards(JwtAuthGuard)
|
||||
export class ReceiptScanController {
|
||||
constructor(
|
||||
private readonly importJobs: ImportJobsService,
|
||||
private readonly llmParse: LlmParseService,
|
||||
private readonly db: DatabaseService,
|
||||
private readonly permissions: PermissionsService,
|
||||
private readonly addons: AddonsService,
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
@UseInterceptors(FileInterceptor('file', { storage: memoryStorage(), limits: { fileSize: MAX_FILE_BYTES, files: 1 }, defParamCharset: 'utf8' }))
|
||||
async scan(
|
||||
@CurrentUser() user: User,
|
||||
@Param('tripId') tripId: string,
|
||||
@UploadedFile() file: Express.Multer.File | undefined,
|
||||
): Promise<ReceiptScanStartResponse> {
|
||||
const trip = this.db.canAccessTrip(tripId, user.id);
|
||||
if (!trip) throw new HttpException({ error: 'Trip not found' }, 404);
|
||||
if (!this.addons.isAddonEnabled(ADDON_IDS.BUDGET)) throw new HttpException({ error: 'Costs addon is not enabled' }, 404);
|
||||
if (!this.permissions.checkPermission('budget_edit', user.role, trip.user_id, user.id, trip.user_id !== user.id)) {
|
||||
throw new HttpException({ error: 'No permission' }, 403);
|
||||
}
|
||||
if (!file) throw new HttpException({ error: 'No file uploaded' }, 400);
|
||||
if (!imageMimeType(file.originalname)) {
|
||||
throw new HttpException({ error: `Unsupported file type: ${file.originalname}. Accepted: JPG, PNG, WEBP` }, 400);
|
||||
}
|
||||
if (!(await this.llmParse.readsImages(user.id))) {
|
||||
throw new HttpException({ error: 'The configured AI model does not read photos' }, 400);
|
||||
}
|
||||
return { jobId: this.importJobs.startReceipt(tripId, file, user.id) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReceiptScanController } from './receipt-scan.controller';
|
||||
import { BookingImportModule } from '../booking-import/booking-import.module';
|
||||
import { LlmParseModule } from '../llm-parse/llm-parse.module';
|
||||
import { AddonsModule } from '../addons/addons.module';
|
||||
import { PermissionsModule } from '../permissions/permissions.module';
|
||||
|
||||
/** The Costs tab's receipt scan: an HTTP surface over the import job queue. */
|
||||
@Module({
|
||||
imports: [BookingImportModule, LlmParseModule, AddonsModule, PermissionsModule],
|
||||
controllers: [ReceiptScanController],
|
||||
})
|
||||
export class ReceiptScanModule {}
|
||||
@@ -27,8 +27,9 @@ import type { AirtrailImportResult } from '@trek/shared';
|
||||
import { bookingImportModeSchema } from '@trek/shared';
|
||||
import type { BookingImportPreviewItem, BookingImportPreviewResponse, BookingImportConfirmResponse, BookingImportMode } from '@trek/shared';
|
||||
import { BookingImportConfirmDto, BookingImportPreviewDto } from './reservation-import.dto';
|
||||
import { IMAGE_EXTENSIONS, imageMimeType } from '../llm-parse/image-input';
|
||||
|
||||
const ACCEPTED_EXTS = new Set(['.eml', '.pdf', '.pkpass', '.html', '.htm', '.txt']);
|
||||
const ACCEPTED_EXTS = new Set(['.eml', '.pdf', '.pkpass', '.html', '.htm', '.txt', ...IMAGE_EXTENSIONS]);
|
||||
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
||||
const MAX_FILES = 5;
|
||||
|
||||
@@ -99,7 +100,7 @@ export class ReservationImportController {
|
||||
|
||||
|
||||
/** Shared validation for both the sync and async import endpoints; returns the parsed mode. */
|
||||
private validateImport(tripId: string, user: User, files: Express.Multer.File[] | undefined, rawMode?: string): BookingImportMode {
|
||||
private async validateImport(tripId: string, user: User, files: Express.Multer.File[] | undefined, rawMode?: string): Promise<BookingImportMode> {
|
||||
|
||||
const modeResult = bookingImportModeSchema.safeParse(rawMode ?? 'no-ai');
|
||||
if (!modeResult.success) throw new HttpException({ error: 'Invalid mode' }, 400);
|
||||
@@ -118,12 +119,19 @@ export class ReservationImportController {
|
||||
throw new HttpException({ error: `Unsupported file type: ${f.originalname}. Accepted: EML, PDF, PKPass, HTML, TXT` }, 400);
|
||||
}
|
||||
}
|
||||
// A photo has no text layer and no structure: only a model that reads images
|
||||
// can do anything with it, so it is refused up front rather than coming back
|
||||
// as an empty preview.
|
||||
if (files.some((f) => imageMimeType(f.originalname)) && (mode === 'no-ai' || !(await this.bookingImport.readsImages(user.id)))) {
|
||||
throw new HttpException({ error: 'The configured AI model does not read photos' }, 400);
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/trips/:tripId/reservations/import/booking
|
||||
* Accepts up to 5 booking confirmation files (EML, PDF, PKPass, HTML, TXT).
|
||||
* Accepts up to 5 booking confirmation files (EML, PDF, PKPass, HTML, TXT, and
|
||||
* photos when the AI model reads images).
|
||||
* Returns a preview list without persisting anything.
|
||||
*/
|
||||
@RequirePermission('reservation_edit')
|
||||
@@ -135,7 +143,7 @@ export class ReservationImportController {
|
||||
@UploadedFiles() files: Express.Multer.File[] | undefined,
|
||||
@Body() body: BookingImportPreviewDto,
|
||||
): Promise<BookingImportPreviewResponse> {
|
||||
const mode = this.validateImport(tripId, user, files, body?.mode);
|
||||
const mode = await this.validateImport(tripId, user, files, body?.mode);
|
||||
return this.bookingImport.preview(files!, mode, user.id);
|
||||
}
|
||||
|
||||
@@ -155,7 +163,7 @@ export class ReservationImportController {
|
||||
@UploadedFiles() files: Express.Multer.File[] | undefined,
|
||||
@Body() body: BookingImportPreviewDto,
|
||||
): Promise<{ jobId: string }> {
|
||||
const mode = this.validateImport(tripId, user, files, body?.mode);
|
||||
const mode = await this.validateImport(tripId, user, files, body?.mode);
|
||||
const jobId = this.importJobs.start(tripId, files!, mode, user.id);
|
||||
return { jobId };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
/**
|
||||
* Photos through the AI Parsing addon, end to end: the booking import taking a
|
||||
* photo, the Costs receipt scan and its job, and the capability the pickers ask.
|
||||
*
|
||||
* Real: the guards, multer, LlmParseService with the instance's `vision`
|
||||
* setting, the image cap, the OpenAI-compatible client, the import job queue
|
||||
* and its status route. Replaced: the kitinerary binary (absent) and the one
|
||||
* outbound call, `safeFetchLlm`, which answers as the provider would.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
|
||||
import request from 'supertest';
|
||||
import cookieParser from 'cookie-parser';
|
||||
import type { Server } from 'http';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { Jimp } from 'jimp';
|
||||
import { sessionCookie } from './harness';
|
||||
|
||||
const { canAccessTrip, safeFetchLlm } = vi.hoisted(() => ({ canAccessTrip: vi.fn(), safeFetchLlm: vi.fn() }));
|
||||
vi.mock('../../src/db/database', async () => {
|
||||
const { default: Database } = await import('better-sqlite3');
|
||||
const tmp = new Database(':memory:');
|
||||
tmp.exec('PRAGMA journal_mode = WAL');
|
||||
tmp.exec('PRAGMA foreign_keys = ON');
|
||||
return { db: tmp, canAccessTrip, isOwner: vi.fn(() => true), getPlaceWithTags: vi.fn(), closeDb: () => {}, reinitialize: () => {} };
|
||||
});
|
||||
vi.mock('../../src/websocket', () => ({ broadcast: vi.fn(), broadcastToUser: vi.fn() }));
|
||||
vi.mock('../../src/utils/ssrfGuard', async (orig) => ({ ...(await orig<Record<string, unknown>>()), safeFetchLlm }));
|
||||
|
||||
import { db } from '../../src/db/database';
|
||||
import { createTables } from '../../src/db/schema';
|
||||
import { runMigrations } from '../../src/db/migrations';
|
||||
import { DatabaseModule } from '../../src/nest/database/database.module';
|
||||
import { RealtimeModule } from '../../src/nest/realtime/realtime.module';
|
||||
import { ReservationImportModule } from '../../src/nest/reservation-import/reservation-import.module';
|
||||
import { ReceiptScanModule } from '../../src/nest/receipt-scan/receipt-scan.module';
|
||||
import { LlmParseModule } from '../../src/nest/llm-parse/llm-parse.module';
|
||||
import { KitineraryExtractorService } from '../../src/nest/booking-import/kitinerary-extractor.service';
|
||||
import { NotificationsService } from '../../src/nest/notifications/notifications.service';
|
||||
import { PermissionsService } from '../../src/nest/permissions/permissions.service';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
|
||||
|
||||
function setVision(vision: string) {
|
||||
db.prepare("UPDATE addons SET config = ? WHERE id = 'llm_parsing'").run(
|
||||
JSON.stringify({ provider: 'openai', model: 'gpt-4.1-mini', apiKey: '', vision }),
|
||||
);
|
||||
}
|
||||
|
||||
/** The provider's chat-completions answer, carrying `payload` as its content. */
|
||||
function providerAnswers(payload: unknown) {
|
||||
safeFetchLlm.mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ choices: [{ message: { content: JSON.stringify(payload) } }] }),
|
||||
text: async () => '',
|
||||
} as unknown as Response);
|
||||
}
|
||||
|
||||
const sentUserContent = () => JSON.parse((safeFetchLlm.mock.calls[0][1] as RequestInit).body as string).messages[1].content;
|
||||
|
||||
describe('Photos through AI Parsing e2e', () => {
|
||||
let server: Server;
|
||||
let app: Awaited<ReturnType<typeof build>>;
|
||||
let tripId: number;
|
||||
let photo: Buffer;
|
||||
|
||||
async function build() {
|
||||
const moduleRef = await Test.createTestingModule({ imports: [DatabaseModule, RealtimeModule, LlmParseModule, ReservationImportModule, ReceiptScanModule] })
|
||||
.overrideProvider(KitineraryExtractorService)
|
||||
.useValue({ onModuleInit: () => {}, isAvailable: () => false, extract: vi.fn(), describe: () => ({ available: false }) })
|
||||
.overrideProvider(NotificationsService)
|
||||
.useValue({ send: vi.fn().mockResolvedValue(undefined) })
|
||||
.compile();
|
||||
const nest = moduleRef.createNestApplication();
|
||||
nest.use(cookieParser());
|
||||
nest.useGlobalFilters(new TrekExceptionFilter());
|
||||
nest.useGlobalPipes(new ZodValidationPipe());
|
||||
await nest.init();
|
||||
return nest;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
createTables(db);
|
||||
runMigrations(db);
|
||||
db.prepare(
|
||||
"INSERT INTO users (id, username, email, password_hash, role, password_version) VALUES (1, 'e2e-user', 'e2e@example.test', 'x', 'user', 0)",
|
||||
).run();
|
||||
tripId = Number(db.prepare("INSERT INTO trips (user_id, title) VALUES (1, 'Lyon')").run().lastInsertRowid);
|
||||
db.prepare(
|
||||
`INSERT INTO addons (id, name, type, enabled, config) VALUES ('llm_parsing', 'AI Parsing', 'integration', 1, '{}')
|
||||
ON CONFLICT(id) DO UPDATE SET enabled = 1`,
|
||||
).run();
|
||||
db.prepare(
|
||||
`INSERT INTO addons (id, name, type, enabled) VALUES ('budget', 'Costs', 'trip', 1)
|
||||
ON CONFLICT(id) DO UPDATE SET enabled = 1`,
|
||||
).run();
|
||||
photo = Buffer.from(await new Jimp({ width: 40, height: 60, color: 0xffffffff }).getBuffer('image/png'));
|
||||
app = await build();
|
||||
vi.spyOn(app.get(PermissionsService), 'checkPermission').mockReturnValue(true);
|
||||
server = app.getHttpServer();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
canAccessTrip.mockImplementation((id: unknown) => db.prepare('SELECT * FROM trips WHERE id = ?').get(id));
|
||||
safeFetchLlm.mockReset();
|
||||
setVision('on');
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
describe('GET /api/llm/capabilities', () => {
|
||||
it('401 without a cookie', async () => {
|
||||
expect((await request(server).get('/api/llm/capabilities')).status).toBe(401);
|
||||
});
|
||||
|
||||
it('answers the instance setting, and no for a cloud model on Automatic', async () => {
|
||||
expect((await request(server).get('/api/llm/capabilities').set('Cookie', sessionCookie(1))).body).toEqual({ images: true });
|
||||
setVision('auto');
|
||||
expect((await request(server).get('/api/llm/capabilities').set('Cookie', sessionCookie(1))).body).toEqual({ images: false });
|
||||
expect(safeFetchLlm).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('booking import', () => {
|
||||
const upload = () =>
|
||||
request(server)
|
||||
.post(`/api/trips/${tripId}/reservations/import/booking`)
|
||||
.set('Cookie', sessionCookie(1))
|
||||
.field('mode', 'fallback-on-empty')
|
||||
.attach('files', photo, { filename: 'ticket.png', contentType: 'image/png' });
|
||||
|
||||
it('sends a photo to the model as an image, and maps its answer', async () => {
|
||||
providerAnswers({ reservations: [{ '@type': 'TrainReservation', reservationFor: { departureStation: { name: 'Lyon Part-Dieu' }, arrivalStation: { name: 'Paris Gare de Lyon' }, departureTime: '2026-10-03T08:04:00' } }] });
|
||||
const res = await upload();
|
||||
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.items).toHaveLength(1);
|
||||
expect(res.body.files).toEqual([{ fileName: 'ticket.png', aiAvailable: true, aiUsed: true }]);
|
||||
const image = sentUserContent().find((p: { type: string }) => p.type === 'image_url');
|
||||
expect(image.image_url.url).toMatch(/^data:image\/png;base64,/);
|
||||
});
|
||||
|
||||
it('refuses a photo with 400 when the model reads no images', async () => {
|
||||
setVision('off');
|
||||
const res = await upload();
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.error).toBe('The configured AI model does not read photos');
|
||||
expect(safeFetchLlm).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('POST /api/trips/:tripId/budget/receipt-scan', () => {
|
||||
const scan = (name = 'bill.png') =>
|
||||
request(server)
|
||||
.post(`/api/trips/${tripId}/budget/receipt-scan`)
|
||||
.set('Cookie', sessionCookie(1))
|
||||
.attach('file', photo, { filename: name, contentType: name.endsWith('.png') ? 'image/png' : 'application/pdf' });
|
||||
|
||||
it('401 without a cookie, 404 for a trip out of reach', async () => {
|
||||
expect((await request(server).post(`/api/trips/${tripId}/budget/receipt-scan`)).status).toBe(401);
|
||||
canAccessTrip.mockReturnValue(undefined);
|
||||
expect((await scan()).status).toBe(404);
|
||||
});
|
||||
|
||||
it('reads the receipt in a job whose status answers what was read', async () => {
|
||||
providerAnswers({ receipts: [{ merchant: 'Boulangerie du Port', date: '2026-09-21', total: 18.1, currency: 'EUR', items: [{ name: 'Quiche', price: 6.9 }] }] });
|
||||
const res = await scan();
|
||||
expect(res.status).toBe(201);
|
||||
const { jobId } = res.body;
|
||||
|
||||
let status: request.Response | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
status = await request(server).get(`/api/trips/${tripId}/reservations/import/jobs/${jobId}`).set('Cookie', sessionCookie(1));
|
||||
expect(status.body.status).toBe('done');
|
||||
});
|
||||
expect(status!.body.result).toEqual({
|
||||
receipt: { merchant: 'Boulangerie du Port', date: '2026-09-21', total: 18.1, currency: 'EUR', items: [{ name: 'Quiche', price: 6.9 }] },
|
||||
warnings: [],
|
||||
});
|
||||
const body = JSON.parse((safeFetchLlm.mock.calls[0][1] as RequestInit).body as string);
|
||||
expect(body.response_format.json_schema.name).toBe('receipts');
|
||||
});
|
||||
|
||||
it('refuses a file that is not a photo, and any photo when the model reads no images', async () => {
|
||||
expect((await scan('bill.pdf')).status).toBe(400);
|
||||
setVision('off');
|
||||
expect((await scan()).status).toBe(400);
|
||||
expect(safeFetchLlm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('404 while Costs is off', async () => {
|
||||
db.prepare("UPDATE addons SET enabled = 0 WHERE id = 'budget'").run();
|
||||
try {
|
||||
const res = await scan();
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.body.error).toBe('Costs addon is not enabled');
|
||||
} finally {
|
||||
db.prepare("UPDATE addons SET enabled = 1 WHERE id = 'budget'").run();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -79,6 +79,18 @@ describe('BookingImportService.preview', () => {
|
||||
expect(res.files![0].aiUsed).toBe(false);
|
||||
});
|
||||
|
||||
it('fallback-on-empty: a photo goes straight to the LLM, kitinerary is not asked', async () => {
|
||||
const { svc, extractor, llmParse } = make({
|
||||
kit: true, ai: true,
|
||||
extract: async () => [HOTEL_KI],
|
||||
parse: async () => ({ kiItems: [HOTEL_KI], warnings: [] }),
|
||||
});
|
||||
const res = await svc.preview([file('ticket.jpg')], 'fallback-on-empty', 1);
|
||||
expect(extractor.extract).not.toHaveBeenCalled();
|
||||
expect(llmParse.parse).toHaveBeenCalledWith({ buffer: expect.any(Buffer), originalName: 'ticket.jpg' }, 1);
|
||||
expect(res.items[0].needs_review).toBe(true);
|
||||
});
|
||||
|
||||
it('force-ai: skips kitinerary entirely and uses the LLM', async () => {
|
||||
const { svc, extractor, llmParse } = make({
|
||||
kit: true, ai: true,
|
||||
|
||||
@@ -7,8 +7,8 @@ import { ImportJobsService } from '../../../../src/nest/booking-import/import-jo
|
||||
import { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
|
||||
|
||||
type Preview = ReturnType<typeof vi.fn>;
|
||||
function makeService(preview: Preview) {
|
||||
return new ImportJobsService({ preview } as never, new RealtimeService());
|
||||
function makeService(preview: Preview, readReceipt: Preview = vi.fn()) {
|
||||
return new ImportJobsService({ preview } as never, new RealtimeService(), { readReceipt } as never);
|
||||
}
|
||||
const files = (n: number) => Array.from({ length: n }, (_, i) => ({ originalname: `f${i}.pdf` })) as never;
|
||||
const eventsFor = (jobId: string) => broadcastToUser.mock.calls.map((c) => c[1]).filter((p) => p.jobId === jobId);
|
||||
@@ -101,3 +101,30 @@ describe('ImportJobsService', () => {
|
||||
expect(svc.get(id, 99)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ImportJobsService.startReceipt', () => {
|
||||
it('reads the receipt off-request and pushes the read as the result', async () => {
|
||||
const result = { receipt: { merchant: 'Shop', date: null, total: 4, currency: 'EUR', items: [] }, warnings: [] };
|
||||
const readReceipt = vi.fn(async () => result);
|
||||
const svc = makeService(vi.fn(), readReceipt);
|
||||
const id = svc.startReceipt('7', { originalname: 'r.jpg', buffer: Buffer.from('x') } as never, 42);
|
||||
|
||||
await vi.waitFor(() => expect(svc.get(id, 42)?.status).toBe('done'));
|
||||
expect(svc.get(id, 42)).toMatchObject({ result, done: 1, total: 1 });
|
||||
expect(readReceipt).toHaveBeenCalledWith({ buffer: Buffer.from('x'), originalName: 'r.jpg' }, 42);
|
||||
expect(eventsFor(id).find((p) => p.type === 'import:done')).toMatchObject({ result, tripId: '7' });
|
||||
});
|
||||
|
||||
it('queues behind the same user\'s booking parse rather than running beside it', async () => {
|
||||
let release!: () => void;
|
||||
const preview = vi.fn(() => new Promise((resolve) => { release = () => resolve({ items: [] }); }));
|
||||
const readReceipt = vi.fn(async () => ({ receipt: null, warnings: [] }));
|
||||
const svc = makeService(preview, readReceipt);
|
||||
svc.start('7', files(1), 'force-ai', 42);
|
||||
const id = svc.startReceipt('7', { originalname: 'r.jpg', buffer: Buffer.from('x') } as never, 42);
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
expect(readReceipt).not.toHaveBeenCalled();
|
||||
release();
|
||||
await vi.waitFor(() => expect(svc.get(id, 42)?.status).toBe('done'));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@ function make(over: Partial<BookingImportService> = {}) {
|
||||
canEdit: vi.fn(() => true),
|
||||
isAvailable: vi.fn(() => true),
|
||||
aiAvailable: vi.fn(() => true),
|
||||
readsImages: vi.fn(async () => true),
|
||||
preview: vi.fn(async () => ({ items: [], warnings: [], files: [] })),
|
||||
...over,
|
||||
} as unknown as BookingImportService;
|
||||
@@ -55,6 +56,28 @@ describe('ReservationImportController.preview', () => {
|
||||
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'fallback-on-empty', 1);
|
||||
});
|
||||
|
||||
it('takes a photo when the model reads images, without asking otherwise', async () => {
|
||||
const { c, svc } = make();
|
||||
await c.preview(user, 't1', [file('ticket.JPG')], { mode: 'fallback-on-empty' });
|
||||
expect(svc.preview).toHaveBeenCalledWith([expect.anything()], 'fallback-on-empty', 1);
|
||||
(svc.readsImages as ReturnType<typeof vi.fn>).mockClear();
|
||||
await c.preview(user, 't1', [file('a.pdf')], { mode: 'fallback-on-empty' });
|
||||
expect(svc.readsImages).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses a photo with 400 when the model does not read images, or AI is not asked', async () => {
|
||||
const { c, svc } = make({ readsImages: vi.fn(async () => false) });
|
||||
expect(await status(() => c.preview(user, 't1', [file('a.pdf'), file('ticket.png')], { mode: 'fallback-on-empty' }))).toBe(400);
|
||||
const { c: c2 } = make();
|
||||
expect(await status(() => c2.preview(user, 't1', [file('ticket.png')], { mode: 'no-ai' }))).toBe(400);
|
||||
expect(svc.preview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still refuses a file type it does not know, HEIC included', async () => {
|
||||
const { c } = make();
|
||||
expect(await status(() => c.preview(user, 't1', [file('IMG_1.heic')], { mode: 'fallback-on-empty' }))).toBe(400);
|
||||
});
|
||||
|
||||
it('defaults the mode to no-ai when omitted', async () => {
|
||||
const { c, svc } = make();
|
||||
await c.preview(user, 't1', [file()], {});
|
||||
|
||||
@@ -275,6 +275,15 @@ describe('OpenAiCompatibleClient', () => {
|
||||
expect(safeFetchLlmMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('sends each page of a scanned PDF as its own image_url, in order', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
|
||||
const page = (n: string) => ({ mimeType: 'image/png', data: Buffer.from(n) });
|
||||
await new OpenAiCompatibleClient().extract({ ...baseInput, text: undefined, file: page('p1'), pageImages: [page('p2')] });
|
||||
const parts = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string).messages[1].content;
|
||||
const urls = parts.filter((p: { type: string }) => p.type === 'image_url').map((p: { image_url: { url: string } }) => p.image_url.url);
|
||||
expect(urls).toEqual([`data:image/png;base64,${Buffer.from('p1').toString('base64')}`, `data:image/png;base64,${Buffer.from('p2').toString('base64')}`]);
|
||||
});
|
||||
|
||||
it('sends an image natively as image_url but never a file/pdf part', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ choices: [{ message: { content: '{"reservations":[]}' } }] }));
|
||||
await new OpenAiCompatibleClient().extract({ ...baseInput, file: { mimeType: 'image/png', data: Buffer.from('IMG') } });
|
||||
@@ -380,6 +389,28 @@ describe('OpenAiCompatibleClient — NuExtract path', () => {
|
||||
});
|
||||
|
||||
describe('AnthropicClient', () => {
|
||||
it('sends the pages of a scanned PDF as image blocks after the first', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }),
|
||||
);
|
||||
const page = (n: string) => ({ mimeType: 'image/png', data: Buffer.from(n) });
|
||||
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: page('p1'), pageImages: [page('p2')] });
|
||||
const content = JSON.parse((fetchFn.mock.calls[0][1] as RequestInit).body as string).messages[0].content;
|
||||
expect(content.map((b: { type: string }) => b.type)).toEqual(['image', 'image', 'text']);
|
||||
expect(content[1].source.data).toBe(Buffer.from('p2').toString('base64'));
|
||||
});
|
||||
|
||||
it('sends a photo as an image block and a PDF as a document block', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [] } }] }),
|
||||
);
|
||||
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: { mimeType: 'image/jpeg', data: Buffer.from('jpg') } });
|
||||
await new AnthropicClient().extract({ ...baseInput, text: undefined, file: { mimeType: 'application/pdf', data: Buffer.from('pdf') } });
|
||||
const blockOf = (i: number) => JSON.parse((fetchFn.mock.calls[i][1] as RequestInit).body as string).messages[0].content[0];
|
||||
expect(blockOf(0)).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: Buffer.from('jpg').toString('base64') } });
|
||||
expect(blockOf(1).type).toBe('document');
|
||||
});
|
||||
|
||||
it('forces the emit_reservations tool and reads its input', async () => {
|
||||
const fetchFn = mockFetch(() =>
|
||||
jsonResponse({ stop_reason: 'tool_use', content: [{ type: 'tool_use', name: 'emit_reservations', input: { reservations: [{ '@type': 'LodgingReservation' }] } }] }),
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
detectFlightNumbers,
|
||||
fixArrivalDate,
|
||||
routeExtraction,
|
||||
routeImageExtraction,
|
||||
} from '../../../../src/nest/llm-parse/router/extraction-router';
|
||||
|
||||
const CTX = { baseUrl: 'http://ollama:11434/v1', model: 'qwen3:8b' };
|
||||
@@ -257,3 +258,64 @@ describe('printed 12-hour clocks and unreadable types (#2094, #2076)', () => {
|
||||
expect(flats[0].type_guessed).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('routeImageExtraction', () => {
|
||||
it('makes one enforced call with the photo attached and lets the model pick the type', async () => {
|
||||
extractEnforced.mockResolvedValue({ type: 'train', from_name: 'Lyon', to_name: 'Paris', price: '42', currency: 'EUR' });
|
||||
const out = await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(out).toEqual({ kiItems: [{ '@type': 'Mock' }], warnings: [] });
|
||||
expect(extractEnforced).toHaveBeenCalledTimes(1);
|
||||
const call = extractEnforced.mock.calls[0][0];
|
||||
expect(call.images).toEqual([Buffer.from('photo').toString('base64')]);
|
||||
expect(call.numCtx).toBe(16384);
|
||||
expect(call.user).not.toMatch(/Document:/);
|
||||
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ type: 'train', from_name: 'Lyon', price: '42', currency: 'EUR' });
|
||||
});
|
||||
|
||||
it('tells the model what each type\'s fields mean, since a photo has no keyword to pick a schema', async () => {
|
||||
extractEnforced.mockResolvedValue({ type: 'train' });
|
||||
await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
const system = extractEnforced.mock.calls[0][0].system;
|
||||
expect(system).toMatch(/from_name\/to_name = stations/);
|
||||
expect(system).toMatch(/full ISO/);
|
||||
});
|
||||
|
||||
it('reads a price the model wrapped as an object inside the string', async () => {
|
||||
extractEnforced.mockResolvedValue({ type: 'train', price: '{"amount": 49.00, "currency": "EUR"}' });
|
||||
await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ price: '49', currency: 'EUR' });
|
||||
|
||||
mapToKi.mockClear();
|
||||
extractEnforced.mockResolvedValue({ type: 'train', price: '{"amount": "12,50"}', currency: 'CHF' });
|
||||
await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(mapToKi.mock.calls[0][0][0]).toMatchObject({ price: '12,50', currency: 'CHF' });
|
||||
|
||||
mapToKi.mockClear();
|
||||
extractEnforced.mockResolvedValue({ type: 'train', price: '{"note": "see below"}' });
|
||||
await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(mapToKi.mock.calls[0][0][0]).not.toHaveProperty('price');
|
||||
});
|
||||
|
||||
it('leaves a plain price, or one that only looks like an object, as it came', async () => {
|
||||
for (const price of ['49,00 EUR', '{not json']) {
|
||||
mapToKi.mockClear();
|
||||
extractEnforced.mockResolvedValue({ type: 'train', price });
|
||||
await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(mapToKi.mock.calls[0][0][0].price).toBe(price);
|
||||
}
|
||||
});
|
||||
|
||||
it('attaches every page of a scanned PDF to the one call', async () => {
|
||||
extractEnforced.mockResolvedValue({ type: 'hotel' });
|
||||
await routeImageExtraction([Buffer.from('p1'), Buffer.from('p2')], CTX);
|
||||
expect(extractEnforced).toHaveBeenCalledTimes(1);
|
||||
expect(extractEnforced.mock.calls[0][0].images).toEqual([Buffer.from('p1').toString('base64'), Buffer.from('p2').toString('base64')]);
|
||||
});
|
||||
|
||||
it('degrades to a warning when the call throws', async () => {
|
||||
extractEnforced.mockRejectedValue(new Error('timeout'));
|
||||
const out = await routeImageExtraction([Buffer.from('photo')], CTX);
|
||||
expect(out.kiItems).toEqual([]);
|
||||
expect(out.warnings[0]).toMatch(/timeout/);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { Jimp } from 'jimp';
|
||||
import { capImage, imageMimeType, IMAGE_MAX_EDGE, renderPdfPages } from '../../../../src/nest/llm-parse/image-input';
|
||||
|
||||
async function png(w: number, h: number): Promise<Buffer> {
|
||||
const image = new Jimp({ width: w, height: h, color: 0xffffffff });
|
||||
return Buffer.from(await image.getBuffer('image/png'));
|
||||
}
|
||||
|
||||
describe('imageMimeType', () => {
|
||||
it('knows the photo formats a provider reads, whatever the case', () => {
|
||||
expect(imageMimeType('a.jpg')).toBe('image/jpeg');
|
||||
expect(imageMimeType('A.JPEG')).toBe('image/jpeg');
|
||||
expect(imageMimeType('a.png')).toBe('image/png');
|
||||
expect(imageMimeType('a.webp')).toBe('image/webp');
|
||||
});
|
||||
|
||||
it('is null for everything else, HEIC included', () => {
|
||||
for (const name of ['a.heic', 'a.pdf', 'a.txt', 'jpg', 'a.jpg.pdf']) expect(imageMimeType(name)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('capImage', () => {
|
||||
it('shrinks the long edge of a large photo to the cap, as a JPEG', async () => {
|
||||
const out = await capImage(await png(IMAGE_MAX_EDGE * 2, 1000), 'image/png');
|
||||
expect(out.mimeType).toBe('image/jpeg');
|
||||
const back = await Jimp.read(out.data);
|
||||
expect(back.width).toBe(IMAGE_MAX_EDGE);
|
||||
expect(back.height).toBe(500);
|
||||
});
|
||||
|
||||
it('caps a portrait photo by its height', async () => {
|
||||
const back = await Jimp.read((await capImage(await png(800, IMAGE_MAX_EDGE * 2), 'image/png')).data);
|
||||
expect(back.height).toBe(IMAGE_MAX_EDGE);
|
||||
expect(back.width).toBe(400);
|
||||
});
|
||||
|
||||
it('leaves a photo that is already small enough untouched', async () => {
|
||||
const data = await png(400, 300);
|
||||
expect(await capImage(data, 'image/png')).toEqual({ data, mimeType: 'image/png' });
|
||||
});
|
||||
|
||||
it('passes bytes it cannot decode through unchanged', async () => {
|
||||
const data = Buffer.from('RIFF....WEBP');
|
||||
expect(await capImage(data, 'image/webp')).toEqual({ data, mimeType: 'image/webp' });
|
||||
});
|
||||
});
|
||||
|
||||
/** A PDF with no text layer: each page is one JPEG, as a scanner writes it. */
|
||||
async function scannedPdf(pages: number): Promise<Buffer> {
|
||||
const jpg = Buffer.from(await new Jimp({ width: 300, height: 400, color: 0xddddddff }).getBuffer('image/jpeg'));
|
||||
const objects: Buffer[] = [];
|
||||
const kids = Array.from({ length: pages }, (_, i) => `${3 + i * 3} 0 R`).join(' ');
|
||||
objects.push(Buffer.from('<< /Type /Catalog /Pages 2 0 R >>'));
|
||||
objects.push(Buffer.from(`<< /Type /Pages /Kids [${kids}] /Count ${pages} >>`));
|
||||
for (let i = 0; i < pages; i++) {
|
||||
const page = 3 + i * 3;
|
||||
const content = 'q 300 0 0 400 0 0 cm /Im0 Do Q';
|
||||
objects.push(Buffer.from(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /XObject << /Im0 ${page + 2} 0 R >> >> /Contents ${page + 1} 0 R >>`));
|
||||
objects.push(Buffer.from(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`));
|
||||
objects.push(Buffer.concat([
|
||||
Buffer.from(`<< /Type /XObject /Subtype /Image /Width 300 /Height 400 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode /Length ${jpg.length} >>\nstream\n`),
|
||||
jpg,
|
||||
Buffer.from('\nendstream'),
|
||||
]));
|
||||
}
|
||||
const parts: Buffer[] = [Buffer.from('%PDF-1.4\n')];
|
||||
const offsets: number[] = [];
|
||||
let length = parts[0].length;
|
||||
objects.forEach((body, i) => {
|
||||
offsets.push(length);
|
||||
const obj = Buffer.concat([Buffer.from(`${i + 1} 0 obj\n`), body, Buffer.from('\nendobj\n')]);
|
||||
parts.push(obj);
|
||||
length += obj.length;
|
||||
});
|
||||
const rows = offsets.map((o) => `${String(o).padStart(10, '0')} 00000 n \n`).join('');
|
||||
parts.push(Buffer.from(`xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${rows}trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${length}\n%%EOF\n`));
|
||||
return Buffer.concat(parts);
|
||||
}
|
||||
|
||||
describe('renderPdfPages', () => {
|
||||
it('draws the first two pages of a scan as images no larger than a capped photo', async () => {
|
||||
const pages = await renderPdfPages(await scannedPdf(3));
|
||||
expect(pages).toHaveLength(2);
|
||||
for (const page of pages) {
|
||||
const image = await Jimp.read(page.data);
|
||||
expect(Math.max(image.width, image.height)).toBeLessThanOrEqual(IMAGE_MAX_EDGE);
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('throws for bytes that are not a PDF, which the caller reports as unreadable', async () => {
|
||||
await expect(renderPdfPages(Buffer.from('not a pdf'))).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { GUARDS_METADATA } from '@nestjs/common/constants';
|
||||
import { LlmCapabilitiesController } from '../../../../src/nest/llm-parse/llm-capabilities.controller';
|
||||
import { JwtAuthGuard } from '../../../../src/nest/auth/jwt-auth.guard';
|
||||
import type { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
import type { User } from '../../../../src/types';
|
||||
|
||||
describe('LlmCapabilitiesController', () => {
|
||||
it("answers whether the caller's model reads images", async () => {
|
||||
const readsImages = vi.fn(async (id: number) => id === 1);
|
||||
const c = new LlmCapabilitiesController({ readsImages } as unknown as LlmParseService);
|
||||
await expect(c.capabilities({ id: 1 } as User)).resolves.toEqual({ images: true });
|
||||
await expect(c.capabilities({ id: 2 } as User)).resolves.toEqual({ images: false });
|
||||
expect(readsImages).toHaveBeenCalledWith(2);
|
||||
});
|
||||
|
||||
it('needs a session', () => {
|
||||
expect(Reflect.getMetadata(GUARDS_METADATA, LlmCapabilitiesController)).toContain(JwtAuthGuard);
|
||||
});
|
||||
});
|
||||
@@ -64,16 +64,25 @@ describe('resolveLlmConfig', () => {
|
||||
});
|
||||
|
||||
it('uses instance config when present (and decrypts the key)', () => {
|
||||
setInstanceConfig({ provider: 'anthropic', model: 'claude-opus-4-8', apiKey: 'sk-plain', multimodal: true });
|
||||
setInstanceConfig({ provider: 'anthropic', model: 'claude-opus-4-8', apiKey: 'sk-plain', vision: 'on' });
|
||||
expect(resolver.resolve(1)).toEqual({
|
||||
provider: 'anthropic',
|
||||
model: 'claude-opus-4-8',
|
||||
baseUrl: undefined,
|
||||
apiKey: 'sk-plain',
|
||||
multimodal: true,
|
||||
vision: 'on',
|
||||
});
|
||||
});
|
||||
|
||||
it('reads the instance vision setting as auto when it is missing, unknown, or only the old multimodal flag', () => {
|
||||
for (const extra of [{}, { vision: 'maybe' }, { multimodal: false }, { multimodal: true }]) {
|
||||
setInstanceConfig({ provider: 'local', model: 'qwen3.5:4b', ...extra });
|
||||
expect(resolver.resolve(1)?.vision).toBe('auto');
|
||||
}
|
||||
setInstanceConfig({ provider: 'local', model: 'qwen3.5:4b', vision: 'off' });
|
||||
expect(resolver.resolve(1)?.vision).toBe('off');
|
||||
});
|
||||
|
||||
it('instance config with a base URL still wins for a plain user (#1772 does not touch it)', () => {
|
||||
setInstanceConfig({ provider: 'local', model: 'nuextract', baseUrl: 'http://ollama:11434' });
|
||||
setRole('user');
|
||||
@@ -89,7 +98,7 @@ describe('resolveLlmConfig', () => {
|
||||
model: 'claude-sonnet',
|
||||
baseUrl: undefined,
|
||||
apiKey: 'user-key',
|
||||
multimodal: true,
|
||||
vision: 'on',
|
||||
});
|
||||
expect(getDecryptedUserSetting).toHaveBeenCalledWith(7, 'llm_api_key');
|
||||
});
|
||||
@@ -114,7 +123,7 @@ describe('resolveLlmConfig', () => {
|
||||
model: 'gpt-4o-mini',
|
||||
baseUrl: undefined,
|
||||
apiKey: 'sk-user',
|
||||
multimodal: false,
|
||||
vision: 'off',
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { prepareLlmAddonConfigForWrite } from '../../../../src/nest/llm-parse/llm-config';
|
||||
|
||||
describe('prepareLlmAddonConfigForWrite: the vision setting', () => {
|
||||
it('stores a known mode as sent', () => {
|
||||
for (const vision of ['auto', 'on', 'off']) {
|
||||
expect(prepareLlmAddonConfigForWrite({ provider: 'local', vision }, undefined).vision).toBe(vision);
|
||||
}
|
||||
});
|
||||
|
||||
it('stores anything else as auto, so the resolver never meets a value it cannot read', () => {
|
||||
expect(prepareLlmAddonConfigForWrite({ vision: 'maybe' }, undefined).vision).toBe('auto');
|
||||
expect(prepareLlmAddonConfigForWrite({ vision: true }, undefined).vision).toBe('auto');
|
||||
});
|
||||
|
||||
it('adds no vision key to a config that sent none', () => {
|
||||
expect(prepareLlmAddonConfigForWrite({ provider: 'openai', model: 'm' }, undefined)).not.toHaveProperty('vision');
|
||||
});
|
||||
});
|
||||
@@ -68,3 +68,67 @@ describe('LlmLocalService.pull', () => {
|
||||
expect(JSON.parse(init.body)).toEqual({ model: 'nuextract', stream: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmLocalService.modelCapabilities', () => {
|
||||
const showOk = (body: unknown, headers: Record<string, string> = {}) => ({
|
||||
ok: true,
|
||||
headers: { get: (name: string) => headers[name] ?? null },
|
||||
json: async () => body,
|
||||
});
|
||||
|
||||
it('asks /api/show for the model and answers its capabilities', async () => {
|
||||
const fetchFn = mockFetch(async () => showOk({ capabilities: ['completion', 'vision'] }));
|
||||
await expect(svc().modelCapabilities('http://ollama:11434/v1', 'qwen3.5:4b')).resolves.toEqual(['completion', 'vision']);
|
||||
expect(fetchFn.mock.calls[0][0]).toBe('http://ollama:11434/api/show');
|
||||
expect(JSON.parse(fetchFn.mock.calls[0][1].body)).toEqual({ model: 'qwen3.5:4b' });
|
||||
});
|
||||
|
||||
it('asks the server once per server and model, not once per call', async () => {
|
||||
const fetchFn = mockFetch(async () => showOk({ capabilities: ['completion'] }));
|
||||
const s = svc();
|
||||
await s.modelCapabilities('http://ollama:11434', 'qwen3:8b');
|
||||
await s.modelCapabilities('http://ollama:11434', 'qwen3:8b');
|
||||
expect(fetchFn).toHaveBeenCalledTimes(1);
|
||||
await s.modelCapabilities('http://ollama:11434', 'qwen3.5:4b');
|
||||
expect(fetchFn).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('answers null for a URL it cannot use, without throwing and without a request', async () => {
|
||||
await expect(svc().modelCapabilities('not a url', 'm')).resolves.toBeNull();
|
||||
await expect(svc().modelCapabilities('ftp://x', 'm')).resolves.toBeNull();
|
||||
expect(safeFetchLlmMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('answers null for an unreachable server, a model it lacks, or a body that is not Ollama\'s', async () => {
|
||||
mockFetch(async () => { throw new Error('ECONNREFUSED'); });
|
||||
await expect(svc().modelCapabilities('http://a:1', 'm')).resolves.toBeNull();
|
||||
mockFetch(async () => ({ ok: false, status: 404, body: null }));
|
||||
await expect(svc().modelCapabilities('http://b:1', 'm')).resolves.toBeNull();
|
||||
mockFetch(async () => showOk({ capabilities: 'vision' }));
|
||||
await expect(svc().modelCapabilities('http://c:1', 'm')).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('does not read a body that declares itself larger than the cap', async () => {
|
||||
const json = vi.fn(async () => ({ capabilities: ['vision'] }));
|
||||
mockFetch(async () => ({ ...showOk({}), headers: { get: () => String(2 * 1024 * 1024) }, json }));
|
||||
await expect(svc().modelCapabilities('http://d:1', 'm')).resolves.toBeNull();
|
||||
expect(json).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('asks again a minute after a miss, and keeps an answer for ten', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const s = svc();
|
||||
mockFetch(async () => { throw new Error('down'); });
|
||||
await s.modelCapabilities('http://e:1', 'm');
|
||||
vi.advanceTimersByTime(61_000);
|
||||
mockFetch(async () => showOk({ capabilities: ['vision'] }));
|
||||
await expect(s.modelCapabilities('http://e:1', 'm')).resolves.toEqual(['vision']);
|
||||
vi.advanceTimersByTime(9 * 60_000);
|
||||
mockFetch(async () => showOk({ capabilities: [] }));
|
||||
await expect(s.modelCapabilities('http://e:1', 'm')).resolves.toEqual(['vision']);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,30 +10,49 @@ const { createLlmClient, extract } = vi.hoisted(() => {
|
||||
});
|
||||
vi.mock('../../../../src/nest/llm-parse/llm-client.factory', () => ({ createLlmClient }));
|
||||
|
||||
const { extractEnforced } = vi.hoisted(() => ({ extractEnforced: vi.fn() }));
|
||||
|
||||
const { renderPdfPages } = vi.hoisted(() => ({ renderPdfPages: vi.fn() }));
|
||||
vi.mock('../../../../src/nest/llm-parse/image-input', async (orig) => {
|
||||
const actual = await orig() as Record<string, unknown>;
|
||||
return { ...actual, renderPdfPages };
|
||||
});
|
||||
vi.mock('../../../../src/nest/llm-parse/router/ollama-format.client', async (orig) => {
|
||||
const actual = await orig() as Record<string, unknown>;
|
||||
return { ...actual, extractEnforced };
|
||||
});
|
||||
|
||||
const { extractText } = vi.hoisted(() => ({ extractText: vi.fn(async () => 'Flight AB123') }));
|
||||
vi.mock('../../../../src/nest/llm-parse/text-extract', async (orig) => {
|
||||
const actual = await orig() as Record<string, unknown>;
|
||||
return { ...actual, extractText };
|
||||
});
|
||||
|
||||
const { routeExtraction, detectFlightNumbers } = vi.hoisted(() => ({
|
||||
const { routeExtraction, routeImageExtraction, detectFlightNumbers } = vi.hoisted(() => ({
|
||||
routeExtraction: vi.fn(),
|
||||
routeImageExtraction: vi.fn(),
|
||||
detectFlightNumbers: vi.fn(() => [] as string[]),
|
||||
}));
|
||||
// The router's pure helpers stay real: the currency fallback reads the document's
|
||||
// total through extractTotalPrice.
|
||||
vi.mock('../../../../src/nest/llm-parse/router/extraction-router', async (orig) => {
|
||||
const actual = await orig() as Record<string, unknown>;
|
||||
return { ...actual, routeExtraction, detectFlightNumbers };
|
||||
return { ...actual, routeExtraction, routeImageExtraction, detectFlightNumbers };
|
||||
});
|
||||
|
||||
import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service';
|
||||
import type { LlmConfigResolver } from '../../../../src/nest/llm-parse/llm-config.resolver';
|
||||
import type { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import type { LlmLocalService } from '../../../../src/nest/llm-parse/llm-local.service';
|
||||
|
||||
const cfg = (over: Record<string, unknown> = {}) => ({ provider: 'openai', model: 'm', multimodal: false, ...over });
|
||||
const cfg = (over: Record<string, unknown> = {}) => ({ provider: 'openai', model: 'm', vision: 'off', ...over });
|
||||
const llmConfigStub = { resolve: resolveLlmConfig } as unknown as LlmConfigResolver;
|
||||
const svc = () => new LlmParseService(llmConfigStub, { isManaged: () => false } as unknown as RuntimeEnvService);
|
||||
const modelCapabilities = vi.fn();
|
||||
const svc = () => new LlmParseService(
|
||||
llmConfigStub,
|
||||
{ isManaged: () => false } as unknown as RuntimeEnvService,
|
||||
{ modelCapabilities } as unknown as LlmLocalService,
|
||||
);
|
||||
const file = (name: string, body = 'Flight AB123') => ({ buffer: Buffer.from(body), originalName: name });
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -43,6 +62,9 @@ beforeEach(() => {
|
||||
extractText.mockResolvedValue('Flight AB123');
|
||||
detectFlightNumbers.mockReturnValue([]);
|
||||
routeExtraction.mockResolvedValue({ kiItems: [{ '@type': 'LodgingReservation' }], warnings: [] });
|
||||
routeImageExtraction.mockResolvedValue({ kiItems: [{ '@type': 'TrainReservation' }], warnings: [] });
|
||||
modelCapabilities.mockResolvedValue(null);
|
||||
renderPdfPages.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
describe('LlmParseService', () => {
|
||||
@@ -284,3 +306,165 @@ describe('LlmParseService: cleaning up an AI answer (#2477)', () => {
|
||||
expect(res.kiItems).toEqual([plain]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmParseService: a photo', () => {
|
||||
it('readsImages follows on and off whatever the provider, and never asks a server for them', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on' }));
|
||||
await expect(svc().readsImages(1)).resolves.toBe(true);
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'off' }));
|
||||
await expect(svc().readsImages(1)).resolves.toBe(false);
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'openai', vision: 'on' }));
|
||||
await expect(svc().readsImages(1)).resolves.toBe(true);
|
||||
expect(modelCapabilities).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('readsImages on auto asks a local server, and answers no for a cloud provider', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'auto', baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b' }));
|
||||
modelCapabilities.mockResolvedValue(['completion', 'vision']);
|
||||
await expect(svc().readsImages(1)).resolves.toBe(true);
|
||||
expect(modelCapabilities).toHaveBeenCalledWith('http://ollama:11434/v1', 'qwen3.5:4b');
|
||||
modelCapabilities.mockResolvedValue(['completion']);
|
||||
await expect(svc().readsImages(1)).resolves.toBe(false);
|
||||
modelCapabilities.mockResolvedValue(null);
|
||||
await expect(svc().readsImages(1)).resolves.toBe(false);
|
||||
|
||||
modelCapabilities.mockClear();
|
||||
for (const provider of ['openai', 'anthropic']) {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider, vision: 'auto' }));
|
||||
await expect(svc().readsImages(1)).resolves.toBe(false);
|
||||
}
|
||||
expect(modelCapabilities).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('readsImages is false when AI parsing is not configured', async () => {
|
||||
resolveLlmConfig.mockReturnValue(null);
|
||||
await expect(svc().readsImages(1)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('refuses a photo the model cannot read, without calling it', async () => {
|
||||
const res = await svc().parse(file('ticket.jpg', 'jpeg'), 1);
|
||||
expect(res.kiItems).toEqual([]);
|
||||
expect(res.warnings[0]).toMatch(/does not read images/);
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
expect(extractText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('sends a photo as an image to a cloud provider, with no text', async () => {
|
||||
for (const provider of ['openai', 'anthropic']) {
|
||||
extract.mockClear();
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider, vision: 'on' }));
|
||||
await svc().parse(file('ticket.PNG', 'not really a png'), 1);
|
||||
const input = extract.mock.calls[0][0];
|
||||
// Bytes that do not decode go as they came rather than being refused.
|
||||
expect(input.file).toEqual({ mimeType: 'image/png', data: Buffer.from('not really a png') });
|
||||
expect(input.text).toBeUndefined();
|
||||
}
|
||||
expect(extractText).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('sends a photo through the local router with the image attached', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on', baseUrl: 'http://ollama:11434/v1' }));
|
||||
const res = await svc().parse(file('ticket.jpeg', 'jpeg bytes'), 1);
|
||||
expect(res.kiItems).toEqual([{ '@type': 'TrainReservation' }]);
|
||||
expect(routeImageExtraction).toHaveBeenCalledWith([Buffer.from('jpeg bytes')], expect.objectContaining({ baseUrl: 'http://ollama:11434/v1' }));
|
||||
expect(routeExtraction).not.toHaveBeenCalled();
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a PDF on its text path when the model reads images', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
await svc().parse(file('a.pdf', '%PDF'), 1);
|
||||
expect(extract.mock.calls[0][0].file).toBeUndefined();
|
||||
expect(extractText).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmParseService.readReceipt', () => {
|
||||
const RECEIPT = { merchant: 'Bäckerei', date: '2026-09-20', total: 7.5, currency: 'EUR', items: [{ name: 'Brezel', price: 2.5 }] };
|
||||
|
||||
it('says why when AI parsing is not set up, the file is not a photo, or the model reads no images', async () => {
|
||||
resolveLlmConfig.mockReturnValue(null);
|
||||
expect((await svc().readReceipt(file('r.jpg'), 1)).warnings[0]).toMatch(/not configured/);
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
expect(await svc().readReceipt(file('r.pdf'), 1)).toEqual({ receipt: null, warnings: ['r.pdf: not a photo'] });
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'off' }));
|
||||
expect((await svc().readReceipt(file('r.jpg'), 1)).warnings[0]).toMatch(/does not read images/);
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
expect(extractEnforced).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reads through Ollama\'s native chat on a local provider, with the photo and the receipt schema', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on', baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b' }));
|
||||
extractEnforced.mockResolvedValue(RECEIPT);
|
||||
await expect(svc().readReceipt(file('r.jpg', 'jpeg'), 1)).resolves.toEqual({ receipt: RECEIPT, warnings: [] });
|
||||
const call = extractEnforced.mock.calls[0][0];
|
||||
expect(call).toMatchObject({ baseUrl: 'http://ollama:11434/v1', model: 'qwen3.5:4b', images: [Buffer.from('jpeg').toString('base64')] });
|
||||
expect(call.schema.required).toContain('total');
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('asks a cloud client for a receipts list with the photo attached, and keeps the first', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'anthropic', vision: 'on' }));
|
||||
extract.mockResolvedValue([RECEIPT, { total: 1 }]);
|
||||
await expect(svc().readReceipt(file('r.png', 'png'), 1)).resolves.toEqual({ receipt: RECEIPT, warnings: [] });
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.rootKey).toBe('receipts');
|
||||
expect(input.file).toEqual({ mimeType: 'image/png', data: Buffer.from('png') });
|
||||
expect(input.text).toBeUndefined();
|
||||
});
|
||||
|
||||
it('answers a warning, not a throw, when the model fails or reads nothing', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
extract.mockRejectedValue(new Error('429'));
|
||||
const failed = await svc().readReceipt(file('r.jpg'), 1);
|
||||
expect(failed.receipt).toBeNull();
|
||||
expect(failed.warnings[0]).toMatch(/AI parsing failed — 429/);
|
||||
extract.mockResolvedValue([]);
|
||||
expect(await svc().readReceipt(file('r.jpg'), 1)).toEqual({ receipt: null, warnings: ['r.jpg: no receipt could be read'] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('LlmParseService: a scanned PDF', () => {
|
||||
const page = (n: string) => ({ mimeType: 'image/jpeg', data: Buffer.from(n) });
|
||||
|
||||
beforeEach(() => extractText.mockResolvedValue(' '));
|
||||
|
||||
it('sends the pages as images when the text layer is empty and the model reads images', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
renderPdfPages.mockResolvedValue([page('p1'), page('p2')]);
|
||||
await svc().parse(file('scan.pdf', '%PDF'), 1);
|
||||
const input = extract.mock.calls[0][0];
|
||||
expect(input.file).toEqual(page('p1'));
|
||||
expect(input.pageImages).toEqual([page('p2')]);
|
||||
expect(input.text).toBeUndefined();
|
||||
});
|
||||
|
||||
it('hands the local router every page', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ provider: 'local', vision: 'on' }));
|
||||
renderPdfPages.mockResolvedValue([page('p1'), page('p2')]);
|
||||
await svc().parse(file('scan.pdf', '%PDF'), 1);
|
||||
expect(routeImageExtraction).toHaveBeenCalledWith([Buffer.from('p1'), Buffer.from('p2')], expect.anything());
|
||||
expect(routeExtraction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps the warning, and draws nothing, when the model reads no images', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'off' }));
|
||||
const res = await svc().parse(file('scan.pdf', '%PDF'), 1);
|
||||
expect(res.warnings[0]).toMatch(/no readable text found \(a scanned PDF needs a model that reads images\)/);
|
||||
expect(renderPdfPages).not.toHaveBeenCalled();
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps the warning when no page could be drawn', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
const res = await svc().parse(file('scan.pdf', '%PDF'), 1);
|
||||
expect(res.warnings[0]).toMatch(/no readable text found/);
|
||||
expect(extract).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not draw an empty text file', async () => {
|
||||
resolveLlmConfig.mockReturnValue(cfg({ vision: 'on' }));
|
||||
await svc().parse(file('empty.txt', ''), 1);
|
||||
expect(renderPdfPages).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -52,6 +52,15 @@ describe('extractEnforced', () => {
|
||||
expect((init as RequestInit).headers).not.toHaveProperty('authorization');
|
||||
});
|
||||
|
||||
it('attaches images to the user turn only when given', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ message: { content: '{}' } }));
|
||||
await extractEnforced({ ...INPUT, images: ['aW1n'] });
|
||||
await extractEnforced(INPUT);
|
||||
const userTurn = (i: number) => JSON.parse((fetchFn.mock.calls[i][1] as RequestInit).body as string).messages[1];
|
||||
expect(userTurn(0).images).toEqual(['aW1n']);
|
||||
expect(userTurn(1)).not.toHaveProperty('images');
|
||||
});
|
||||
|
||||
it('sends a bearer header only when an apiKey is given', async () => {
|
||||
const fetchFn = mockFetch(() => jsonResponse({ message: { content: '{}' } }));
|
||||
await extractEnforced({ ...INPUT, apiKey: 'sk-123', numPredict: 900, numCtx: 16000 });
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { buildReceiptPrompt, RECEIPT_LIST_JSON_SCHEMA, RECEIPT_ROOT_KEY, toReceiptRead } from '../../../../src/nest/llm-parse/receipt-read';
|
||||
|
||||
describe('toReceiptRead', () => {
|
||||
it('keeps a clean answer as it came', () => {
|
||||
expect(toReceiptRead({
|
||||
merchant: 'Café de Flore', date: '2026-09-20', total: 23.4, currency: 'EUR',
|
||||
items: [{ name: '2 x Espresso', price: 8 }, { name: 'Croque', price: 15.4 }],
|
||||
})).toEqual({
|
||||
merchant: 'Café de Flore', date: '2026-09-20', total: 23.4, currency: 'EUR',
|
||||
items: [{ name: '2 x Espresso', price: 8 }, { name: 'Croque', price: 15.4 }],
|
||||
});
|
||||
});
|
||||
|
||||
it('reads amounts written as text, in either decimal convention', () => {
|
||||
expect(toReceiptRead({ total: '12,50 €' })?.total).toBe(12.5);
|
||||
expect(toReceiptRead({ total: '1.234,50' })?.total).toBe(1234.5);
|
||||
expect(toReceiptRead({ total: '1,234.50' })?.total).toBe(1234.5);
|
||||
expect(toReceiptRead({ total: 7, items: [{ name: 'Tea', price: '3,20' }] })?.items).toEqual([{ name: 'Tea', price: 3.2 }]);
|
||||
});
|
||||
|
||||
it('turns a currency symbol or name into its code, and drops one nobody can convert', () => {
|
||||
expect(toReceiptRead({ total: 5, currency: '€' })?.currency).toBe('EUR');
|
||||
expect(toReceiptRead({ total: 5, currency: 'euros' })?.currency).toBe('EUR');
|
||||
expect(toReceiptRead({ total: 5, currency: 'Lek' })?.currency).toBeNull();
|
||||
});
|
||||
|
||||
it('keeps only a real calendar date', () => {
|
||||
expect(toReceiptRead({ total: 5, date: '2026-02-28' })?.date).toBe('2026-02-28');
|
||||
for (const date of ['2026-02-30', '28/02/2026', '2026-2-8', '']) expect(toReceiptRead({ total: 5, date })?.date).toBeNull();
|
||||
});
|
||||
|
||||
it('leaves out a total that is not a positive amount, and lines without a name or price', () => {
|
||||
const read = toReceiptRead({ merchant: 'Shop', total: 0, items: [{ name: '', price: 2 }, { name: 'Bread' }, 'x', { name: 'Milk', price: 1.1 }] });
|
||||
expect(read).toEqual({ merchant: 'Shop', date: null, total: null, currency: null, items: [{ name: 'Milk', price: 1.1 }] });
|
||||
});
|
||||
|
||||
it('is null when nothing usable came back', () => {
|
||||
for (const raw of [null, undefined, 'text', [], {}, { total: 'n/a', merchant: ' ' }]) expect(toReceiptRead(raw)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the receipt prompt and schema', () => {
|
||||
it('pins today so an ambiguous date is read towards it', () => {
|
||||
expect(buildReceiptPrompt(new Date('2026-09-24T10:00:00Z'))).toContain('Today is 2026-09-24.');
|
||||
});
|
||||
|
||||
it('wraps one receipt under the key the cloud clients read', () => {
|
||||
expect(RECEIPT_LIST_JSON_SCHEMA.required).toEqual([RECEIPT_ROOT_KEY]);
|
||||
expect(RECEIPT_LIST_JSON_SCHEMA.properties[RECEIPT_ROOT_KEY].items.required).toEqual(['merchant', 'date', 'total', 'currency', 'items']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { HttpException } from '@nestjs/common';
|
||||
import { ReceiptScanController } from '../../../src/nest/receipt-scan/receipt-scan.controller';
|
||||
import type { ImportJobsService } from '../../../src/nest/booking-import/import-jobs.service';
|
||||
import type { LlmParseService } from '../../../src/nest/llm-parse/llm-parse.service';
|
||||
import type { DatabaseService } from '../../../src/nest/database/database.service';
|
||||
import type { PermissionsService } from '../../../src/nest/permissions/permissions.service';
|
||||
import type { AddonsService } from '../../../src/nest/addons/addons.service';
|
||||
import type { User } from '../../../src/types';
|
||||
|
||||
const user = { id: 3, role: 'user' } as User;
|
||||
const photo = (name = 'r.jpg') => ({ originalname: name, buffer: Buffer.from('x') } as Express.Multer.File);
|
||||
|
||||
function make(over: { readsImages?: boolean; trip?: unknown; addon?: boolean; allowed?: boolean } = {}) {
|
||||
const startReceipt = vi.fn(() => 'job-1');
|
||||
const checkPermission = vi.fn(() => over.allowed ?? true);
|
||||
const c = new ReceiptScanController(
|
||||
{ startReceipt } as unknown as ImportJobsService,
|
||||
{ readsImages: vi.fn(async () => over.readsImages ?? true) } as unknown as LlmParseService,
|
||||
{ canAccessTrip: vi.fn(() => ('trip' in over ? over.trip : { user_id: 5 })) } as unknown as DatabaseService,
|
||||
{ checkPermission } as unknown as PermissionsService,
|
||||
{ isAddonEnabled: vi.fn(() => over.addon ?? true) } as unknown as AddonsService,
|
||||
);
|
||||
return { c, startReceipt, checkPermission };
|
||||
}
|
||||
|
||||
async function failure(fn: () => Promise<unknown>): Promise<[number, unknown]> {
|
||||
try { await fn(); } catch (e) { expect(e).toBeInstanceOf(HttpException); return [(e as HttpException).getStatus(), (e as HttpException).getResponse()]; }
|
||||
throw new Error('expected throw');
|
||||
}
|
||||
|
||||
describe('ReceiptScanController', () => {
|
||||
it('queues a photo and answers the job id', async () => {
|
||||
const { c, startReceipt, checkPermission } = make();
|
||||
await expect(c.scan(user, '9', photo())).resolves.toEqual({ jobId: 'job-1' });
|
||||
expect(startReceipt).toHaveBeenCalledWith('9', expect.objectContaining({ originalname: 'r.jpg' }), 3);
|
||||
expect(checkPermission).toHaveBeenCalledWith('budget_edit', 'user', 5, 3, true);
|
||||
});
|
||||
|
||||
it('answers what the guards would: 404 for a trip out of reach or Costs off, 403 without budget_edit', async () => {
|
||||
expect(await failure(() => make({ trip: undefined }).c.scan(user, '9', photo()))).toEqual([404, { error: 'Trip not found' }]);
|
||||
expect(await failure(() => make({ addon: false }).c.scan(user, '9', photo()))).toEqual([404, { error: 'Costs addon is not enabled' }]);
|
||||
const { c, startReceipt } = make({ allowed: false });
|
||||
expect(await failure(() => c.scan(user, '9', photo()))).toEqual([403, { error: 'No permission' }]);
|
||||
expect(startReceipt).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses with 400 a missing file, a file that is not a photo, or a model that reads no images', async () => {
|
||||
expect((await failure(() => make().c.scan(user, '9', undefined)))[0]).toBe(400);
|
||||
expect((await failure(() => make().c.scan(user, '9', photo('r.pdf'))))[0]).toBe(400);
|
||||
const { c, startReceipt } = make({ readsImages: false });
|
||||
expect(await failure(() => c.scan(user, '9', photo()))).toEqual([400, { error: 'The configured AI model does not read photos' }]);
|
||||
expect(startReceipt).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,6 @@
|
||||
import {
|
||||
asLlmVision,
|
||||
LLM_VISION_MODES,
|
||||
adminUserCreateRequestSchema,
|
||||
adminUserUpdateRequestSchema,
|
||||
adminPermissionsRequestSchema,
|
||||
@@ -106,6 +108,13 @@ describe('adminAddonUpdateRequestSchema', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('asLlmVision', () => {
|
||||
it('keeps the three modes and reads anything else as auto', () => {
|
||||
for (const mode of LLM_VISION_MODES) expect(asLlmVision(mode)).toBe(mode);
|
||||
for (const other of [undefined, null, '', 'ON', true, false, 1]) expect(asLlmVision(other)).toBe('auto');
|
||||
});
|
||||
});
|
||||
|
||||
describe('adminCollabFeaturesRequestSchema', () => {
|
||||
it('accepts a single computed key', () => {
|
||||
expect(adminCollabFeaturesRequestSchema.safeParse({ chat: false }).success).toBe(true);
|
||||
|
||||
@@ -80,6 +80,30 @@ export const adminAddonUpdateRequestSchema = z.object({
|
||||
});
|
||||
export type AdminAddonUpdateRequest = z.infer<typeof adminAddonUpdateRequestSchema>;
|
||||
|
||||
/**
|
||||
* Whether the AI Parsing model is handed a photo, stored as `vision` on that
|
||||
* addon's config. `auto` asks a local Ollama server what the model can do and
|
||||
* means no for a cloud provider; `on` and `off` are the admin's word.
|
||||
*/
|
||||
export const LLM_VISION_MODES = ['auto', 'on', 'off'] as const;
|
||||
export type LlmVision = (typeof LLM_VISION_MODES)[number];
|
||||
|
||||
/**
|
||||
* The photos handed to a model that reads images. HEIC is not here: no provider
|
||||
* TREK talks to reads it, and a browser picking from the camera roll hands over
|
||||
* a JPEG anyway.
|
||||
*/
|
||||
export const LLM_PHOTO_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.webp'] as const;
|
||||
|
||||
/** GET /api/llm/capabilities: what the caller's AI model can be handed. */
|
||||
export const llmCapabilitiesResponseSchema = z.object({ images: z.boolean() });
|
||||
export type LlmCapabilitiesResponse = z.infer<typeof llmCapabilitiesResponseSchema>;
|
||||
|
||||
/** A stored or submitted value as a mode; anything else, a missing value included, is `auto`. */
|
||||
export function asLlmVision(value: unknown): LlmVision {
|
||||
return (LLM_VISION_MODES as readonly unknown[]).includes(value) ? (value as LlmVision) : 'auto';
|
||||
}
|
||||
|
||||
// Fully partial — the client PUTs one computed key at a time.
|
||||
export const adminCollabFeaturesRequestSchema = z.object({
|
||||
chat: z.boolean().optional(),
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* A photographed receipt, as the AI model read it for the Costs tab.
|
||||
*
|
||||
* It only pre-fills the expense editor: nothing is saved until the person saves
|
||||
* the expense there, so every field may be missing and each is checked by eye.
|
||||
* There is no category: which kind of spending it was is the person's call.
|
||||
*/
|
||||
export const receiptLineSchema = z.object({
|
||||
name: z.string().min(1),
|
||||
/** What the line cost in total, quantity included, in the receipt's currency. */
|
||||
price: z.number(),
|
||||
});
|
||||
export type ReceiptLine = z.infer<typeof receiptLineSchema>;
|
||||
|
||||
export const receiptReadSchema = z.object({
|
||||
/** The business, which becomes the expense name. */
|
||||
merchant: z.string().min(1).nullable(),
|
||||
/** YYYY-MM-DD. */
|
||||
date: z
|
||||
.string()
|
||||
.regex(/^\d{4}-\d{2}-\d{2}$/)
|
||||
.nullable(),
|
||||
/** The grand total paid. */
|
||||
total: z.number().positive().nullable(),
|
||||
/** ISO 4217. */
|
||||
currency: z
|
||||
.string()
|
||||
.regex(/^[A-Z]{3}$/)
|
||||
.nullable(),
|
||||
/** The itemized lines, which seed the Ticket split. */
|
||||
items: z.array(receiptLineSchema),
|
||||
});
|
||||
export type ReceiptRead = z.infer<typeof receiptReadSchema>;
|
||||
|
||||
/** What a finished receipt scan job answers: the read, or null with the reason in `warnings`. */
|
||||
export const receiptScanResultSchema = z.object({
|
||||
receipt: receiptReadSchema.nullable(),
|
||||
warnings: z.array(z.string()),
|
||||
});
|
||||
export type ReceiptScanResult = z.infer<typeof receiptScanResultSchema>;
|
||||
|
||||
/** POST /api/trips/:tripId/budget/receipt-scan: the background job reading the photo. */
|
||||
export const receiptScanStartResponseSchema = z.object({ jobId: z.string().min(1) });
|
||||
export type ReceiptScanStartResponse = z.infer<typeof receiptScanStartResponseSchema>;
|
||||
@@ -626,6 +626,11 @@ const admin: TranslationStrings = {
|
||||
'قراءة الزيارات والمسارات المسجَّلة من خادم Dawarich يربطه كل قارئ بنفسه',
|
||||
'admin.addons.catalog.llm_parsing.name': 'التحليل بالذكاء الاصطناعي',
|
||||
'admin.addons.catalog.llm_parsing.description': 'يقرأ الحجوزات التي يعجز المحلل المدمج عن فهمها، بنموذج ذكاء اصطناعي تختاره',
|
||||
'admin.addons.llm.vision.auto': 'تلقائي',
|
||||
'admin.addons.llm.vision.on': 'نعم',
|
||||
'admin.addons.llm.vision.off': 'لا',
|
||||
'admin.addons.llm.vision.hintLocal': 'يسأل الخيار «تلقائي» خادم Ollama عمّا إذا كان هذا النموذج يقرأ الصور.',
|
||||
'admin.addons.llm.vision.hintCloud': 'يعني «تلقائي» «لا» للنموذج السحابي. اختر «نعم» إذا كان هذا النموذج يقرأ الصور.',
|
||||
'admin.addons.enabled': 'مفعّل',
|
||||
'admin.addons.disabled': 'معطّل',
|
||||
'admin.addons.type.trip': 'رحلة',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'الإيصالات',
|
||||
'costs.receiptsTitle': 'الإيصالات والفواتير',
|
||||
'costs.attachReceipt': 'إرفاق إيصال / فاتورة',
|
||||
'costs.scan.button': 'مسح إيصال',
|
||||
'costs.scan.reading': 'جارٍ قراءة الإيصال…',
|
||||
'costs.scan.review': 'مراجعة المصروف',
|
||||
'costs.scan.nothingRead': 'تعذّرت قراءة أي إيصال من هذه الصورة.',
|
||||
'costs.scan.failed': 'تعذّر بدء قراءة الإيصال.',
|
||||
'costs.scan.title': 'مسح إيصال',
|
||||
'costs.scan.accepted': 'صورة لإيصال واحد (JPG أو PNG أو WEBP، حتى 10 ميغابايت). يقرأها نموذج الذكاء الاصطناعي في الخلفية، ثم تراجع المصروف قبل حفظه.',
|
||||
'costs.scan.dropHere': 'أفلت صورة الإيصال هنا، أو انقر لاختيار صورة أو التقاطها',
|
||||
'costs.scan.dropActive': 'أفلت الصورة لمسحها',
|
||||
'costs.scan.start': 'مسح',
|
||||
'costs.scan.unsupported': 'يمكن مسح الصور فقط: JPG أو PNG أو WEBP.',
|
||||
'costs.noReceipts': 'لا توجد إيصالات مرفقة',
|
||||
'costs.deleteReceipt': 'حذف الإيصال',
|
||||
'costs.viewReceipt': 'عرض الإيصال',
|
||||
|
||||
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropActive': 'أسقط الملفات للاستيراد',
|
||||
'reservations.import.acceptedFormats':
|
||||
'المقبول: EML، PDF، PKPass، HTML، TXT (بحد أقصى 10 ميغابايت لكل ملف، حتى 5 ملفات)',
|
||||
'reservations.import.acceptedPhotos': 'يقرأ نموذج الذكاء الاصطناعي الصور (JPG وPNG وWEBP).',
|
||||
'reservations.import.parsing': 'جارٍ معالجة الملفات…',
|
||||
'reservations.import.previewHeading': 'تم العثور على {count} حجز/حجوزات',
|
||||
'reservations.import.previewEmpty': 'تعذّر استخراج أي حجوزات من الملفات المُحمَّلة.',
|
||||
|
||||
@@ -383,8 +383,8 @@ const settings: TranslationStrings = {
|
||||
'settings.aiParsing.baseUrlHint': 'المكان الذي يعمل فيه النموذج — خادم Ollama محلي أو نقطة نهاية متوافقة مع OpenAI.',
|
||||
'settings.aiParsing.apiKey': 'مفتاح API',
|
||||
'settings.aiParsing.apiKeyHint': 'يُخزَّن مشفّرًا. اتركه فارغًا للإبقاء على المفتاح الحالي.',
|
||||
'settings.aiParsing.multimodal': 'إرسال المستندات كصور',
|
||||
'settings.aiParsing.multimodalHint': 'للنماذج القادرة على الرؤية — يرسل ملف PDF الأصلي بدلًا من النص المستخرج.',
|
||||
'settings.aiParsing.multimodal': 'النموذج يقرأ الصور',
|
||||
'settings.aiParsing.multimodalHint': 'فعّله لنموذج يقرأ الصور، ليمكن استيراد صورة أو مسحها.',
|
||||
'settings.aiParsing.toast.saved': 'تم حفظ إعدادات الذكاء الاصطناعي',
|
||||
'settings.aiParsing.toast.saveError': 'تعذّر حفظ إعدادات الذكاء الاصطناعي',
|
||||
'settings.tabs.appearance': 'Appearance',
|
||||
|
||||
@@ -733,6 +733,11 @@ const admin: TranslationStrings = {
|
||||
'admin.addons.catalog.llm_parsing.name': 'Süni intellektlə təhlil',
|
||||
'admin.addons.catalog.llm_parsing.description':
|
||||
'Daxili təhlilçinin oxuya bilmədiyi rezervasiyaları seçdiyiniz süni intellekt modeli ilə oxuyur',
|
||||
'admin.addons.llm.vision.auto': 'Avtomatik',
|
||||
'admin.addons.llm.vision.on': 'Bəli',
|
||||
'admin.addons.llm.vision.off': 'Xeyr',
|
||||
'admin.addons.llm.vision.hintLocal': 'Avtomatik rejim bu modelin şəkilləri oxuyub-oxumadığını Ollama serverindən soruşur.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Bulud modeli üçün Avtomatik "xeyr" deməkdir. Bu model şəkilləri oxuyursa, Bəli seç.',
|
||||
'admin.addons.enabled': 'Aktivdir',
|
||||
'admin.addons.disabled': 'Deaktivdir',
|
||||
'admin.addons.type.trip': 'Səyahət',
|
||||
|
||||
@@ -186,6 +186,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Qəbzlər',
|
||||
'costs.receiptsTitle': 'Qəbzlər və fakturalar',
|
||||
'costs.attachReceipt': 'Qəbz / faktura əlavə et',
|
||||
'costs.scan.button': 'Qəbzi skan et',
|
||||
'costs.scan.reading': 'Qəbz oxunur…',
|
||||
'costs.scan.review': 'Xərci yoxla',
|
||||
'costs.scan.nothingRead': 'Bu fotodan qəbz oxunmadı.',
|
||||
'costs.scan.failed': 'Qəbzin oxunmasına başlamaq mümkün olmadı.',
|
||||
'costs.scan.title': 'Qəbzi skan et',
|
||||
'costs.scan.accepted': 'Bir qəbzin fotosu (JPG, PNG və ya WEBP, 10 MB-a qədər). Süni intellekt modeli onu arxa planda oxuyur, sonra xərci saxlamazdan əvvəl yoxlayırsan.',
|
||||
'costs.scan.dropHere': 'Qəbzin fotosunu bura burax və ya seçmək ya çəkmək üçün kliklə',
|
||||
'costs.scan.dropActive': 'Skan etmək üçün fotonu burax',
|
||||
'costs.scan.start': 'Skan et',
|
||||
'costs.scan.unsupported': 'Yalnız foto skan edilə bilər: JPG, PNG və ya WEBP.',
|
||||
'costs.noReceipts': 'Əlavə edilmiş qəbz yoxdur',
|
||||
'costs.deleteReceipt': 'Qəbzi sil',
|
||||
'costs.viewReceipt': 'Qəbzə bax',
|
||||
|
||||
@@ -179,6 +179,7 @@ const reservations: TranslationStrings = {
|
||||
'İdxal etmək üçün faylları buraya buraxın',
|
||||
'reservations.import.acceptedFormats':
|
||||
'Qəbul edilən formatlar: EML, PDF, PKPass, HTML, TXT (hər biri maksimum 10 MB, 5 fayladək)',
|
||||
'reservations.import.acceptedPhotos': 'Fotoları (JPG, PNG, WEBP) süni intellekt modeli oxuyur.',
|
||||
'reservations.import.parsing': 'Fayllar təhlil edilir…',
|
||||
'reservations.import.previewHeading':
|
||||
'{count} rezervasiya tapıldı',
|
||||
|
||||
@@ -497,9 +497,9 @@ const settings: TranslationStrings = {
|
||||
'settings.aiParsing.apiKey': 'API açarı',
|
||||
'settings.aiParsing.apiKeyHint':
|
||||
'Şifrələnmiş şəkildə saxlanılır. Cari açarı saxlamaq üçün boş buraxın.',
|
||||
'settings.aiParsing.multimodal': 'Sənədləri şəkil kimi göndər',
|
||||
'settings.aiParsing.multimodal': 'Model şəkilləri oxuyur',
|
||||
'settings.aiParsing.multimodalHint':
|
||||
'Görmə qabiliyyətli modellər üçün — çıxarılmış mətn əvəzinə orijinal PDF-i göndərir.',
|
||||
'Şəkilləri oxuyan model üçün aç: onda foto idxal oluna və ya skan edilə bilər.',
|
||||
'settings.aiParsing.toast.saved': 'AI tənzimləmələri yadda saxlanıldı',
|
||||
'settings.aiParsing.toast.saveError':
|
||||
'AI tənzimləmələrini yadda saxlamaq mümkün olmadı',
|
||||
|
||||
@@ -618,6 +618,11 @@ const admin: TranslationStrings = {
|
||||
'Leia visitas e rotas gravadas de uma instância do Dawarich que cada leitor conecta por conta própria',
|
||||
'admin.addons.catalog.llm_parsing.name': 'Análise por IA',
|
||||
'admin.addons.catalog.llm_parsing.description': 'Lê reservas que o analisador integrado não entende, usando um modelo de IA à sua escolha',
|
||||
'admin.addons.llm.vision.auto': 'Automático',
|
||||
'admin.addons.llm.vision.on': 'Sim',
|
||||
'admin.addons.llm.vision.off': 'Não',
|
||||
'admin.addons.llm.vision.hintLocal': 'Automático pergunta ao servidor Ollama se este modelo lê imagens.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Automático significa não para um modelo na nuvem. Escolha Sim se este modelo lê imagens.',
|
||||
'admin.addons.enabled': 'Ativado',
|
||||
'admin.addons.disabled': 'Desativado',
|
||||
'admin.addons.type.trip': 'Viagem',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Comprovantes',
|
||||
'costs.receiptsTitle': 'Comprovantes e notas',
|
||||
'costs.attachReceipt': 'Anexar comprovante / nota',
|
||||
'costs.scan.button': 'Escanear comprovante',
|
||||
'costs.scan.reading': 'Lendo o comprovante…',
|
||||
'costs.scan.review': 'Revisar a despesa',
|
||||
'costs.scan.nothingRead': 'Não foi possível ler um comprovante nesta foto.',
|
||||
'costs.scan.failed': 'Não foi possível começar a ler o comprovante.',
|
||||
'costs.scan.title': 'Escanear comprovante',
|
||||
'costs.scan.accepted': 'A foto de um único comprovante (JPG, PNG ou WEBP, até 10 MB). O modelo de IA lê em segundo plano e depois você confere a despesa antes de salvar.',
|
||||
'costs.scan.dropHere': 'Solte aqui a foto do comprovante, ou clique para escolher ou tirar uma',
|
||||
'costs.scan.dropActive': 'Solte a foto para escaneá-la',
|
||||
'costs.scan.start': 'Escanear',
|
||||
'costs.scan.unsupported': 'Só é possível escanear uma foto: JPG, PNG ou WEBP.',
|
||||
'costs.noReceipts': 'Nenhum comprovante anexado',
|
||||
'costs.deleteReceipt': 'Remover comprovante',
|
||||
'costs.viewReceipt': 'Ver comprovante',
|
||||
|
||||
@@ -128,6 +128,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropHere': 'Solte os arquivos de confirmação de reserva aqui ou clique para selecionar',
|
||||
'reservations.import.dropActive': 'Solte os arquivos para importar',
|
||||
'reservations.import.acceptedFormats': 'Aceitos: EML, PDF, PKPass, HTML, TXT (máx. 10 MB cada, até 5 arquivos)',
|
||||
'reservations.import.acceptedPhotos': 'As fotos (JPG, PNG, WEBP) são lidas pelo modelo de IA.',
|
||||
'reservations.import.parsing': 'Analisando arquivos…',
|
||||
'reservations.import.previewHeading': '{count} reserva(s) encontrada(s)',
|
||||
'reservations.import.previewEmpty': 'Nenhuma reserva pôde ser extraída dos arquivos enviados.',
|
||||
|
||||
@@ -398,9 +398,9 @@ const settings: TranslationStrings = {
|
||||
'Onde o modelo é executado — um servidor Ollama local ou um endpoint compatível com OpenAI.',
|
||||
'settings.aiParsing.apiKey': 'Chave de API',
|
||||
'settings.aiParsing.apiKeyHint': 'Armazenada de forma criptografada. Deixe em branco para manter a chave atual.',
|
||||
'settings.aiParsing.multimodal': 'Enviar documentos como imagens',
|
||||
'settings.aiParsing.multimodal': 'O modelo lê imagens',
|
||||
'settings.aiParsing.multimodalHint':
|
||||
'Para modelos com capacidade de visão — envia o PDF original em vez do texto extraído.',
|
||||
'Ative para um modelo que lê imagens, assim uma foto pode ser importada ou escaneada.',
|
||||
'settings.aiParsing.toast.saved': 'Configurações de IA salvas',
|
||||
'settings.aiParsing.toast.saveError': 'Não foi possível salvar as configurações de IA',
|
||||
'settings.tabs.appearance': 'Appearance',
|
||||
|
||||
@@ -233,6 +233,11 @@ const admin: TranslationStrings = {
|
||||
'Llegeix estades i recorreguts enregistrats d’una instància de Dawarich que cada lector connecta pel seu compte',
|
||||
'admin.addons.catalog.llm_parsing.name': 'Anàlisi amb IA',
|
||||
'admin.addons.catalog.llm_parsing.description': 'Llegeix reserves que l’analitzador integrat no entén, amb un model d’IA que triïs',
|
||||
'admin.addons.llm.vision.auto': 'Automàtic',
|
||||
'admin.addons.llm.vision.on': 'Sí',
|
||||
'admin.addons.llm.vision.off': 'No',
|
||||
'admin.addons.llm.vision.hintLocal': 'Automàtic pregunta al servidor Ollama si aquest model llegeix imatges.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Automàtic vol dir no per a un model al núvol. Tria Sí si aquest model llegeix imatges.',
|
||||
'admin.addons.enabled': 'Actiu',
|
||||
'admin.addons.disabled': 'Desactivat',
|
||||
'admin.addons.type.trip': 'Viatge',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Rebuts',
|
||||
'costs.receiptsTitle': 'Rebuts i factures',
|
||||
'costs.attachReceipt': 'Adjuntar rebut / factura',
|
||||
'costs.scan.button': 'Escanejar rebut',
|
||||
'costs.scan.reading': 'Llegint el rebut…',
|
||||
'costs.scan.review': 'Revisar la despesa',
|
||||
'costs.scan.nothingRead': "No s'ha pogut llegir cap rebut d'aquesta foto.",
|
||||
'costs.scan.failed': "No s'ha pogut començar a llegir el rebut.",
|
||||
'costs.scan.title': 'Escanejar rebut',
|
||||
'costs.scan.accepted': "Una foto d'un sol rebut (JPG, PNG o WEBP, fins a 10 MB). El model d'IA la llegeix en segon pla i després revises la despesa abans de desar-la.",
|
||||
'costs.scan.dropHere': 'Deixa aquí la foto del rebut, o fes clic per triar-ne o fer-ne una',
|
||||
'costs.scan.dropActive': 'Deixa anar la foto per escanejar-la',
|
||||
'costs.scan.start': 'Escanejar',
|
||||
'costs.scan.unsupported': 'Només es pot escanejar una foto: JPG, PNG o WEBP.',
|
||||
'costs.noReceipts': 'Cap rebut adjunt',
|
||||
'costs.deleteReceipt': 'Eliminar rebut',
|
||||
'costs.viewReceipt': 'Veure rebut',
|
||||
|
||||
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropActive': 'Deixa anar els fitxers per importar',
|
||||
'reservations.import.acceptedFormats':
|
||||
'Acceptats: EML, PDF, PKPass, HTML, TXT (màx. 10 MB per fitxer, fins a 5 fitxers)',
|
||||
'reservations.import.acceptedPhotos': "El model d'IA llegeix les fotos (JPG, PNG, WEBP).",
|
||||
'reservations.import.parsing': 'Analitzant els fitxers…',
|
||||
'reservations.import.previewHeading': '{count} reserva(es) trobada(es)',
|
||||
'reservations.import.previewEmpty': "No s'han pogut extreure reserves dels fitxers pujats.",
|
||||
|
||||
@@ -379,9 +379,9 @@ const settings: TranslationStrings = {
|
||||
"On s'executa el model — un servidor local d'Ollama o un punt final compatible amb OpenAI.",
|
||||
'settings.aiParsing.apiKey': "Clau de l'API",
|
||||
'settings.aiParsing.apiKeyHint': "S'emmagatzema xifrada. Deixa-ho en blanc per mantenir la clau actual.",
|
||||
'settings.aiParsing.multimodal': 'Enviar documents com a imatges',
|
||||
'settings.aiParsing.multimodal': 'El model llegeix imatges',
|
||||
'settings.aiParsing.multimodalHint':
|
||||
'Per a models amb capacitats de visió — envia el PDF original en comptes del text extret.',
|
||||
'Activa-ho per a un model que llegeix imatges, així es pot importar o escanejar una foto.',
|
||||
'settings.aiParsing.toast.saved': "Ajustos d'IA desats",
|
||||
'settings.aiParsing.toast.saveError': "No s'han pogut desar els ajustos d'IA",
|
||||
'settings.tabs.appearance': 'Aparença',
|
||||
|
||||
@@ -608,6 +608,11 @@ const admin: TranslationStrings = {
|
||||
'Načtěte návštěvy a zaznamenané trasy z instance Dawarich, kterou si každý čtenář připojí sám',
|
||||
'admin.addons.catalog.llm_parsing.name': 'Analýza pomocí AI',
|
||||
'admin.addons.catalog.llm_parsing.description': 'Přečte rezervace, které vestavěný analyzátor nezvládne, pomocí zvoleného modelu AI',
|
||||
'admin.addons.llm.vision.auto': 'Automaticky',
|
||||
'admin.addons.llm.vision.on': 'Ano',
|
||||
'admin.addons.llm.vision.off': 'Ne',
|
||||
'admin.addons.llm.vision.hintLocal': 'Automaticky se zeptá serveru Ollama, zda tento model čte obrázky.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Automaticky znamená u cloudového modelu ne. Zvolte Ano, pokud tento model čte obrázky.',
|
||||
'admin.addons.catalog.roadtrip.name': 'Cesta autem',
|
||||
'admin.addons.catalog.roadtrip.description': 'Plánujte jízdy se zastávkami – doby jízdy a časy příjezdu se přepočítají samy',
|
||||
'admin.addons.enabled': 'Povoleno',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Účtenky',
|
||||
'costs.receiptsTitle': 'Účtenky a faktury',
|
||||
'costs.attachReceipt': 'Připojit účtenku / fakturu',
|
||||
'costs.scan.button': 'Naskenovat účtenku',
|
||||
'costs.scan.reading': 'Čtení účtenky…',
|
||||
'costs.scan.review': 'Zkontrolovat výdaj',
|
||||
'costs.scan.nothingRead': 'Z této fotografie se nepodařilo přečíst žádnou účtenku.',
|
||||
'costs.scan.failed': 'Čtení účtenky se nepodařilo spustit.',
|
||||
'costs.scan.title': 'Naskenovat účtenku',
|
||||
'costs.scan.accepted': 'Fotografie jedné účtenky (JPG, PNG nebo WEBP, do 10 MB). Model AI ji přečte na pozadí a vy pak výdaj před uložením zkontrolujete.',
|
||||
'costs.scan.dropHere': 'Přetáhněte sem fotografii účtenky, nebo klikněte a vyberte či pořiďte novou',
|
||||
'costs.scan.dropActive': 'Pusťte fotografii pro naskenování',
|
||||
'costs.scan.start': 'Skenovat',
|
||||
'costs.scan.unsupported': 'Skenovat lze jen fotografii: JPG, PNG nebo WEBP.',
|
||||
'costs.noReceipts': 'Žádné připojené účtenky',
|
||||
'costs.deleteReceipt': 'Odebrat účtenku',
|
||||
'costs.viewReceipt': 'Zobrazit účtenku',
|
||||
|
||||
@@ -129,6 +129,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropActive': 'Pusťte soubory pro import',
|
||||
'reservations.import.acceptedFormats':
|
||||
'Přijímané formáty: EML, PDF, PKPass, HTML, TXT (max. 10 MB každý, až 5 souborů)',
|
||||
'reservations.import.acceptedPhotos': 'Fotografie (JPG, PNG, WEBP) čte model AI.',
|
||||
'reservations.import.parsing': 'Zpracování souborů…',
|
||||
'reservations.import.previewHeading': 'Nalezeno {count} rezervace/í',
|
||||
'reservations.import.previewEmpty': 'Z nahraných souborů se nepodařilo extrahovat žádné rezervace.',
|
||||
|
||||
@@ -390,9 +390,9 @@ const settings: TranslationStrings = {
|
||||
'settings.aiParsing.baseUrlHint': 'Kde model běží — místní server Ollama nebo koncový bod kompatibilní s OpenAI.',
|
||||
'settings.aiParsing.apiKey': 'Klíč API',
|
||||
'settings.aiParsing.apiKeyHint': 'Ukládá se šifrovaně. Ponechte prázdné pro zachování aktuálního klíče.',
|
||||
'settings.aiParsing.multimodal': 'Odesílat dokumenty jako obrázky',
|
||||
'settings.aiParsing.multimodal': 'Model čte obrázky',
|
||||
'settings.aiParsing.multimodalHint':
|
||||
'Pro modely se schopností zpracovat obraz — odešle původní PDF místo extrahovaného textu.',
|
||||
'Zapněte pro model, který čte obrázky, aby šlo fotografii importovat nebo naskenovat.',
|
||||
'settings.aiParsing.toast.saved': 'Nastavení AI uloženo',
|
||||
'settings.aiParsing.toast.saveError': 'Nastavení AI se nepodařilo uložit',
|
||||
'settings.tabs.appearance': 'Appearance',
|
||||
|
||||
@@ -624,6 +624,11 @@ const admin: TranslationStrings = {
|
||||
'Aufenthalte und aufgezeichnete Routen aus einer Dawarich-Instanz lesen, die jeder selbst verbindet',
|
||||
'admin.addons.catalog.llm_parsing.name': 'KI-Auswertung',
|
||||
'admin.addons.catalog.llm_parsing.description': 'Liest Buchungen, an denen der eingebaute Parser scheitert, mit einem KI-Modell deiner Wahl',
|
||||
'admin.addons.llm.vision.auto': 'Automatisch',
|
||||
'admin.addons.llm.vision.on': 'Ja',
|
||||
'admin.addons.llm.vision.off': 'Nein',
|
||||
'admin.addons.llm.vision.hintLocal': 'Automatisch fragt den Ollama-Server, ob dieses Modell Bilder liest.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Automatisch bedeutet bei einem Cloud-Modell Nein. Wähle Ja, wenn dieses Modell Bilder liest.',
|
||||
'admin.addons.enabled': 'Aktiviert',
|
||||
'admin.addons.disabled': 'Deaktiviert',
|
||||
'admin.addons.type.trip': 'Reise',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Belege',
|
||||
'costs.receiptsTitle': 'Belege & Rechnungen',
|
||||
'costs.attachReceipt': 'Beleg / Rechnung anhängen',
|
||||
'costs.scan.button': 'Beleg scannen',
|
||||
'costs.scan.reading': 'Beleg wird gelesen…',
|
||||
'costs.scan.review': 'Ausgabe prüfen',
|
||||
'costs.scan.nothingRead': 'Aus diesem Foto ließ sich kein Beleg lesen.',
|
||||
'costs.scan.failed': 'Der Beleg konnte nicht eingelesen werden.',
|
||||
'costs.scan.title': 'Beleg scannen',
|
||||
'costs.scan.accepted': 'Ein Foto eines Belegs (JPG, PNG oder WEBP, bis 10 MB). Das KI-Modell liest es im Hintergrund, danach prüfst du die Ausgabe, bevor sie gespeichert wird.',
|
||||
'costs.scan.dropHere': 'Belegfoto hier ablegen oder klicken, um eins auszuwählen oder aufzunehmen',
|
||||
'costs.scan.dropActive': 'Foto ablegen, um es zu scannen',
|
||||
'costs.scan.start': 'Scannen',
|
||||
'costs.scan.unsupported': 'Nur Fotos lassen sich scannen: JPG, PNG oder WEBP.',
|
||||
'costs.noReceipts': 'Keine Belege angehängt',
|
||||
'costs.deleteReceipt': 'Beleg entfernen',
|
||||
'costs.viewReceipt': 'Beleg ansehen',
|
||||
|
||||
@@ -132,6 +132,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropActive': 'Dateien zum Importieren ablegen',
|
||||
'reservations.import.acceptedFormats':
|
||||
'Akzeptiert: EML, PDF, PKPass, HTML, TXT (max. 10 MB pro Datei, bis zu 5 Dateien)',
|
||||
'reservations.import.acceptedPhotos': 'Fotos (JPG, PNG, WEBP) liest das KI-Modell.',
|
||||
'reservations.import.parsing': 'Dateien werden verarbeitet…',
|
||||
'reservations.import.previewHeading': '{count} Reservierung(en) gefunden',
|
||||
'reservations.import.previewEmpty': 'Aus den hochgeladenen Dateien konnten keine Reservierungen extrahiert werden.',
|
||||
|
||||
@@ -401,9 +401,9 @@ const settings: TranslationStrings = {
|
||||
'Wo das Modell läuft — ein lokaler Ollama-Server oder ein OpenAI-kompatibler Endpunkt.',
|
||||
'settings.aiParsing.apiKey': 'API-Schlüssel',
|
||||
'settings.aiParsing.apiKeyHint': 'Verschlüsselt gespeichert. Leer lassen, um den aktuellen Schlüssel zu behalten.',
|
||||
'settings.aiParsing.multimodal': 'Dokumente als Bilder senden',
|
||||
'settings.aiParsing.multimodal': 'Modell liest Bilder',
|
||||
'settings.aiParsing.multimodalHint':
|
||||
'Für Modelle mit Bildverständnis — sendet das Original-PDF statt extrahiertem Text.',
|
||||
'Für ein Modell, das Bilder liest: Dann lässt sich ein Foto importieren oder scannen.',
|
||||
'settings.aiParsing.toast.saved': 'KI-Einstellungen gespeichert',
|
||||
'settings.aiParsing.toast.saveError': 'KI-Einstellungen konnten nicht gespeichert werden',
|
||||
'settings.tabs.appearance': 'Erscheinungsbild',
|
||||
|
||||
@@ -663,6 +663,11 @@ const admin: TranslationStrings = {
|
||||
'Read visits and recorded routes from a Dawarich instance each reader connects themselves',
|
||||
'admin.addons.catalog.llm_parsing.name': 'AI Parsing',
|
||||
'admin.addons.catalog.llm_parsing.description': 'Reads bookings the built-in parser cannot, using an AI model you choose',
|
||||
'admin.addons.llm.vision.auto': 'Automatic',
|
||||
'admin.addons.llm.vision.on': 'Yes',
|
||||
'admin.addons.llm.vision.off': 'No',
|
||||
'admin.addons.llm.vision.hintLocal': 'Automatic asks the Ollama server whether this model reads images.',
|
||||
'admin.addons.llm.vision.hintCloud': 'Automatic means no for a cloud model. Choose Yes if this model reads images.',
|
||||
'admin.addons.enabled': 'Enabled',
|
||||
'admin.addons.disabled': 'Disabled',
|
||||
'admin.addons.type.trip': 'Trip',
|
||||
|
||||
@@ -154,6 +154,17 @@ const budget: TranslationStrings = {
|
||||
'costs.receipts': 'Receipts',
|
||||
'costs.receiptsTitle': 'Receipts & Invoices',
|
||||
'costs.attachReceipt': 'Attach receipt / invoice',
|
||||
'costs.scan.button': 'Scan receipt',
|
||||
'costs.scan.reading': 'Reading the receipt…',
|
||||
'costs.scan.review': 'Review expense',
|
||||
'costs.scan.nothingRead': 'No receipt could be read from this photo.',
|
||||
'costs.scan.failed': 'Could not start reading the receipt.',
|
||||
'costs.scan.title': 'Scan a receipt',
|
||||
'costs.scan.accepted': 'A photo of one receipt (JPG, PNG or WEBP, up to 10 MB). The AI model reads it in the background, then you check the expense before it is saved.',
|
||||
'costs.scan.dropHere': 'Drop the receipt photo here, or click to pick or take one',
|
||||
'costs.scan.dropActive': 'Drop the photo to scan it',
|
||||
'costs.scan.start': 'Scan',
|
||||
'costs.scan.unsupported': 'Only a photo can be scanned: JPG, PNG or WEBP.',
|
||||
'costs.noReceipts': 'No receipts attached',
|
||||
'costs.deleteReceipt': 'Remove receipt',
|
||||
'costs.viewReceipt': 'View receipt',
|
||||
|
||||
@@ -133,6 +133,7 @@ const reservations: TranslationStrings = {
|
||||
'reservations.import.dropHere': 'Drop booking confirmation files here, or click to select',
|
||||
'reservations.import.dropActive': 'Drop files to import',
|
||||
'reservations.import.acceptedFormats': 'Accepted: EML, PDF, PKPass, HTML, TXT (max 10 MB each, up to 5 files)',
|
||||
'reservations.import.acceptedPhotos': 'Photos (JPG, PNG, WEBP) are read by the AI model.',
|
||||
'reservations.import.parsing': 'Parsing files…',
|
||||
'reservations.import.previewHeading': '{count} reservation(s) found',
|
||||
'reservations.import.previewEmpty': 'No reservations could be extracted from the uploaded files.',
|
||||
|
||||
@@ -400,8 +400,8 @@ const settings: TranslationStrings = {
|
||||
'settings.aiParsing.baseUrlHint': 'Where the model runs — a local Ollama server or an OpenAI-compatible endpoint.',
|
||||
'settings.aiParsing.apiKey': 'API key',
|
||||
'settings.aiParsing.apiKeyHint': 'Stored encrypted. Leave blank to keep the current key.',
|
||||
'settings.aiParsing.multimodal': 'Send documents as images',
|
||||
'settings.aiParsing.multimodalHint': 'For vision-capable models — sends the original PDF instead of extracted text.',
|
||||
'settings.aiParsing.multimodal': 'Model reads images',
|
||||
'settings.aiParsing.multimodalHint': 'Turn on for a model that reads images, so a photo can be imported or scanned.',
|
||||
'settings.aiParsing.toast.saved': 'AI settings saved',
|
||||
'settings.aiParsing.toast.saveError': 'Could not save AI settings',
|
||||
'settings.tabs.appearance': 'Appearance',
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user