k1nq/TREK
1
0
Fork 0
mirror of https://github.com/mauriceboe/TREK.git synced 2026-10-23 23:46:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm-parse): read a photo, as a booking or as a receipt in Costs

Browse Source 
Whether the AI Parsing model reads images is an admin setting now, and it
lands with the two features that send a photo, so it has something to
control.

The setting is `vision: 'auto' | 'on' | 'off'` on the addon config. On
Automatic a local Ollama is asked what the model can do (`/api/show`), and a
cloud provider means no; Yes and No are applied as set. The old `multimodal`
flag is not read: the admin screens wrote it false on every save, so a
stored false meant nothing. The per-user toggle keeps its on/off meaning and
is read by the extraction now. The lookup is server-side only: capped read,
validated body, never throws, cached per server and model.
GET /api/llm/capabilities answers it for the signed-in caller, and both
photo pickers below ask it.

The booking import takes JPG, PNG and WEBP when the model reads images.
The photo goes as an image to every provider (image_url, an Anthropic image
block, Ollama's native chat with `images`), capped at 1600px, and skips KDE
Itinerary. A scanned PDF, which used to reach the model as empty text, has
its first two pages drawn by pdf-parse and sent the same way. The rest is
the import as it was: background job, review, the file attached to the
booking, the linked cost. On a real qwen3.5:4b a train ticket came back as
bare clock times until the router gave the model every type's field
meanings, and its price as an object inside a string, which read as 4900;
both are handled.

Costs gets Scan receipt: a dialog laid out like the booking import's takes
one photo, read on the import job queue, then reviewed in the existing
expense editor pre-filled with merchant, total, currency, day and lines,
the photo waiting in Receipts & Invoices to be uploaded on save. No table,
no migration, no category guess, no document-type detection.
This commit is contained in:
RemEx
2026-09-24 17:12:54 +00:00
committed by Maurice
parent 706c6c3959
commit b1c05bca83
183 changed files with 3098 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/components/BackgroundTasks/BackgroundTasksWidget.extra.test.tsx
+41
View File
@@ -304,3 +304,44 @@ describe('BackgroundTasksWidget: the mark on a finished job (#2477)', () => {
expect(baseElement.querySelector('svg.lucide-alert-triangle')).toBeNull()
})
})
describe('BackgroundTasksWidget — a receipt scanned from Costs', () => {
const RECEIPT = { merchant: 'Café', date: '2026-09-20', total: 12.5, currency: 'EUR', items: [] }
it('FE-W4BGT-025: says it is reading the receipt while the job runs', () => {
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
render(<BackgroundTasksWidget />)
expect(screen.getByText('Reading the receipt…')).toBeInTheDocument()
})
it('FE-W4BGT-026: import:done with a read receipt offers to review the expense, which opens the trip', () => {
useBackgroundTasksStore.setState({ tasks: [task({ status: 'running', kind: 'costs', label: 'bill.jpg', items: undefined })] })
render(<BackgroundTasksWidget />)
const handler = vi.mocked(addListener).mock.calls[0][0] as WsHandler
act(() => { handler({ type: 'import:done', jobId: 'j1', tripId: 't1', result: { receipt: RECEIPT, warnings: [] } }) })
expect(useBackgroundTasksStore.getState().tasks[0].receipt).toEqual(RECEIPT)
fireEvent.click(screen.getByRole('button', { name: 'Review expense' }))
expect(useBackgroundTasksStore.getState().tasks[0].reviewRequested).toBe(true)
expect(navigate).toHaveBeenCalledWith('/trips/t1')
})
it('FE-W4BGT-027: a photo nothing could be read from says so, with its warning and no AI retry', async () => {
vi.mocked(healthApi.features).mockResolvedValue({ bookingImport: true, aiParsing: true })
const photo = new File(['x'], 'bill.jpg', { type: 'image/jpeg' })
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', receipt: null, warnings: ['bill.jpg: no receipt could be read'], sourceFiles: [photo] })] })
render(<BackgroundTasksWidget />)
expect(screen.getByText('No receipt could be read from this photo.')).toBeInTheDocument()
expect(screen.getByText('bill.jpg: no receipt could be read')).toBeInTheDocument()
await waitFor(() => expect(healthApi.features).toHaveBeenCalled())
expect(screen.queryByRole('button', { name: 'Try AI parsing' })).not.toBeInTheDocument()
})
it('FE-W4BGT-028: the reload reconcile reads a receipt job\'s status too', async () => {
vi.mocked(reservationsApi.importJobStatus).mockResolvedValue({ status: 'done', done: 1, total: 1, result: { receipt: RECEIPT, warnings: [] } as never })
useBackgroundTasksStore.setState({ tasks: [task({ kind: 'costs', items: undefined })] })
render(<BackgroundTasksWidget />)
expect(await screen.findByRole('button', { name: 'Review expense' })).toBeInTheDocument()
})
})
client/src/components/BackgroundTasks/BackgroundTasksWidget.tsx
+25 -17
View File
@@ -6,7 +6,18 @@ import { useTranslation } from '../../i18n'
import { addListener, removeListener } from '../../api/websocket'
import { reservationsApi, healthApi } from '../../api/client'
import { saveImportFiles } from '../../db/offlineDb'
import { useBackgroundTasksStore, type BackgroundImportTask } from '../../store/backgroundTasksStore'
import { useBackgroundTasksStore, taskFoundSomething, type BackgroundImportTask } from '../../store/backgroundTasksStore'
import type { ReceiptRead } from '@trek/shared'
/**
* One reading of a finished job's result, wherever it arrives from (the push,
* the reload reconcile, the poll): a booking parse answers `items`, a receipt
* scan `receipt`.
*/
function settleJob(id: string, tripId: string, result: unknown): void {
const r = result as { items?: unknown[]; warnings?: string[]; receipt?: ReceiptRead | null } | undefined
useBackgroundTasksStore.getState().setDone(id, tripId, (r?.items ?? []) as never, r?.warnings ?? [], r?.receipt)
}
/**
* Global, route-independent widget (bottom-right) that tracks background booking
@@ -20,7 +31,6 @@ export default function BackgroundTasksWidget() {
const navigate = useNavigate()
const tasks = useBackgroundTasksStore((s) => s.tasks)
const setProgress = useBackgroundTasksStore((s) => s.setProgress)
const setDone = useBackgroundTasksStore((s) => s.setDone)
const setError = useBackgroundTasksStore((s) => s.setError)
const requestReview = useBackgroundTasksStore((s) => s.requestReview)
const dismiss = useBackgroundTasksStore((s) => s.dismiss)
@@ -66,7 +76,7 @@ export default function BackgroundTasksWidget() {
reservationsApi
.importJobStatus(task.tripId, task.id)
.then((s) => {
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
else setProgress(task.id, task.tripId, s.done, s.total)
})
@@ -87,14 +97,12 @@ export default function BackgroundTasksWidget() {
const tripId = String(e.tripId ?? '')
if (!id) return
if (type === 'import:progress') setProgress(id, tripId, Number(e.done ?? 0), Number(e.total ?? 1))
else if (type === 'import:done') {
const result = e.result as { items?: unknown[]; warnings?: string[] } | undefined
setDone(id, tripId, (result?.items ?? []) as never, result?.warnings ?? [])
} else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
else if (type === 'import:done') settleJob(id, tripId, e.result)
else if (type === 'import:error') setError(id, tripId, String(e.message ?? 'error'))
}
addListener(handler)
return () => removeListener(handler)
}, [setProgress, setDone, setError])
}, [setProgress, setError])
// Backstop: poll jobs whose state we still need — running ones (in case a WebSocket push
// was missed) and a restored 'done' task whose items haven't been re-fetched yet (so a
@@ -107,7 +115,7 @@ export default function BackgroundTasksWidget() {
reservationsApi
.importJobStatus(task.tripId, task.id)
.then((s) => {
if (s.status === 'done') setDone(task.id, task.tripId, (s.result?.items ?? []) as never, s.result?.warnings ?? [])
if (s.status === 'done') settleJob(task.id, task.tripId, s.result)
else if (s.status === 'error') setError(task.id, task.tripId, s.error ?? 'error')
else setProgress(task.id, task.tripId, s.done, s.total)
})
@@ -120,7 +128,7 @@ export default function BackgroundTasksWidget() {
}
}, 5000)
return () => clearInterval(iv)
}, [tasks, setProgress, setDone, setError, t])
}, [tasks, setProgress, setError, t])
if (tasks.length === 0) return null
@@ -141,11 +149,11 @@ export default function BackgroundTasksWidget() {
>
<div style={{ flexShrink: 0, marginTop: 1 }}>
{(task.status === 'running' || (task.status === 'done' && task.items === undefined)) && <Loader2 size={16} className="animate-spin" color="var(--accent)" />}
{task.status === 'done' && task.items !== undefined && task.items.length > 0 && <CheckCircle2 size={16} color="var(--success)" />}
{task.status === 'done' && task.items !== undefined && taskFoundSomething(task) && <CheckCircle2 size={16} color="var(--success)" />}
{/* A parse that found nothing is not a success, whatever the job
status says: the same card used to show a green tick next to
"no reservations could be extracted" (#2477). */}
{task.status === 'done' && task.items?.length === 0 && <AlertTriangle size={16} color="var(--warning)" />}
{task.status === 'done' && task.items !== undefined && !taskFoundSomething(task) && <AlertTriangle size={16} color="var(--warning)" />}
{task.status === 'error' && <AlertCircle size={16} color="var(--danger)" />}
</div>
@@ -156,7 +164,7 @@ export default function BackgroundTasksWidget() {
{task.status === 'running' && (
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
{t('reservations.import.parsing')}
{t(task.kind === 'costs' ? 'costs.scan.reading' : 'reservations.import.parsing')}
{task.total > 1 ? ` · ${task.done}/${task.total}` : ''}
</div>
)}
@@ -165,14 +173,14 @@ export default function BackgroundTasksWidget() {
task.items === undefined ? (
// Restored from a reload; items are being re-fetched (see the poll backstop).
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>{t('reservations.import.parsing')}</div>
) : task.items.length > 0 ? (
) : taskFoundSomething(task) ? (
<div>
<button type="button"
onClick={() => review(task)}
className="bg-accent text-accent-text"
style={{ marginTop: 4, border: 'none', borderRadius: 8, padding: '4px 12px', fontSize: 'calc(11.5px * var(--fs-scale-caption, 1))', fontWeight: 600, cursor: 'pointer', fontFamily: 'inherit' }}
>
{t('common.import')}
{t(task.kind === 'costs' ? 'costs.scan.review' : 'common.import')}
</button>
{/* A partly-understood import warns too. Warnings used to be
shown only when nothing at all was found, so anything the
@@ -188,14 +196,14 @@ export default function BackgroundTasksWidget() {
) : (
<div>
<div style={{ fontSize: 'calc(11px * var(--fs-scale-caption, 1))', color: 'var(--text-faint)', marginTop: 1 }}>
{t('reservations.import.previewEmpty')}
{t(task.kind === 'costs' ? 'costs.scan.nothingRead' : 'reservations.import.previewEmpty')}
{(task.warnings?.length ?? 0) > 0 && (
<div style={{ color: '#b45309', marginTop: 3, whiteSpace: 'pre-wrap', wordBreak: 'break-word', maxHeight: 96, overflowY: 'auto' }}>
{task.warnings!.join('\n')}
</div>
)}
</div>
{aiParsing && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
{aiParsing && task.kind !== 'costs' && task.mode !== 'force-ai' && task.sourceFiles && task.sourceFiles.length > 0 && (
<button type="button"
onClick={() => retryWithAi(task)}
disabled={retrying === task.id}