From 89b67a791f0158f81c581bbf71dae83f6b484931 Mon Sep 17 00:00:00 2001 From: Maurice Date: Fri, 13 Nov 2026 19:31:13 +0100 Subject: [PATCH] fix(mail-ingest): Outlook HTML forwards, the default AI limit, secret env vars, and the pipeline tests - mime-walker: trim each line of a stripped HTML body, so the space a tag leaves is not read as a folded header line that swallows the booking under an Outlook forward's header block. - settings: an unset daily AI limit is the default of ten, not zero (Number(null) is 0), which kept the AI from ever reading a mail. - env: MAIL_INGEST_IMAP_PASS and MAIL_INGEST_WEBHOOK_SECRET are secrets and never printed in the boot report. - pipeline test: the subject is kept without its forward prefixes, the demo move adds its booking before the second trip exists, the typed notification mock, and AI reading stays offered while the person switched it off. - e2e: the pipeline tick runs in a request context of its own, as the cron registrar runs it, and the 401 checks run without awaiting in a loop. --- server/src/app-config/env.ts | 2 ++ server/src/nest/llm-parse/mime-walker.ts | 15 +++++++++-- .../mail-ingest-settings.service.ts | 4 ++- server/tests/e2e/mail-ingest.e2e.test.ts | 15 +++++++---- .../mail-ingest/mail-ingest.pipeline.test.ts | 26 ++++++++++++++++--- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/server/src/app-config/env.ts b/server/src/app-config/env.ts index 2d6d34ed2..7187b749a 100644 --- a/server/src/app-config/env.ts +++ b/server/src/app-config/env.ts @@ -33,6 +33,8 @@ export const SECRET_ENV_KEYS: ReadonlySet = new Set([ 'MAPBOX_ACCESS_TOKEN', 'CARTO_API_KEY', 'DEMO_ADMIN_PASS', + 'MAIL_INGEST_IMAP_PASS', + 'MAIL_INGEST_WEBHOOK_SECRET', ]); /** How a variable's value appears in the boot report. */ diff --git a/server/src/nest/llm-parse/mime-walker.ts b/server/src/nest/llm-parse/mime-walker.ts index a9c5dcec7..e2f8fe530 100644 --- a/server/src/nest/llm-parse/mime-walker.ts +++ b/server/src/nest/llm-parse/mime-walker.ts @@ -284,9 +284,20 @@ function safeFileName(name: string, kind: MailAttachmentKind, position: number): /** Line breaks and block ends, so the header lines of a forwarded HTML mail stay lines without their tags. */ const HTML_LINE_BREAK = /<(?:br|hr)\b[^>]*>|<\/(?:p|div|tr|li|h[1-6]|blockquote|table)\s*>/gi; -/** A section as text: its HTML stripped when it has a readable one, else its plain text. */ +/** + * A section as text: its HTML stripped when it has a readable one, else its plain text. + * Each line of the HTML is trimmed: a stripped tag leaves a space in front of a + * line, which the forward unwrapping would read as a folded header continuation, + * dropping the booking under a forwarded header block with it. HTML folds no headers. + */ function sectionText(section: Section): string { - const html = section.html === null ? '' : stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n')); + const html = + section.html === null + ? '' + : stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n')) + .split('\n') + .map((line) => line.trim()) + .join('\n'); return html || (section.text === null ? '' : collapseWhitespace(section.text)); } diff --git a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts index 39f623d24..067d18675 100644 --- a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts @@ -100,8 +100,10 @@ export class MailIngestSettingsService { return this.config.webhookSecret ?? (decrypt_api_key(stored) || null); } + /** The AI reads a person gets a day; an unset or blank setting is the default, not zero (`Number('')` is 0). */ async aiDailyCap(): Promise { - const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily')); + const stored = (await this.settings.getAppSetting('mail_ingest_ai_per_user_daily'))?.trim(); + const value = stored ? Number(stored) : Number.NaN; return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY; } diff --git a/server/tests/e2e/mail-ingest.e2e.test.ts b/server/tests/e2e/mail-ingest.e2e.test.ts index b59915a6b..9432396d0 100644 --- a/server/tests/e2e/mail-ingest.e2e.test.ts +++ b/server/tests/e2e/mail-ingest.e2e.test.ts @@ -13,6 +13,7 @@ import { importExternalId } from '../../src/nest/booking-import/booking-origin'; import { encrypt_api_key } from '../../src/nest/common/crypto/apiKeyCrypto'; import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { withRequestContext } from '../../src/nest/database/request-context'; import { FilesService } from '../../src/nest/files/files.service'; import { MailReadService } from '../../src/nest/llm-parse/mail-read.service'; import { walkMail } from '../../src/nest/llm-parse/mime-walker'; @@ -29,6 +30,7 @@ import { makeAdmin, makeUser } from '../helpers/factories/users'; import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm'; import { TestUnitOfWorkModule } from '../helpers/test-uow'; import { sessionCookie } from './harness'; +import { MikroORM } from '@mikro-orm/core'; import { Test } from '@nestjs/testing'; import type { BookingImportPreviewItem } from '@trek/shared'; @@ -128,8 +130,11 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => { const as = (userId: number) => ({ Cookie: sessionCookie(userId) }); - /** Runs the pipeline over what came in, the way its two-minute job does. */ - const readQueued = () => app.get(MailIngestProcessJob).tick(); + /** + * Runs the pipeline over what came in, the way its two-minute job does: the + * cron registrar runs every tick in a request context of its own. + */ + const readQueued = () => withRequestContext(app.get(MikroORM), () => app.get(MailIngestProcessJob).tick()); beforeAll(async () => { orm = await createTestOrm(db); @@ -150,9 +155,9 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => { }); it('MAIL-E2E-001: 401 without a session on every route a person or an admin uses', async () => { - for (const path of ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status']) { - expect((await request(server).get(path)).status, path).toBe(401); - } + const paths = ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status']; + const answers = await Promise.all(paths.map((path) => request(server).get(path))); + answers.forEach((answer, i) => expect(answer.status, paths[i]).toBe(401)); expect((await request(server).post(`/api/trips/${tripId}/mail-ingest/address`)).status).toBe(401); }); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts index a086962a6..2f9b46c5e 100644 --- a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts +++ b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts @@ -84,7 +84,10 @@ const files = { createFileFromBytes: vi.fn(async () => ({ id: 77 })), softDeleteFile: vi.fn(async () => undefined), }; -const notifications = { send: vi.fn(async () => ({ attempted: 1, delivered: 1 })), registerResponseAction: vi.fn() }; +const notifications = { + send: vi.fn(async (_payload: { event: string; scope: string }) => ({ attempted: 1, delivered: 1 })), + registerResponseAction: vi.fn(), +}; const db = createSnapshotTestDb(); let t: TestModule; @@ -322,8 +325,10 @@ describe('a booking that passes every check', () => { it('is not moved by a demo account', async () => { const user = await person(); await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); - const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' }); + // The booking is added to the one trip in its dates first; a second one then would leave it waiting. const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + expect(entry).toMatchObject({ resolution: 'auto_applied' }); + const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' }); const demo = vi.spyOn(t.get(DemoService), 'isDemoUserId').mockResolvedValue(true); try { await expect(actions().move(entry.id, user.id, second.id, true, NOW)).rejects.toThrow( @@ -509,7 +514,8 @@ describe('what the person sees', () => { reason: 'ask_first', suggested_trip: { id: trip.id, title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }, undo_until: null, - message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Fwd: Your flight LH 716' }, + // The subject is kept without the prefixes forwarding stacked onto it. + message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Your flight LH 716' }, }); expect(page.entries[0].message).not.toHaveProperty('raw_eml_enc'); expect((await inbox().inbox(other.id, { status: 'open', limit: 50 }, NOW)).entries).toEqual([]); @@ -538,7 +544,7 @@ describe('what the person sees', () => { const messageId = await forward(user.id, eml('nothing')); const preview = await inbox().preview(user.id, messageId); - expect(preview).toMatchObject({ id: messageId, subject: 'Fwd: Your flight LH 716', attachments: [] }); + expect(preview).toMatchObject({ id: messageId, subject: 'Your flight LH 716', attachments: [] }); expect(preview.text).toContain('Booking reference ABC123'); expect(preview.text).not.toContain('mia@example.com'); await expect(inbox().preview(other.id, messageId)).rejects.toThrow('Mail not found'); @@ -561,6 +567,18 @@ describe('what the person sees', () => { expect(await inbox().count(user.id)).toEqual({ pending: 0 }); }); + it('keeps AI reading offered while the person switched it off, so the switch can come back on', async () => { + const user = await person(); + reader.isAvailable.mockResolvedValue(true); + const updated = await inbox().updatePrefs(user.id, { ai_enabled: false }); + expect(updated).toMatchObject({ prefs: { ai_enabled: false }, ai_available: true }); + expect(await inbox().settingsOf(user.id)).toMatchObject({ + available: true, + prefs: { enabled: true, ai_enabled: false, auto_add: true }, + ai_available: true, + }); + }); + it('reads an uploaded mail at once, knows one it has, and refuses what it cannot take', async () => { const user = await person(); const soon = vi.spyOn(job(), 'runSoon');