diff --git a/server/src/app-config/env.ts b/server/src/app-config/env.ts index 2d6d34ed2..7187b749a 100644 --- a/server/src/app-config/env.ts +++ b/server/src/app-config/env.ts @@ -33,6 +33,8 @@ export const SECRET_ENV_KEYS: ReadonlySet = new Set([ 'MAPBOX_ACCESS_TOKEN', 'CARTO_API_KEY', 'DEMO_ADMIN_PASS', + 'MAIL_INGEST_IMAP_PASS', + 'MAIL_INGEST_WEBHOOK_SECRET', ]); /** How a variable's value appears in the boot report. */ diff --git a/server/src/nest/llm-parse/mime-walker.ts b/server/src/nest/llm-parse/mime-walker.ts index a9c5dcec7..e2f8fe530 100644 --- a/server/src/nest/llm-parse/mime-walker.ts +++ b/server/src/nest/llm-parse/mime-walker.ts @@ -284,9 +284,20 @@ function safeFileName(name: string, kind: MailAttachmentKind, position: number): /** Line breaks and block ends, so the header lines of a forwarded HTML mail stay lines without their tags. */ const HTML_LINE_BREAK = /<(?:br|hr)\b[^>]*>|<\/(?:p|div|tr|li|h[1-6]|blockquote|table)\s*>/gi; -/** A section as text: its HTML stripped when it has a readable one, else its plain text. */ +/** + * A section as text: its HTML stripped when it has a readable one, else its plain text. + * Each line of the HTML is trimmed: a stripped tag leaves a space in front of a + * line, which the forward unwrapping would read as a folded header continuation, + * dropping the booking under a forwarded header block with it. HTML folds no headers. + */ function sectionText(section: Section): string { - const html = section.html === null ? '' : stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n')); + const html = + section.html === null + ? '' + : stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n')) + .split('\n') + .map((line) => line.trim()) + .join('\n'); return html || (section.text === null ? '' : collapseWhitespace(section.text)); } diff --git a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts index 39f623d24..067d18675 100644 --- a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts @@ -100,8 +100,10 @@ export class MailIngestSettingsService { return this.config.webhookSecret ?? (decrypt_api_key(stored) || null); } + /** The AI reads a person gets a day; an unset or blank setting is the default, not zero (`Number('')` is 0). */ async aiDailyCap(): Promise { - const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily')); + const stored = (await this.settings.getAppSetting('mail_ingest_ai_per_user_daily'))?.trim(); + const value = stored ? Number(stored) : Number.NaN; return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY; } diff --git a/server/tests/e2e/mail-ingest.e2e.test.ts b/server/tests/e2e/mail-ingest.e2e.test.ts index b59915a6b..9432396d0 100644 --- a/server/tests/e2e/mail-ingest.e2e.test.ts +++ b/server/tests/e2e/mail-ingest.e2e.test.ts @@ -13,6 +13,7 @@ import { importExternalId } from '../../src/nest/booking-import/booking-origin'; import { encrypt_api_key } from '../../src/nest/common/crypto/apiKeyCrypto'; import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { withRequestContext } from '../../src/nest/database/request-context'; import { FilesService } from '../../src/nest/files/files.service'; import { MailReadService } from '../../src/nest/llm-parse/mail-read.service'; import { walkMail } from '../../src/nest/llm-parse/mime-walker'; @@ -29,6 +30,7 @@ import { makeAdmin, makeUser } from '../helpers/factories/users'; import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm'; import { TestUnitOfWorkModule } from '../helpers/test-uow'; import { sessionCookie } from './harness'; +import { MikroORM } from '@mikro-orm/core'; import { Test } from '@nestjs/testing'; import type { BookingImportPreviewItem } from '@trek/shared'; @@ -128,8 +130,11 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => { const as = (userId: number) => ({ Cookie: sessionCookie(userId) }); - /** Runs the pipeline over what came in, the way its two-minute job does. */ - const readQueued = () => app.get(MailIngestProcessJob).tick(); + /** + * Runs the pipeline over what came in, the way its two-minute job does: the + * cron registrar runs every tick in a request context of its own. + */ + const readQueued = () => withRequestContext(app.get(MikroORM), () => app.get(MailIngestProcessJob).tick()); beforeAll(async () => { orm = await createTestOrm(db); @@ -150,9 +155,9 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => { }); it('MAIL-E2E-001: 401 without a session on every route a person or an admin uses', async () => { - for (const path of ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status']) { - expect((await request(server).get(path)).status, path).toBe(401); - } + const paths = ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status']; + const answers = await Promise.all(paths.map((path) => request(server).get(path))); + answers.forEach((answer, i) => expect(answer.status, paths[i]).toBe(401)); expect((await request(server).post(`/api/trips/${tripId}/mail-ingest/address`)).status).toBe(401); }); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts index a086962a6..2f9b46c5e 100644 --- a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts +++ b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts @@ -84,7 +84,10 @@ const files = { createFileFromBytes: vi.fn(async () => ({ id: 77 })), softDeleteFile: vi.fn(async () => undefined), }; -const notifications = { send: vi.fn(async () => ({ attempted: 1, delivered: 1 })), registerResponseAction: vi.fn() }; +const notifications = { + send: vi.fn(async (_payload: { event: string; scope: string }) => ({ attempted: 1, delivered: 1 })), + registerResponseAction: vi.fn(), +}; const db = createSnapshotTestDb(); let t: TestModule; @@ -322,8 +325,10 @@ describe('a booking that passes every check', () => { it('is not moved by a demo account', async () => { const user = await person(); await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); - const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' }); + // The booking is added to the one trip in its dates first; a second one then would leave it waiting. const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + expect(entry).toMatchObject({ resolution: 'auto_applied' }); + const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' }); const demo = vi.spyOn(t.get(DemoService), 'isDemoUserId').mockResolvedValue(true); try { await expect(actions().move(entry.id, user.id, second.id, true, NOW)).rejects.toThrow( @@ -509,7 +514,8 @@ describe('what the person sees', () => { reason: 'ask_first', suggested_trip: { id: trip.id, title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }, undo_until: null, - message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Fwd: Your flight LH 716' }, + // The subject is kept without the prefixes forwarding stacked onto it. + message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Your flight LH 716' }, }); expect(page.entries[0].message).not.toHaveProperty('raw_eml_enc'); expect((await inbox().inbox(other.id, { status: 'open', limit: 50 }, NOW)).entries).toEqual([]); @@ -538,7 +544,7 @@ describe('what the person sees', () => { const messageId = await forward(user.id, eml('nothing')); const preview = await inbox().preview(user.id, messageId); - expect(preview).toMatchObject({ id: messageId, subject: 'Fwd: Your flight LH 716', attachments: [] }); + expect(preview).toMatchObject({ id: messageId, subject: 'Your flight LH 716', attachments: [] }); expect(preview.text).toContain('Booking reference ABC123'); expect(preview.text).not.toContain('mia@example.com'); await expect(inbox().preview(other.id, messageId)).rejects.toThrow('Mail not found'); @@ -561,6 +567,18 @@ describe('what the person sees', () => { expect(await inbox().count(user.id)).toEqual({ pending: 0 }); }); + it('keeps AI reading offered while the person switched it off, so the switch can come back on', async () => { + const user = await person(); + reader.isAvailable.mockResolvedValue(true); + const updated = await inbox().updatePrefs(user.id, { ai_enabled: false }); + expect(updated).toMatchObject({ prefs: { ai_enabled: false }, ai_available: true }); + expect(await inbox().settingsOf(user.id)).toMatchObject({ + available: true, + prefs: { enabled: true, ai_enabled: false, auto_add: true }, + ai_available: true, + }); + }); + it('reads an uploaded mail at once, knows one it has, and refuses what it cannot take', async () => { const user = await person(); const soon = vi.spyOn(job(), 'runSoon');