feat(mail-ingest): read forwarded mails from an IMAP box, a signed webhook and the TREK Cloud relay

The IMAP job reads the box every two minutes through the SSRF guard, keeps a uid cursor and moves read mails to TREK/processed. The webhook takes raw MIME signed with HMAC-SHA256 over a timestamp, the recipient and the mail, and is a 404 unless it is the transport. The relay job collects a managed install's mails.
2026-11-14 00:01:03 +00:00 · 2026-11-13 16:42:43 +01:00
parent 6309064f40
commit 1bfaf4ea98
15 changed files with 1256 additions and 7 deletions
@@ -0,0 +1,112 @@
+/**
+ * The IMAP client the import reads a box with, over a stand-in for imapflow:
+ * the host goes through the SSRF guard and the connection to the address it
+ * vetted, the certificate checked against the name; the session filters the
+ * newest mail IMAP always names, makes the processed folder, moves by uid and
+ * says goodbye even over a broken connection.
+ */
+import { MailIngestImapClient } from '../../../../src/nest/mail-ingest/mail-ingest-imap.client';
+import { SsrfBlockedError } from '../../../../src/utils/ssrfGuard';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { checkSsrf, flow, ImapFlow } = vi.hoisted(() => {
+  const flow = {
+    connect: vi.fn(async () => undefined),
+    mailboxOpen: vi.fn(async () => ({ uidValidity: 77n })),
+    fetch: vi.fn(),
+    fetchOne: vi.fn(),
+    mailboxCreate: vi.fn(async () => ({ path: 'TREK.processed', created: false })),
+    messageMove: vi.fn(async () => true),
+    logout: vi.fn(async () => undefined),
+    close: vi.fn(),
+  };
+  // A function, not an arrow: the client builds it with `new`.
+  const ImapFlow = vi.fn(function () {
+    return flow;
+  });
+  return { checkSsrf: vi.fn(), flow, ImapFlow };
+});
+vi.mock('imapflow', () => ({ ImapFlow }));
+vi.mock('../../../../src/utils/ssrfGuard', async (orig) => ({
+  ...(await orig<Record<string, unknown>>()),
+  checkSsrf,
+}));
+
+const CONFIG = {
+  host: 'imap.example.com',
+  port: 143,
+  tls: 'starttls',
+  user: 'u',
+  pass: 'p',
+  mailbox: 'INBOX',
+} as const;
+
+async function* messages(...uids: number[]) {
+  for (const uid of uids) yield { uid, size: uid * 10 };
+}
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  checkSsrf.mockResolvedValue({ allowed: true, isPrivate: false, resolvedIp: '203.0.113.5' });
+});
+
+describe('MailIngestImapClient', () => {
+  it('connects to the vetted address, checking the certificate against the name', async () => {
+    await new MailIngestImapClient().connect(CONFIG);
+    expect(checkSsrf).toHaveBeenCalledWith('https://imap.example.com/');
+    expect(ImapFlow).toHaveBeenCalledWith(
+      expect.objectContaining({
+        host: '203.0.113.5',
+        port: 143,
+        secure: false,
+        doSTARTTLS: true,
+        servername: 'imap.example.com',
+        auth: { user: 'u', pass: 'p' },
+        logger: false,
+      }),
+    );
+    expect(flow.connect).toHaveBeenCalled();
+  });
+
+  it('refuses a host the guard blocks, before any connection', async () => {
+    checkSsrf.mockResolvedValue({ allowed: false, isPrivate: true, error: 'Private addresses are not allowed' });
+    await expect(new MailIngestImapClient().connect(CONFIG)).rejects.toBeInstanceOf(SsrfBlockedError);
+    expect(ImapFlow).not.toHaveBeenCalled();
+  });
+
+  it('reads a box through one session: uid validity, the mails after the cursor, a source, the move', async () => {
+    const session = await new MailIngestImapClient().connect({ ...CONFIG, host: '2001:db8::1', tls: 'tls' });
+    expect(checkSsrf).toHaveBeenCalledWith('https://[2001:db8::1]/');
+    expect(ImapFlow).toHaveBeenCalledWith(expect.objectContaining({ secure: true, servername: undefined }));
+
+    expect(await session.open('INBOX')).toEqual({ uidValidity: 77 });
+    // `41:*` names the newest mail even when it is older than 41.
+    flow.fetch.mockReturnValue(messages(40, 42, 41));
+    expect(await session.listAfter(40)).toEqual([
+      { uid: 41, size: 410 },
+      { uid: 42, size: 420 },
+    ]);
+    expect(flow.fetch).toHaveBeenCalledWith('41:*', { uid: true, size: true }, { uid: true });
+
+    flow.fetchOne.mockResolvedValueOnce({ uid: 41, source: Buffer.from('raw') }).mockResolvedValueOnce(false);
+    expect(await session.source(41)).toEqual(Buffer.from('raw'));
+    expect(await session.source(43)).toBeNull();
+
+    expect(await session.ensureFolder(['TREK', 'processed'])).toBe('TREK.processed');
+    expect(flow.mailboxCreate).toHaveBeenCalledWith(['TREK', 'processed']);
+    await session.move([41, 42], 'TREK.processed');
+    expect(flow.messageMove).toHaveBeenCalledWith('41,42', 'TREK.processed', { uid: true });
+    await session.move([], 'TREK.processed');
+    expect(flow.messageMove).toHaveBeenCalledTimes(1);
+  });
+
+  it('logs out, and closes a connection that broke on its own', async () => {
+    const session = await new MailIngestImapClient().connect(CONFIG);
+    await session.close();
+    expect(flow.logout).toHaveBeenCalled();
+    flow.logout.mockRejectedValueOnce(new Error('socket closed'));
+    await session.close();
+    expect(flow.close).toHaveBeenCalled();
+  });
+});
@@ -0,0 +1,350 @@
+/**
+ * How mail reaches the import and how the admin sees it run, over stubs: the
+ * signed webhook (its window, its signature over the recipient and the raw
+ * mail, its refusals), the IMAP run (cursor, renumbered box, size limit, the
+ * move to TREK/processed, a failure the status card reads), the TREK Cloud
+ * relay (hash check, acknowledgement, idle without a relay) and the status
+ * card and connection test.
+ */
+import { MailIngestCandidates } from '../../../../src/db/entities/MailIngestCandidates.entity';
+import { MailIngestCursor } from '../../../../src/db/entities/MailIngestCursor.entity';
+import { MailIngestMessages } from '../../../../src/db/entities/MailIngestMessages.entity';
+import { DomainError } from '../../../../src/nest/common/domain-error';
+import {
+  MailIngestImapClient,
+  describeImapFailure,
+  type ImapSession,
+} from '../../../../src/nest/mail-ingest/mail-ingest-imap.client';
+import { MailIngestImapJob } from '../../../../src/nest/mail-ingest/mail-ingest-imap.job';
+import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service';
+import { MailIngestRelayJob } from '../../../../src/nest/mail-ingest/mail-ingest-relay.job';
+import { MailIngestRelayHost } from '../../../../src/nest/mail-ingest/mail-ingest-relay.service';
+import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service';
+import { MailIngestStatusService } from '../../../../src/nest/mail-ingest/mail-ingest-status.service';
+import {
+  MailIngestWebhookService,
+  signWebhookMail,
+} from '../../../../src/nest/mail-ingest/mail-ingest-webhook.service';
+import { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service';
+import { SsrfBlockedError } from '../../../../src/utils/ssrfGuard';
+import { getRepositoryToken } from '@mikro-orm/nestjs';
+import { Test } from '@nestjs/testing';
+
+import { createHash } from 'node:crypto';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('../../../../src/nest/audit/audit-log.logger', () => ({
+  LOG_LEVEL: 'error',
+  logInfo: vi.fn(),
+  logDebug: vi.fn(),
+  logError: vi.fn(),
+  logWarn: vi.fn(),
+}));
+
+const IMAP = { host: 'imap.example.com', port: 993, tls: 'tls', user: 'u', pass: 'p', mailbox: 'INBOX' } as const;
+
+const settings = {
+  isAvailable: vi.fn(async () => true),
+  transport: vi.fn(async () => 'webhook'),
+  webhookSecret: vi.fn(async (): Promise<string | null> => 'a'.repeat(32)),
+  imapConfig: vi.fn(async (): Promise<typeof IMAP | null> => IMAP),
+  maxMailBytes: vi.fn(() => 1000),
+};
+const intake = { receive: vi.fn() };
+const cursor = {
+  acquireLease: vi.fn(async () => true),
+  releaseLease: vi.fn(async () => undefined),
+  recordOk: vi.fn(async () => undefined),
+  recordError: vi.fn(async () => undefined),
+  find: vi.fn(),
+};
+const messages = { tallySince: vi.fn(), countAllAiReadSince: vi.fn() };
+const candidates = { tallyResolutionsSince: vi.fn(), pendingPerUser: vi.fn() };
+const session = {
+  open: vi.fn(),
+  listAfter: vi.fn(),
+  source: vi.fn(),
+  ensureFolder: vi.fn(async () => 'TREK.processed'),
+  move: vi.fn(async () => undefined),
+  close: vi.fn(async () => undefined),
+};
+const imap = { connect: vi.fn(async (): Promise<ImapSession> => session) };
+const registrar = { isEnabled: () => false, register: vi.fn() };
+let relays: MailIngestRelayHost;
+
+async function build() {
+  const moduleRef = await Test.createTestingModule({
+    providers: [
+      MailIngestWebhookService,
+      MailIngestImapJob,
+      MailIngestRelayJob,
+      MailIngestStatusService,
+      { provide: MailIngestRelayHost, useValue: relays },
+      { provide: MailIngestSettingsService, useValue: settings },
+      { provide: MailIngestIntakeService, useValue: intake },
+      { provide: MailIngestImapClient, useValue: imap },
+      { provide: CronRegistrarService, useValue: registrar },
+      { provide: getRepositoryToken(MailIngestCursor), useValue: cursor },
+      { provide: getRepositoryToken(MailIngestMessages), useValue: messages },
+      { provide: getRepositoryToken(MailIngestCandidates), useValue: candidates },
+    ],
+  }).compile();
+  return moduleRef;
+}
+
+let app: Awaited<ReturnType<typeof build>>;
+beforeEach(async () => {
+  vi.clearAllMocks();
+  settings.isAvailable.mockResolvedValue(true);
+  settings.webhookSecret.mockResolvedValue('a'.repeat(32));
+  settings.imapConfig.mockResolvedValue(IMAP);
+  cursor.acquireLease.mockResolvedValue(true);
+  cursor.find.mockResolvedValue(null);
+  relays = new MailIngestRelayHost();
+  app = await build();
+});
+
+describe('the signed webhook', () => {
+  const SECRET = 'a'.repeat(32);
+  const RAW = Buffer.from('Subject: Your booking\r\n\r\nABC123');
+  const NOW = Date.parse('2030-01-15T10:00:00Z');
+  const STAMP = String(NOW / 1000);
+  const service = () => app.get(MailIngestWebhookService);
+  const signed = (over: { timestamp?: string; recipient?: string; secret?: string } = {}) => {
+    const timestamp = over.timestamp ?? STAMP;
+    const recipient = over.recipient ?? 'trek+tok@example.com';
+    const signature = `sha256=${signWebhookMail(over.secret ?? SECRET, timestamp, recipient, RAW)}`;
+    return { timestamp, recipient, signature };
+  };
+
+  it('opens only while the import is on, the webhook is its transport and a secret is set', async () => {
+    settings.transport.mockResolvedValue('webhook');
+    expect(await service().secret()).toBe(SECRET);
+    settings.transport.mockResolvedValue('imap');
+    await expect(service().secret()).rejects.toThrow('Not found');
+    settings.transport.mockResolvedValue('webhook');
+    settings.webhookSecret.mockResolvedValue(null);
+    await expect(service().secret()).rejects.toThrow('Not found');
+    settings.webhookSecret.mockResolvedValue(SECRET);
+    settings.isAvailable.mockResolvedValue(false);
+    await expect(service().secret()).rejects.toThrow('Not found');
+  });
+
+  it('takes a mail whose signature covers it, its recipient and a fresh timestamp', async () => {
+    intake.receive.mockResolvedValue({ status: 'accepted', messageId: 1 });
+    expect(await service().receive(SECRET, RAW, signed(), NOW)).toEqual({ accepted: true, duplicate: false });
+    expect(intake.receive).toHaveBeenCalledWith(RAW, 'webhook', ['trek+tok@example.com']);
+    intake.receive.mockResolvedValue({ status: 'duplicate', messageId: 1 });
+    const noRecipient = signed({ recipient: '' });
+    expect(await service().receive(SECRET, RAW, noRecipient, NOW)).toEqual({ accepted: true, duplicate: true });
+    expect(intake.receive).toHaveBeenLastCalledWith(RAW, 'webhook', []);
+  });
+
+  it('refuses another secret, another recipient, a changed mail and a timestamp out of its window', async () => {
+    const refused = (parts: ReturnType<typeof signed>, raw = RAW) =>
+      expect(service().receive(SECRET, raw, parts, NOW)).rejects.toThrow('The signature does not match');
+    await refused(signed({ secret: 'b'.repeat(32) }));
+    await refused({ ...signed(), recipient: 'trek+other@example.com' });
+    await refused(signed(), Buffer.from('Subject: Your booking\r\n\r\nXYZ999'));
+    await refused(signed({ timestamp: String(NOW / 1000 - 301) }));
+    await refused({ ...signed(), signature: 'sha256=nothex' });
+    await refused({ ...signed(), timestamp: undefined as never });
+    expect(intake.receive).not.toHaveBeenCalled();
+  });
+
+  it('answers what the intake refused with its own status', async () => {
+    const cases: Array<[string, number]> = [
+      ['unknown_address', 422],
+      ['too_large', 413],
+      ['address_daily_limit', 429],
+      ['unavailable', 404],
+    ];
+    for (const [reason, status] of cases) {
+      intake.receive.mockResolvedValueOnce({ status: 'refused', reason });
+      const err = await service()
+        .receive(SECRET, RAW, signed(), NOW)
+        .catch((e: unknown) => e);
+      expect(err, reason).toBeInstanceOf(DomainError);
+      expect((err as DomainError).getStatus(), reason).toBe(status);
+    }
+  });
+});
+
+describe('the IMAP run', () => {
+  const job = () => app.get(MailIngestImapJob);
+
+  beforeEach(() => {
+    settings.transport.mockResolvedValue('imap');
+    session.open.mockResolvedValue({ uidValidity: 9 });
+    session.listAfter.mockResolvedValue([]);
+    session.source.mockImplementation(async (uid: number) => Buffer.from(`mail ${uid}`));
+    intake.receive.mockResolvedValue({ status: 'accepted', messageId: 1 });
+  });
+
+  it('takes the new mails after the cursor, moves them on, and notes where it got to', async () => {
+    cursor.find.mockResolvedValue({ uid_validity: 9, last_uid: 40 });
+    session.listAfter.mockResolvedValue([
+      { uid: 41, size: 10 },
+      { uid: 42, size: 10 },
+    ]);
+    expect(await job().tick()).toBe(2);
+    expect(session.listAfter).toHaveBeenCalledWith(40);
+    expect(intake.receive.mock.calls.map(([raw]) => String(raw))).toEqual(['mail 41', 'mail 42']);
+    expect(session.ensureFolder).toHaveBeenCalledWith(['TREK', 'processed']);
+    expect(session.move).toHaveBeenCalledWith([41, 42], 'TREK.processed');
+    expect(cursor.recordOk).toHaveBeenCalledWith('imap', 'INBOX', expect.any(String), { uidValidity: 9, lastUid: 42 });
+    expect(session.close).toHaveBeenCalled();
+    expect(cursor.releaseLease).toHaveBeenCalled();
+  });
+
+  it('starts over when the server renumbered the box, and leaves a mail over the limit unread', async () => {
+    cursor.find.mockResolvedValue({ uid_validity: 3, last_uid: 400 });
+    session.listAfter.mockResolvedValue([
+      { uid: 1, size: 5000 },
+      { uid: 2, size: 10 },
+    ]);
+    expect(await job().tick()).toBe(2);
+    expect(session.listAfter).toHaveBeenCalledWith(0);
+    expect(session.source).toHaveBeenCalledTimes(1);
+    expect(session.move).toHaveBeenCalledWith([1, 2], 'TREK.processed');
+  });
+
+  it('touches nothing with an empty box, and nothing at all unless IMAP is the transport', async () => {
+    expect(await job().tick()).toBe(0);
+    expect(session.ensureFolder).not.toHaveBeenCalled();
+    expect(cursor.recordOk).toHaveBeenCalledWith('imap', 'INBOX', expect.any(String), { uidValidity: 9, lastUid: 0 });
+    settings.transport.mockResolvedValue('webhook');
+    vi.clearAllMocks();
+    expect(await job().tick()).toBe(0);
+    expect(imap.connect).not.toHaveBeenCalled();
+    settings.transport.mockResolvedValue('imap');
+    cursor.acquireLease.mockResolvedValue(false);
+    expect(await job().tick()).toBe(0);
+    expect(imap.connect).not.toHaveBeenCalled();
+  });
+
+  it('records why a run failed for the status card, and keeps the cursor where it was', async () => {
+    imap.connect.mockRejectedValueOnce(Object.assign(new Error('Command failed'), { authenticationFailed: true }));
+    expect(await job().tick()).toBe(0);
+    const refused = 'The IMAP server refused the user name or password';
+    expect(cursor.recordError).toHaveBeenCalledWith('imap', 'INBOX', refused);
+    expect(cursor.recordOk).not.toHaveBeenCalled();
+    expect(cursor.releaseLease).toHaveBeenCalled();
+  });
+
+  it('describes a failure without anything of the login', () => {
+    expect(describeImapFailure(new SsrfBlockedError('Requests to loopback are not allowed'))).toBe(
+      'Requests to loopback are not allowed',
+    );
+    expect(describeImapFailure(Object.assign(new Error('connect'), { code: 'ECONNREFUSED' }))).toBe(
+      'Could not reach the IMAP server (ECONNREFUSED)',
+    );
+    expect(describeImapFailure(Object.assign(new Error('x'), { responseText: 'Mailbox does not exist' }))).toBe(
+      'The IMAP server answered: Mailbox does not exist',
+    );
+  });
+});
+
+describe('the TREK Cloud relay', () => {
+  const job = () => app.get(MailIngestRelayJob);
+  const relay = { list: vi.fn(), ack: vi.fn(async () => undefined) };
+  const mail = (id: string, body: string, sha = createHash('sha256').update(body).digest('hex')) => ({
+    id,
+    recipients: ['ab12tok@in.example.com'],
+    sha256: sha,
+    raw: Buffer.from(body).toString('base64'),
+  });
+
+  beforeEach(() => {
+    settings.transport.mockResolvedValue('cloud');
+    relays.attach(relay);
+  });
+
+  it('stays idle without a relay attached', async () => {
+    relays = new MailIngestRelayHost();
+    app = await build();
+    expect(await job().tick()).toBe(0);
+    expect(cursor.acquireLease).not.toHaveBeenCalled();
+  });
+
+  it('takes each whole mail in under the address it was sent to and acknowledges it', async () => {
+    const broken = mail('m2', 'two', 'f'.repeat(64));
+    relay.list.mockResolvedValue({ mails: [mail('m1', 'one'), broken, mail('m3', 'three')] });
+    intake.receive
+      .mockResolvedValueOnce({ status: 'accepted', messageId: 1 })
+      .mockResolvedValueOnce({ status: 'refused', reason: 'unknown_address' });
+    expect(await job().tick()).toBe(1);
+    expect(relay.list).toHaveBeenCalledWith(10);
+    expect(intake.receive).toHaveBeenCalledWith(Buffer.from('one'), 'cloud', ['ab12tok@in.example.com']);
+    // m2 did not arrive whole and stays with the relay; m3 is refused for good and dropped.
+    expect(relay.ack.mock.calls).toEqual([['m1'], ['m3']]);
+    expect(cursor.recordOk).toHaveBeenCalledWith('cloud', '', expect.any(String));
+  });
+
+  it('keeps a mail the import cannot take right now, and refuses an inbox it cannot read', async () => {
+    relay.list.mockResolvedValue({ mails: [mail('m1', 'one')] });
+    intake.receive.mockResolvedValue({ status: 'refused', reason: 'unavailable' });
+    await job().tick();
+    expect(relay.ack).not.toHaveBeenCalled();
+
+    relay.list.mockResolvedValue({ mails: [{ id: '../../etc', raw: 'x' }] });
+    expect(await job().tick()).toBe(0);
+    const unreadable = 'The relay answered with an inbox this server cannot read';
+    expect(cursor.recordError).toHaveBeenCalledWith('cloud', '', unreadable);
+  });
+});
+
+describe('the status card and the connection test', () => {
+  const status = () => app.get(MailIngestStatusService);
+
+  it('counts the last day by status and reason, and names no mail and no person', async () => {
+    settings.transport.mockResolvedValue('cloud');
+    cursor.find.mockResolvedValue({ last_ok_at: '2030-01-15 09:58:00', last_error: null, lease_until: null });
+    messages.tallySince.mockResolvedValue([
+      { status: 'processed', reason: null, n: 4 },
+      { status: 'discarded', reason: 'newsletter', n: 2 },
+      { status: 'processed', reason: 'no_trip_in_range', n: 1 },
+    ]);
+    messages.countAllAiReadSince.mockResolvedValue(3);
+    candidates.tallyResolutionsSince.mockResolvedValue([
+      { resolution: 'auto_applied', n: 5 },
+      { resolution: 'undone', n: 1 },
+    ]);
+    candidates.pendingPerUser.mockResolvedValue([4, 1]);
+
+    expect(await status().status(new Date('2030-01-15T10:00:00Z'))).toEqual({
+      transport: 'cloud',
+      last_ok_at: '2030-01-15T09:58:00Z',
+      last_error: null,
+      lease_until: null,
+      last_day: {
+        by_status: { processed: 5, discarded: 2 },
+        by_reason: { newsletter: 2, no_trip_in_range: 1 },
+      },
+      auto_applied: 5,
+      undone: 1,
+      ai_calls_today: 3,
+      backlog: [4, 1],
+    });
+    expect(cursor.find).toHaveBeenCalledWith('cloud', '');
+    expect(messages.countAllAiReadSince).toHaveBeenCalledWith('2030-01-15 00:00:00');
+  });
+
+  it('tests each transport the way the import reaches it', async () => {
+    settings.transport.mockResolvedValue('off');
+    expect(await status().test()).toMatchObject({ ok: false, error: expect.stringContaining('upload') });
+    settings.transport.mockResolvedValue('webhook');
+    expect(await status().test()).toEqual({ transport: 'webhook', ok: true, error: null });
+    settings.transport.mockResolvedValue('cloud');
+    expect(await status().test()).toMatchObject({ ok: false, error: 'The TREK Cloud relay is not connected' });
+    settings.transport.mockResolvedValue('imap');
+    session.open.mockResolvedValue({ uidValidity: 1 });
+    expect(await status().test()).toEqual({ transport: 'imap', ok: true, error: null });
+    expect(session.open).toHaveBeenCalledWith('INBOX');
+    expect(session.close).toHaveBeenCalled();
+    imap.connect.mockRejectedValueOnce(Object.assign(new Error('x'), { code: 'ETIMEDOUT' }));
+    expect(await status().test()).toMatchObject({ ok: false, error: 'Could not reach the IMAP server (ETIMEDOUT)' });
+    settings.imapConfig.mockResolvedValue(null);
+    expect(await status().test()).toMatchObject({ ok: false, error: 'The IMAP host, user or password is missing' });
+  });
+});