feat(mail-ingest): read forwarded mails from an IMAP box, a signed webhook and the TREK Cloud relay

The IMAP job reads the box every two minutes through the SSRF guard, keeps a
uid cursor and moves read mails to TREK/processed. The webhook takes raw MIME
signed with HMAC-SHA256 over a timestamp, the recipient and the mail, and is a
404 unless it is the transport. The relay job collects a managed install's mails.
This commit is contained in:
Maurice
2026-11-13 16:42:43 +01:00
parent 6309064f40
commit 1bfaf4ea98
15 changed files with 1256 additions and 7 deletions
@@ -88,7 +88,7 @@ interface MailIngestTripPlaceRow {
}
/** A trip as the inbox names it to a person who can see it. */
export interface MailIngestTripRefRow {
interface MailIngestTripRefRow {
id: number;
title: string;
start_date: string | null;
@@ -132,6 +132,10 @@ export const PUBLIC_ROUTE_ALLOW_LIST: string[] = [
'JourneyPublicController.legacyPhoto',
'JourneyPublicController.mapLayers',
'JourneyPublicController.photo',
// The booking mail webhook. A mail service holds no session; the HMAC over
// the raw mail is the credential, and the route is a 404 unless the admin
// made the webhook the import's transport and set its secret.
'MailIngestInboundController.receive',
// The MCP transport — bearer tokens are verified inside the handler.
// The same documents under the resource URL, for clients that append the
// well-known path to the server address they were handed.
@@ -0,0 +1,126 @@
import { checkSsrf, SsrfBlockedError } from '../../utils/ssrfGuard';
import type { MailIngestImapConfig } from './mail-ingest-settings.service';
import { Injectable } from '@nestjs/common';
import { ImapFlow } from 'imapflow';
import { isIP } from 'node:net';
/** How long connecting and the server's greeting may take, and how long the socket may idle after. */
const CONNECT_TIMEOUT_MS = 15_000;
const SOCKET_TIMEOUT_MS = 60_000;
/** A mail waiting in the box: its uid and its size, before anything of it is downloaded. */
export interface ImapMailRef {
uid: number;
size: number;
}
/** What the import asks of an IMAP box, one connection per run. */
export interface ImapSession {
/** Opens the folder; answers the uid validity, which changes when the server renumbers it. */
open(mailbox: string): Promise<{ uidValidity: number }>;
/** The mails with a uid above `lastUid`, oldest first. */
listAfter(lastUid: number): Promise<ImapMailRef[]>;
/** One mail's raw bytes; null for a uid that is gone. */
source(uid: number): Promise<Buffer | null>;
/** Makes the folder (made of these parts, in the server's own delimiter) unless it exists; answers its path. */
ensureFolder(parts: readonly string[]): Promise<string>;
move(uids: readonly number[], folder: string): Promise<void>;
close(): Promise<void>;
}
/**
* Opens the admin's IMAP box. The host goes through the SSRF guard first,
* with the same allowance for a box on the own network as every other
* address an admin configures (ALLOW_INTERNAL_NETWORK), and the connection
* goes to the address the guard vetted, the certificate still checked
* against the host's name, so a name that resolves differently a moment
* later cannot redirect it.
*/
@Injectable()
export class MailIngestImapClient {
async connect(config: MailIngestImapConfig): Promise<ImapSession> {
const address = await vettedAddress(config.host);
const client = new ImapFlow({
host: address,
port: config.port,
secure: config.tls === 'tls',
doSTARTTLS: config.tls === 'starttls' ? true : undefined,
servername: isIP(config.host) ? undefined : config.host,
auth: { user: config.user, pass: config.pass },
logger: false,
disableAutoIdle: true,
connectionTimeout: CONNECT_TIMEOUT_MS,
greetingTimeout: CONNECT_TIMEOUT_MS,
socketTimeout: SOCKET_TIMEOUT_MS,
});
await client.connect();
return new ImapFlowSession(client);
}
}
/** The address a connection to `host` may use, or a refusal naming why. */
async function vettedAddress(host: string): Promise<string> {
const verdict = await checkSsrf(`https://${isIP(host) === 6 ? `[${host}]` : host}/`);
if (!verdict.allowed || !verdict.resolvedIp) {
throw new SsrfBlockedError(verdict.error ?? 'This IMAP host is not allowed');
}
return verdict.resolvedIp;
}
class ImapFlowSession implements ImapSession {
constructor(private readonly client: ImapFlow) {}
async open(mailbox: string): Promise<{ uidValidity: number }> {
const box = await this.client.mailboxOpen(mailbox);
return { uidValidity: Number(box.uidValidity) };
}
async listAfter(lastUid: number): Promise<ImapMailRef[]> {
const found: ImapMailRef[] = [];
// `n:*` always names the newest mail, even below n, so the range alone does not filter.
for await (const message of this.client.fetch(`${lastUid + 1}:*`, { uid: true, size: true }, { uid: true })) {
if (message.uid > lastUid) found.push({ uid: message.uid, size: message.size ?? 0 });
}
return found.sort((a, b) => a.uid - b.uid);
}
async source(uid: number): Promise<Buffer | null> {
const message = await this.client.fetchOne(String(uid), { source: true }, { uid: true });
return message && Buffer.isBuffer(message.source) ? message.source : null;
}
async ensureFolder(parts: readonly string[]): Promise<string> {
const created = await this.client.mailboxCreate([...parts]);
return created.path;
}
async move(uids: readonly number[], folder: string): Promise<void> {
if (uids.length === 0) return;
await this.client.messageMove(uids.join(','), folder, { uid: true });
}
async close(): Promise<void> {
try {
await this.client.logout();
} catch {
// The connection broke on its own; there is nothing left to say goodbye to.
this.client.close();
}
}
}
/** What an admin reads when the box cannot be read: the reason, never the password. */
export function describeImapFailure(err: unknown): string {
if (err instanceof SsrfBlockedError) return err.message;
const failure = (err ?? {}) as {
authenticationFailed?: boolean;
code?: string;
responseText?: string;
message?: string;
};
if (failure.authenticationFailed) return 'The IMAP server refused the user name or password';
if (failure.code && /^E[A-Z]+$/.test(failure.code)) return `Could not reach the IMAP server (${failure.code})`;
const said = failure.responseText || failure.message || String(err);
return `The IMAP server answered: ${said}`.slice(0, 300);
}
@@ -0,0 +1,106 @@
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
import { dbNow } from '../../db/types';
import { logError, logWarn } from '../audit/audit-log.logger';
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
import {
describeImapFailure,
MailIngestImapClient,
type ImapMailRef,
type ImapSession,
} from './mail-ingest-imap.client';
import { MailIngestIntakeService } from './mail-ingest-intake.service';
import { MailIngestSettingsService, type MailIngestImapConfig } from './mail-ingest-settings.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
/** How long a run holds the box before another process may take it over. */
const LEASE_MS = 10 * 60 * 1000;
/** The mails one run takes from the box; the rest wait for the next run. */
const MAILS_PER_RUN = 20;
/** Where read mails go: kept, never deleted, out of the folder the next run reads. */
const PROCESSED_FOLDER = ['TREK', 'processed'] as const;
/**
* Reads the admin's IMAP box every two minutes, with a client of its own for
* every run. Each new mail is handed to the intake, which stores it once,
* and then moved to TREK/processed. The cursor (uid validity, last uid)
* keeps a mail the move missed from being read twice, and the intake's
* dedupe keeps it so when the server renumbers the box and the run starts
* over. One process holds the box at a time, by the cursor's lease.
*/
@Injectable()
export class MailIngestImapJob implements OnApplicationBootstrap {
constructor(
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
private readonly settings: MailIngestSettingsService,
private readonly intake: MailIngestIntakeService,
private readonly imap: MailIngestImapClient,
private readonly registrar: CronRegistrarService,
) {}
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
this.registrar.register('mail-ingest-imap', '*/2 * * * *', () => this.tick());
}
/**
* One run; answers how many mails it took from the box. Nothing happens
* unless the import is on, IMAP is its transport and the box is complete.
*
* @txIndependent each mail is stored on its own as it is read; the cursor
* notes where the run got to once the box is done with.
*/
async tick(now: Date = new Date()): Promise<number> {
if (!(await this.settings.isAvailable()) || (await this.settings.transport()) !== 'imap') return 0;
const config = await this.settings.imapConfig();
if (!config) return 0;
const until = dbNow(new Date(now.getTime() + LEASE_MS));
if (!(await this.cursor.acquireLease('imap', config.mailbox, LEASE_OWNER, until, dbNow(now)))) return 0;
try {
const { taken, uidValidity, lastUid } = await this.poll(config);
await this.cursor.recordOk('imap', config.mailbox, dbNow(new Date()), { uidValidity, lastUid });
return taken;
} catch (err) {
const reason = describeImapFailure(err);
logError(`[mail-ingest] Reading the IMAP box failed: ${reason}`);
await this.cursor.recordError('imap', config.mailbox, reason);
return 0;
} finally {
await this.cursor.releaseLease('imap', config.mailbox, LEASE_OWNER);
}
}
private async poll(config: MailIngestImapConfig): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
const session = await this.imap.connect(config);
try {
const { uidValidity } = await session.open(config.mailbox);
const position = await this.cursor.find('imap', config.mailbox);
// A changed uid validity means the server renumbered the box: start over, the dedupe keeps each mail once.
const after = position?.uid_validity === uidValidity ? (position.last_uid ?? 0) : 0;
const due = (await session.listAfter(after)).slice(0, MAILS_PER_RUN);
if (due.length === 0) return { taken: 0, uidValidity, lastUid: after };
const folder = await session.ensureFolder(PROCESSED_FOLDER);
await due.reduce(async (previous, mail) => {
await previous;
await this.takeIn(session, mail);
}, Promise.resolve());
await session.move(due.map((mail) => mail.uid), folder);
return { taken: due.length, uidValidity, lastUid: Math.max(...due.map((mail) => mail.uid)) };
} finally {
await session.close();
}
}
/** One mail into the intake. One larger than the import takes is left unread, and moved on with the rest. */
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<void> {
if (mail.size > this.settings.maxMailBytes()) {
logWarn(`[mail-ingest] An IMAP mail of ${mail.size} bytes is larger than the import takes; left unread`);
return;
}
const raw = await session.source(mail.uid);
if (!raw) return;
const result = await this.intake.receive(raw, 'imap');
if (result.status === 'refused') logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
}
}
@@ -0,0 +1,65 @@
import { Public } from '../auth-core/public.decorator';
import { DomainError } from '../common/domain-error';
import { ResponseContract } from '../common/response-contract';
import {
MailIngestWebhookService,
WEBHOOK_RECIPIENT_HEADER,
WEBHOOK_SIGNATURE_HEADER,
WEBHOOK_TIMESTAMP_HEADER,
} from './mail-ingest-webhook.service';
import { Controller, HttpCode, Post, Req } from '@nestjs/common';
import { mailIngestInboundResponseSchema, type MailIngestInboundResponse } from '@trek/shared';
import type { Request } from 'express';
const TOO_LARGE = 'This mail is larger than the import takes';
/**
* `POST /api/mail-ingest/inbound`: the signed webhook a mail service posts
* every mail to, raw (message/rfc822). It does not exist (404) unless the
* import is on, the webhook is its transport and a secret is set; then the
* body is read up to the import's limit, and the signature over it decides.
* The JSON parsers leave a message/rfc822 body alone, so this route reads
* the stream itself and nothing of the mail reaches the request log.
*/
@Controller('api/mail-ingest/inbound')
export class MailIngestInboundController {
constructor(private readonly webhook: MailIngestWebhookService) {}
@Post()
@Public(
'A mail service cannot hold a TREK session; an HMAC signature over the raw mail, its recipient and a timestamp within five minutes is the credential, and the route answers 404 unless the webhook is the configured transport.',
)
@HttpCode(200)
@ResponseContract(mailIngestInboundResponseSchema)
async receive(@Req() req: Request): Promise<MailIngestInboundResponse> {
const secret = await this.webhook.secret();
if (!req.is('message/rfc822')) {
throw new DomainError(415, 'Send the raw mail as message/rfc822');
}
const raw = await readBody(req, this.webhook.maxBytes());
return this.webhook.receive(secret, raw, {
timestamp: req.get(WEBHOOK_TIMESTAMP_HEADER),
signature: req.get(WEBHOOK_SIGNATURE_HEADER),
recipient: req.get(WEBHOOK_RECIPIENT_HEADER),
});
}
}
/**
* The request body, up to `maxBytes`. Past the limit the rest is drained
* unread, so the sender gets the 413 rather than a reset connection.
*/
function readBody(req: Request, maxBytes: number): Promise<Buffer> {
if (Number(req.get('content-length')) > maxBytes) return Promise.reject(new DomainError(413, TOO_LARGE));
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let size = 0;
req.on('data', (chunk: Buffer) => {
size += chunk.length;
if (size <= maxBytes) chunks.push(chunk);
});
req.on('end', () => (size > maxBytes ? reject(new DomainError(413, TOO_LARGE)) : resolve(Buffer.concat(chunks))));
req.on('error', reject);
});
}
@@ -0,0 +1,108 @@
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
import { dbNow } from '../../db/types';
import { logError, logWarn } from '../audit/audit-log.logger';
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
import { MailIngestIntakeService } from './mail-ingest-intake.service';
import { MailIngestRelayHost, type MailInboxRelay } from './mail-ingest-relay.service';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
import { createHash } from 'node:crypto';
import { z } from 'zod';
/** How long a run holds the relay before another process may take it over. */
const LEASE_MS = 10 * 60 * 1000;
/** The mails one run asks the relay for. */
const PAGE = 10;
/** The base64 length of the largest mail the relay hands on. */
const MAX_RAW_CHARS = Math.ceil(MAIL_INGEST_LIMITS.cloudMailBytes / 3) * 4;
/** The relay's answer, checked before anything of it is used. */
const relayInboxSchema = z.object({
mails: z
.array(
z.object({
id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/),
/** The addresses it was sent to, as the relay's mail server took them (RCPT TO). */
recipients: z.array(z.string().min(3).max(320)).min(1).max(5),
sha256: z.string().regex(/^[0-9a-f]{64}$/),
/** The mail itself, base64. */
raw: z.string().max(MAX_RAW_CHARS),
}),
)
.max(PAGE),
});
type RelayedMail = z.infer<typeof relayInboxSchema>['mails'][number];
/**
* Collects this instance's forwarded mails from the TREK Cloud relay every
* two minutes: a page of at most ten, each checked against its hash, handed
* to the intake under the address it was sent to, and acknowledged once it
* is in (or once it is clear it never will be), so the relay drops it. A
* mail that does not arrive whole is left for the next run. Idle wherever
* no relay is attached, which is every self-hosted install.
*/
@Injectable()
export class MailIngestRelayJob implements OnApplicationBootstrap {
constructor(
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
private readonly settings: MailIngestSettingsService,
private readonly intake: MailIngestIntakeService,
private readonly relays: MailIngestRelayHost,
private readonly registrar: CronRegistrarService,
) {}
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
this.registrar.register('mail-ingest-relay', '*/2 * * * *', () => this.tick());
}
/**
* One run; answers how many mails it took in.
*
* @txIndependent each mail is stored on its own and acknowledged after it, network I/O between them.
*/
async tick(now: Date = new Date()): Promise<number> {
const relay = this.relays.current();
if (!relay || !(await this.settings.isAvailable()) || (await this.settings.transport()) !== 'cloud') return 0;
const until = dbNow(new Date(now.getTime() + LEASE_MS));
if (!(await this.cursor.acquireLease('cloud', '', LEASE_OWNER, until, dbNow(now)))) return 0;
try {
const page = relayInboxSchema.safeParse(await relay.list(PAGE));
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
const taken = await page.data.mails.reduce(async (previous, mail) => {
const count = await previous;
return count + ((await this.takeIn(relay, mail)) ? 1 : 0);
}, Promise.resolve(0));
await this.cursor.recordOk('cloud', '', dbNow(new Date()));
return taken;
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
logError(`[mail-ingest] Collecting mails from the relay failed: ${reason}`);
await this.cursor.recordError('cloud', '', reason);
return 0;
} finally {
await this.cursor.releaseLease('cloud', '', LEASE_OWNER);
}
}
/** One relayed mail into the intake; answers whether it was taken in. */
private async takeIn(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
const raw = Buffer.from(mail.raw, 'base64');
if (createHash('sha256').update(raw).digest('hex') !== mail.sha256) {
logWarn(`[mail-ingest] Relayed mail ${mail.id} did not arrive whole; left for the next run`);
return false;
}
const result = await this.intake.receive(raw, 'cloud', mail.recipients);
// Switched off in between: the relay keeps the mail until the import is back or the mail expires.
if (result.status === 'refused' && result.reason === 'unavailable') return false;
if (result.status === 'refused') {
logWarn(`[mail-ingest] Relayed mail ${mail.id} was not taken in: ${result.reason}`);
}
await relay.ack(mail.id);
return result.status === 'accepted';
}
}
@@ -0,0 +1,33 @@
import { Injectable } from '@nestjs/common';
/**
* The TREK Cloud control plane's queue of mails for this instance, as the
* managed install reaches it with its instance token. The control plane's
* mail server takes a forwarded mail for a contract's route key, keeps it
* encrypted, and hands it over here; nothing of a person reaches it.
*/
export interface MailInboxRelay {
/** `GET /v1/instance/inbox?limit=n`: the waiting mails, the answer's JSON as parsed, checked by the caller. */
list(limit: number): Promise<unknown>;
/** `POST /v1/instance/inbox/:id/ack`: the mail is taken in (or never will be), and the relay drops it. */
ack(id: string): Promise<void>;
}
/**
* Where the managed install plugs its control-plane client in. This
* repository has none: a self-hosted install has no relay, and the pull job
* stays idle until a module of the managed build attaches one when it
* starts, through the client it already talks to the control plane with.
*/
@Injectable()
export class MailIngestRelayHost {
private relay: MailInboxRelay | null = null;
attach(relay: MailInboxRelay): void {
this.relay = relay;
}
current(): MailInboxRelay | null {
return this.relay;
}
}
@@ -79,7 +79,9 @@ export class MailIngestSettingsService {
const user = await this.pick(this.config.imapUser, 'mail_ingest_imap_user');
const pass = this.config.imapPass ?? decrypt_api_key(await this.settings.getAppSetting('mail_ingest_imap_pass'));
if (!host || !user || !pass) return null;
const tls = mailIngestImapTlsSchema.catch('tls').parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls'));
const tls = mailIngestImapTlsSchema
.catch('tls')
.parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls'));
const port = Number(await this.pick(this.config.imapPort, 'mail_ingest_imap_port'));
const mailbox = await this.pick(this.config.imapMailbox, 'mail_ingest_imap_mailbox');
return {
@@ -0,0 +1,115 @@
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository';
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
import { dbNow, utcSuffix } from '../../db/types';
import { describeImapFailure, MailIngestImapClient } from './mail-ingest-imap.client';
import { MailIngestRelayHost } from './mail-ingest-relay.service';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
import {
mailIngestMessageStatusSchema,
mailIngestReasonSchema,
type MailIngestAdminStatusResponse,
type MailIngestAdminTestResponse,
type MailIngestTransportSetting,
} from '@trek/shared';
const DAY_MS = 24 * 60 * 60 * 1000;
/**
* The admin's view of the import: how its transport runs and what became of
* the mails of the last day, in counts only. No subject, sender, address or
* content of any mail is part of it, and the backlog is a list of numbers,
* not of people. The connection test reaches the transport the way the
* import does and says what failed.
*/
@Injectable()
export class MailIngestStatusService {
constructor(
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
private readonly settings: MailIngestSettingsService,
private readonly imap: MailIngestImapClient,
private readonly relays: MailIngestRelayHost,
) {}
async status(now: Date = new Date()): Promise<MailIngestAdminStatusResponse> {
const transport = await this.settings.transport();
const reader = await this.readerCursor(transport);
const since = dbNow(new Date(now.getTime() - DAY_MS));
const byStatus: MailIngestAdminStatusResponse['last_day']['by_status'] = {};
const byReason: MailIngestAdminStatusResponse['last_day']['by_reason'] = {};
for (const { status, reason, n } of await this.messages.tallySince(since)) {
const knownStatus = mailIngestMessageStatusSchema.safeParse(status);
if (knownStatus.success) byStatus[knownStatus.data] = (byStatus[knownStatus.data] ?? 0) + n;
const knownReason = mailIngestReasonSchema.safeParse(reason);
if (knownReason.success) byReason[knownReason.data] = (byReason[knownReason.data] ?? 0) + n;
}
const resolutions = new Map(
(await this.candidates.tallyResolutionsSince(since)).map(({ resolution, n }) => [resolution, n]),
);
const startOfDay = dbNow(new Date(`${now.toISOString().slice(0, 10)}T00:00:00Z`));
return {
transport,
last_ok_at: utcSuffix(reader?.last_ok_at),
last_error: reader?.last_error ?? null,
lease_until: utcSuffix(reader?.lease_until),
last_day: { by_status: byStatus, by_reason: byReason },
auto_applied: resolutions.get('auto_applied') ?? 0,
undone: resolutions.get('undone') ?? 0,
ai_calls_today: await this.messages.countAllAiReadSince(startOfDay),
backlog: await this.candidates.pendingPerUser(),
};
}
/** Whether the configured transport can reach its mail now, and if not, why. */
async test(): Promise<MailIngestAdminTestResponse> {
const transport = await this.settings.transport();
const error = await this.problemWith(transport);
return { transport, ok: error === null, error };
}
private async problemWith(transport: MailIngestTransportSetting): Promise<string | null> {
switch (transport) {
case 'off':
return 'No transport is set; mails come in by upload only';
case 'webhook':
return (await this.settings.webhookSecret()) ? null : 'No webhook secret is set';
case 'cloud':
return this.relays.current() ? null : 'The TREK Cloud relay is not connected';
case 'imap':
return this.imapProblem();
}
}
/** Opens the box and its folder the way a run does, and lets go again. */
private async imapProblem(): Promise<string | null> {
const config = await this.settings.imapConfig();
if (!config) return 'The IMAP host, user or password is missing';
try {
const session = await this.imap.connect(config);
try {
await session.open(config.mailbox);
} finally {
await session.close();
}
return null;
} catch (err) {
return describeImapFailure(err);
}
}
/** Where the reader of the transport stands: the IMAP box's or the relay's cursor, else the pipeline's own. */
private async readerCursor(transport: MailIngestTransportSetting) {
if (transport === 'imap') {
const config = await this.settings.imapConfig();
return config ? this.cursor.find('imap', config.mailbox) : null;
}
return this.cursor.find(transport === 'cloud' ? 'cloud' : 'pipeline', '');
}
}
@@ -0,0 +1,107 @@
import { DomainError } from '../common/domain-error';
import { MailIngestIntakeService, type MailIntakeRefusal } from './mail-ingest-intake.service';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { Injectable } from '@nestjs/common';
import type { MailIngestInboundResponse } from '@trek/shared';
import { createHmac, timingSafeEqual } from 'node:crypto';
/** The headers a webhook sender signs a mail with, and the address it was sent to (the RCPT TO). */
export const WEBHOOK_TIMESTAMP_HEADER = 'x-trek-timestamp';
export const WEBHOOK_SIGNATURE_HEADER = 'x-trek-signature';
export const WEBHOOK_RECIPIENT_HEADER = 'x-trek-recipient';
/** How far a signed timestamp may lie from the server's clock, either way. */
const WINDOW_MS = 5 * 60 * 1000;
/** How a mail the intake refused answers the sender, by why. */
const REFUSALS: Record<MailIntakeRefusal, [status: number, error: string]> = {
unavailable: [404, 'Not found'],
too_large: [413, 'This mail is larger than the import takes'],
not_a_mail: [422, 'This is not a mail'],
unknown_address: [422, 'No address of this server takes this mail'],
address_daily_limit: [429, 'This address took all the mails it takes today'],
inbox_full: [429, 'The inbox of this address is full'],
};
/** The signed parts of a webhook request, as the sender put them in its headers. */
interface WebhookSignedParts {
timestamp?: string;
signature?: string;
recipient?: string;
}
/**
* The signed webhook: a mail service (a Cloudflare Email Worker, Postmark,
* Mailgun) posts each mail to the install raw, signed with the admin's
* secret. The signature covers the time it was sent, the address it was
* sent to and every byte of the mail, so neither can be changed on the way,
* and a request older than five minutes is refused. A request replayed
* within that window stores nothing new: the intake keeps one copy of a mail
* per address.
*/
@Injectable()
export class MailIngestWebhookService {
constructor(
private readonly settings: MailIngestSettingsService,
private readonly intake: MailIngestIntakeService,
) {}
/**
* The secret mails are checked against. While the import is off, the
* webhook is not its transport or no secret is set, the route answers as
* if it did not exist.
*/
async secret(): Promise<string> {
const open = (await this.settings.isAvailable()) && (await this.settings.transport()) === 'webhook';
const secret = open ? await this.settings.webhookSecret() : null;
if (!secret) throw new DomainError(404, 'Not found');
return secret;
}
maxBytes(): number {
return this.settings.maxMailBytes();
}
async receive(
secret: string,
raw: Buffer,
signed: WebhookSignedParts,
now: number = Date.now(),
): Promise<MailIngestInboundResponse> {
const recipient = signed.recipient?.trim() ?? '';
if (!verifyWebhook(secret, signed, recipient, raw, now)) throw new DomainError(401, 'The signature does not match');
const result = await this.intake.receive(raw, 'webhook', recipient ? [recipient] : []);
if (result.status === 'refused') {
const [status, error] = REFUSALS[result.reason];
throw new DomainError(status, error);
}
return { accepted: true, duplicate: result.status === 'duplicate' };
}
}
/**
* The signature a sender puts on a mail: hex HMAC-SHA256, keyed with the
* secret, over the timestamp (Unix seconds), a newline, the recipient
* address (empty when the sender names none), a newline, and the raw mail.
*/
export function signWebhookMail(secret: string, timestamp: string, recipient: string, raw: Buffer): string {
return createHmac('sha256', secret).update(`${timestamp}\n${recipient}\n`).update(raw).digest('hex');
}
function verifyWebhook(
secret: string,
signed: WebhookSignedParts,
recipient: string,
raw: Buffer,
now: number,
): boolean {
const { timestamp, signature } = signed;
if (!timestamp || !/^\d{1,12}$/.test(timestamp) || Math.abs(now - Number(timestamp) * 1000) > WINDOW_MS) {
return false;
}
const given = (signature ?? '').trim().replace(/^sha256=/i, '').toLowerCase();
if (!/^[0-9a-f]{64}$/.test(given) || recipient.length > 320) return false;
const expected = signWebhookMail(secret, timestamp, recipient, raw);
return timingSafeEqual(Buffer.from(given, 'hex'), Buffer.from(expected, 'hex'));
}
@@ -21,6 +21,9 @@ import { SettingsModule } from '../settings/settings.module';
import { MailIngestActionsService } from './mail-ingest-actions.service';
import { MailIngestAddressService } from './mail-ingest-address.service';
import { MailIngestApplyService } from './mail-ingest-apply.service';
import { MailIngestImapClient } from './mail-ingest-imap.client';
import { MailIngestImapJob } from './mail-ingest-imap.job';
import { MailIngestInboundController } from './mail-ingest-inbound.controller';
import { MailIngestInboxService } from './mail-ingest-inbox.service';
import { MailIngestIntakeService } from './mail-ingest-intake.service';
import { MailIngestNotificationActions } from './mail-ingest-notification-actions';
@@ -28,19 +31,25 @@ import { MailIngestNotifyService } from './mail-ingest-notify.service';
import { MailIngestProcessJob } from './mail-ingest-process.job';
import { MailIngestProcessService } from './mail-ingest-process.service';
import { MailIngestReadingService } from './mail-ingest-reading.service';
import { MailIngestRelayJob } from './mail-ingest-relay.job';
import { MailIngestRelayHost } from './mail-ingest-relay.service';
import { MailIngestRetentionJob } from './mail-ingest-retention.job';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { MailIngestStatusService } from './mail-ingest-status.service';
import { MailIngestTripsService } from './mail-ingest-trips.service';
import { MailIngestWebhookService } from './mail-ingest-webhook.service';
import { MikroOrmModule } from '@mikro-orm/nestjs';
import { Module } from '@nestjs/common';
/**
* The booking mail import: a person forwards a booking mail to an address of
* their own, and the booking lands in the matching trip, or waits in their
* inbox with a reason. Transports (IMAP, the signed webhook, the TREK Cloud
* relay, an .eml upload) only hand mail to MailIngestIntakeService; the
* processing job reads it; everything a person does with an entry goes
* through MailIngestActionsService.
* inbox with a reason. Transports (the IMAP job, the signed webhook, the
* TREK Cloud relay job, an .eml upload) only hand mail to
* MailIngestIntakeService; the processing job reads it; everything a person
* does with an entry goes through MailIngestActionsService, from REST and
* MCP alike. MailIngestRelayHost is exported for the managed build, which
* attaches its control-plane client to it.
*/
@Module({
imports: [
@@ -67,6 +76,7 @@ import { Module } from '@nestjs/common';
MailIngestCandidates,
]),
],
controllers: [MailIngestInboundController],
providers: [
MailIngestSettingsService,
MailIngestTripsService,
@@ -78,10 +88,22 @@ import { Module } from '@nestjs/common';
MailIngestProcessService,
MailIngestActionsService,
MailIngestInboxService,
MailIngestStatusService,
MailIngestWebhookService,
MailIngestImapClient,
MailIngestRelayHost,
MailIngestNotificationActions,
MailIngestProcessJob,
MailIngestRetentionJob,
MailIngestImapJob,
MailIngestRelayJob,
],
exports: [
MailIngestSettingsService,
MailIngestAddressService,
MailIngestIntakeService,
MailIngestActionsService,
MailIngestRelayHost,
],
exports: [MailIngestSettingsService, MailIngestAddressService, MailIngestIntakeService, MailIngestActionsService],
})
export class MailIngestModule {}