mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-14 00:01:03 +00:00
feat(mail-ingest): read forwarded mails from an IMAP box, a signed webhook and the TREK Cloud relay
The IMAP job reads the box every two minutes through the SSRF guard, keeps a uid cursor and moves read mails to TREK/processed. The webhook takes raw MIME signed with HMAC-SHA256 over a timestamp, the recipient and the mail, and is a 404 unless it is the transport. The relay job collects a managed install's mails.
This commit is contained in:
@@ -88,7 +88,7 @@ interface MailIngestTripPlaceRow {
|
||||
}
|
||||
|
||||
/** A trip as the inbox names it to a person who can see it. */
|
||||
export interface MailIngestTripRefRow {
|
||||
interface MailIngestTripRefRow {
|
||||
id: number;
|
||||
title: string;
|
||||
start_date: string | null;
|
||||
|
||||
@@ -132,6 +132,10 @@ export const PUBLIC_ROUTE_ALLOW_LIST: string[] = [
|
||||
'JourneyPublicController.legacyPhoto',
|
||||
'JourneyPublicController.mapLayers',
|
||||
'JourneyPublicController.photo',
|
||||
// The booking mail webhook. A mail service holds no session; the HMAC over
|
||||
// the raw mail is the credential, and the route is a 404 unless the admin
|
||||
// made the webhook the import's transport and set its secret.
|
||||
'MailIngestInboundController.receive',
|
||||
// The MCP transport — bearer tokens are verified inside the handler.
|
||||
// The same documents under the resource URL, for clients that append the
|
||||
// well-known path to the server address they were handed.
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import { checkSsrf, SsrfBlockedError } from '../../utils/ssrfGuard';
|
||||
import type { MailIngestImapConfig } from './mail-ingest-settings.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { ImapFlow } from 'imapflow';
|
||||
import { isIP } from 'node:net';
|
||||
|
||||
/** How long connecting and the server's greeting may take, and how long the socket may idle after. */
|
||||
const CONNECT_TIMEOUT_MS = 15_000;
|
||||
const SOCKET_TIMEOUT_MS = 60_000;
|
||||
|
||||
/** A mail waiting in the box: its uid and its size, before anything of it is downloaded. */
|
||||
export interface ImapMailRef {
|
||||
uid: number;
|
||||
size: number;
|
||||
}
|
||||
|
||||
/** What the import asks of an IMAP box, one connection per run. */
|
||||
export interface ImapSession {
|
||||
/** Opens the folder; answers the uid validity, which changes when the server renumbers it. */
|
||||
open(mailbox: string): Promise<{ uidValidity: number }>;
|
||||
/** The mails with a uid above `lastUid`, oldest first. */
|
||||
listAfter(lastUid: number): Promise<ImapMailRef[]>;
|
||||
/** One mail's raw bytes; null for a uid that is gone. */
|
||||
source(uid: number): Promise<Buffer | null>;
|
||||
/** Makes the folder (made of these parts, in the server's own delimiter) unless it exists; answers its path. */
|
||||
ensureFolder(parts: readonly string[]): Promise<string>;
|
||||
move(uids: readonly number[], folder: string): Promise<void>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens the admin's IMAP box. The host goes through the SSRF guard first,
|
||||
* with the same allowance for a box on the own network as every other
|
||||
* address an admin configures (ALLOW_INTERNAL_NETWORK), and the connection
|
||||
* goes to the address the guard vetted, the certificate still checked
|
||||
* against the host's name, so a name that resolves differently a moment
|
||||
* later cannot redirect it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestImapClient {
|
||||
async connect(config: MailIngestImapConfig): Promise<ImapSession> {
|
||||
const address = await vettedAddress(config.host);
|
||||
const client = new ImapFlow({
|
||||
host: address,
|
||||
port: config.port,
|
||||
secure: config.tls === 'tls',
|
||||
doSTARTTLS: config.tls === 'starttls' ? true : undefined,
|
||||
servername: isIP(config.host) ? undefined : config.host,
|
||||
auth: { user: config.user, pass: config.pass },
|
||||
logger: false,
|
||||
disableAutoIdle: true,
|
||||
connectionTimeout: CONNECT_TIMEOUT_MS,
|
||||
greetingTimeout: CONNECT_TIMEOUT_MS,
|
||||
socketTimeout: SOCKET_TIMEOUT_MS,
|
||||
});
|
||||
await client.connect();
|
||||
return new ImapFlowSession(client);
|
||||
}
|
||||
}
|
||||
|
||||
/** The address a connection to `host` may use, or a refusal naming why. */
|
||||
async function vettedAddress(host: string): Promise<string> {
|
||||
const verdict = await checkSsrf(`https://${isIP(host) === 6 ? `[${host}]` : host}/`);
|
||||
if (!verdict.allowed || !verdict.resolvedIp) {
|
||||
throw new SsrfBlockedError(verdict.error ?? 'This IMAP host is not allowed');
|
||||
}
|
||||
return verdict.resolvedIp;
|
||||
}
|
||||
|
||||
class ImapFlowSession implements ImapSession {
|
||||
constructor(private readonly client: ImapFlow) {}
|
||||
|
||||
async open(mailbox: string): Promise<{ uidValidity: number }> {
|
||||
const box = await this.client.mailboxOpen(mailbox);
|
||||
return { uidValidity: Number(box.uidValidity) };
|
||||
}
|
||||
|
||||
async listAfter(lastUid: number): Promise<ImapMailRef[]> {
|
||||
const found: ImapMailRef[] = [];
|
||||
// `n:*` always names the newest mail, even below n, so the range alone does not filter.
|
||||
for await (const message of this.client.fetch(`${lastUid + 1}:*`, { uid: true, size: true }, { uid: true })) {
|
||||
if (message.uid > lastUid) found.push({ uid: message.uid, size: message.size ?? 0 });
|
||||
}
|
||||
return found.sort((a, b) => a.uid - b.uid);
|
||||
}
|
||||
|
||||
async source(uid: number): Promise<Buffer | null> {
|
||||
const message = await this.client.fetchOne(String(uid), { source: true }, { uid: true });
|
||||
return message && Buffer.isBuffer(message.source) ? message.source : null;
|
||||
}
|
||||
|
||||
async ensureFolder(parts: readonly string[]): Promise<string> {
|
||||
const created = await this.client.mailboxCreate([...parts]);
|
||||
return created.path;
|
||||
}
|
||||
|
||||
async move(uids: readonly number[], folder: string): Promise<void> {
|
||||
if (uids.length === 0) return;
|
||||
await this.client.messageMove(uids.join(','), folder, { uid: true });
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
await this.client.logout();
|
||||
} catch {
|
||||
// The connection broke on its own; there is nothing left to say goodbye to.
|
||||
this.client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** What an admin reads when the box cannot be read: the reason, never the password. */
|
||||
export function describeImapFailure(err: unknown): string {
|
||||
if (err instanceof SsrfBlockedError) return err.message;
|
||||
const failure = (err ?? {}) as {
|
||||
authenticationFailed?: boolean;
|
||||
code?: string;
|
||||
responseText?: string;
|
||||
message?: string;
|
||||
};
|
||||
if (failure.authenticationFailed) return 'The IMAP server refused the user name or password';
|
||||
if (failure.code && /^E[A-Z]+$/.test(failure.code)) return `Could not reach the IMAP server (${failure.code})`;
|
||||
const said = failure.responseText || failure.message || String(err);
|
||||
return `The IMAP server answered: ${said}`.slice(0, 300);
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
|
||||
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { logError, logWarn } from '../audit/audit-log.logger';
|
||||
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
|
||||
import {
|
||||
describeImapFailure,
|
||||
MailIngestImapClient,
|
||||
type ImapMailRef,
|
||||
type ImapSession,
|
||||
} from './mail-ingest-imap.client';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestSettingsService, type MailIngestImapConfig } from './mail-ingest-settings.service';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
/** How long a run holds the box before another process may take it over. */
|
||||
const LEASE_MS = 10 * 60 * 1000;
|
||||
/** The mails one run takes from the box; the rest wait for the next run. */
|
||||
const MAILS_PER_RUN = 20;
|
||||
/** Where read mails go: kept, never deleted, out of the folder the next run reads. */
|
||||
const PROCESSED_FOLDER = ['TREK', 'processed'] as const;
|
||||
|
||||
/**
|
||||
* Reads the admin's IMAP box every two minutes, with a client of its own for
|
||||
* every run. Each new mail is handed to the intake, which stores it once,
|
||||
* and then moved to TREK/processed. The cursor (uid validity, last uid)
|
||||
* keeps a mail the move missed from being read twice, and the intake's
|
||||
* dedupe keeps it so when the server renumbers the box and the run starts
|
||||
* over. One process holds the box at a time, by the cursor's lease.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly intake: MailIngestIntakeService,
|
||||
private readonly imap: MailIngestImapClient,
|
||||
private readonly registrar: CronRegistrarService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('mail-ingest-imap', '*/2 * * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* One run; answers how many mails it took from the box. Nothing happens
|
||||
* unless the import is on, IMAP is its transport and the box is complete.
|
||||
*
|
||||
* @txIndependent each mail is stored on its own as it is read; the cursor
|
||||
* notes where the run got to once the box is done with.
|
||||
*/
|
||||
async tick(now: Date = new Date()): Promise<number> {
|
||||
if (!(await this.settings.isAvailable()) || (await this.settings.transport()) !== 'imap') return 0;
|
||||
const config = await this.settings.imapConfig();
|
||||
if (!config) return 0;
|
||||
const until = dbNow(new Date(now.getTime() + LEASE_MS));
|
||||
if (!(await this.cursor.acquireLease('imap', config.mailbox, LEASE_OWNER, until, dbNow(now)))) return 0;
|
||||
try {
|
||||
const { taken, uidValidity, lastUid } = await this.poll(config);
|
||||
await this.cursor.recordOk('imap', config.mailbox, dbNow(new Date()), { uidValidity, lastUid });
|
||||
return taken;
|
||||
} catch (err) {
|
||||
const reason = describeImapFailure(err);
|
||||
logError(`[mail-ingest] Reading the IMAP box failed: ${reason}`);
|
||||
await this.cursor.recordError('imap', config.mailbox, reason);
|
||||
return 0;
|
||||
} finally {
|
||||
await this.cursor.releaseLease('imap', config.mailbox, LEASE_OWNER);
|
||||
}
|
||||
}
|
||||
|
||||
private async poll(config: MailIngestImapConfig): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
|
||||
const session = await this.imap.connect(config);
|
||||
try {
|
||||
const { uidValidity } = await session.open(config.mailbox);
|
||||
const position = await this.cursor.find('imap', config.mailbox);
|
||||
// A changed uid validity means the server renumbered the box: start over, the dedupe keeps each mail once.
|
||||
const after = position?.uid_validity === uidValidity ? (position.last_uid ?? 0) : 0;
|
||||
const due = (await session.listAfter(after)).slice(0, MAILS_PER_RUN);
|
||||
if (due.length === 0) return { taken: 0, uidValidity, lastUid: after };
|
||||
const folder = await session.ensureFolder(PROCESSED_FOLDER);
|
||||
await due.reduce(async (previous, mail) => {
|
||||
await previous;
|
||||
await this.takeIn(session, mail);
|
||||
}, Promise.resolve());
|
||||
await session.move(due.map((mail) => mail.uid), folder);
|
||||
return { taken: due.length, uidValidity, lastUid: Math.max(...due.map((mail) => mail.uid)) };
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** One mail into the intake. One larger than the import takes is left unread, and moved on with the rest. */
|
||||
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<void> {
|
||||
if (mail.size > this.settings.maxMailBytes()) {
|
||||
logWarn(`[mail-ingest] An IMAP mail of ${mail.size} bytes is larger than the import takes; left unread`);
|
||||
return;
|
||||
}
|
||||
const raw = await session.source(mail.uid);
|
||||
if (!raw) return;
|
||||
const result = await this.intake.receive(raw, 'imap');
|
||||
if (result.status === 'refused') logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Public } from '../auth-core/public.decorator';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { ResponseContract } from '../common/response-contract';
|
||||
import {
|
||||
MailIngestWebhookService,
|
||||
WEBHOOK_RECIPIENT_HEADER,
|
||||
WEBHOOK_SIGNATURE_HEADER,
|
||||
WEBHOOK_TIMESTAMP_HEADER,
|
||||
} from './mail-ingest-webhook.service';
|
||||
import { Controller, HttpCode, Post, Req } from '@nestjs/common';
|
||||
import { mailIngestInboundResponseSchema, type MailIngestInboundResponse } from '@trek/shared';
|
||||
|
||||
import type { Request } from 'express';
|
||||
|
||||
const TOO_LARGE = 'This mail is larger than the import takes';
|
||||
|
||||
/**
|
||||
* `POST /api/mail-ingest/inbound`: the signed webhook a mail service posts
|
||||
* every mail to, raw (message/rfc822). It does not exist (404) unless the
|
||||
* import is on, the webhook is its transport and a secret is set; then the
|
||||
* body is read up to the import's limit, and the signature over it decides.
|
||||
* The JSON parsers leave a message/rfc822 body alone, so this route reads
|
||||
* the stream itself and nothing of the mail reaches the request log.
|
||||
*/
|
||||
@Controller('api/mail-ingest/inbound')
|
||||
export class MailIngestInboundController {
|
||||
constructor(private readonly webhook: MailIngestWebhookService) {}
|
||||
|
||||
@Post()
|
||||
@Public(
|
||||
'A mail service cannot hold a TREK session; an HMAC signature over the raw mail, its recipient and a timestamp within five minutes is the credential, and the route answers 404 unless the webhook is the configured transport.',
|
||||
)
|
||||
@HttpCode(200)
|
||||
@ResponseContract(mailIngestInboundResponseSchema)
|
||||
async receive(@Req() req: Request): Promise<MailIngestInboundResponse> {
|
||||
const secret = await this.webhook.secret();
|
||||
if (!req.is('message/rfc822')) {
|
||||
throw new DomainError(415, 'Send the raw mail as message/rfc822');
|
||||
}
|
||||
const raw = await readBody(req, this.webhook.maxBytes());
|
||||
return this.webhook.receive(secret, raw, {
|
||||
timestamp: req.get(WEBHOOK_TIMESTAMP_HEADER),
|
||||
signature: req.get(WEBHOOK_SIGNATURE_HEADER),
|
||||
recipient: req.get(WEBHOOK_RECIPIENT_HEADER),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The request body, up to `maxBytes`. Past the limit the rest is drained
|
||||
* unread, so the sender gets the 413 rather than a reset connection.
|
||||
*/
|
||||
function readBody(req: Request, maxBytes: number): Promise<Buffer> {
|
||||
if (Number(req.get('content-length')) > maxBytes) return Promise.reject(new DomainError(413, TOO_LARGE));
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
req.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size <= maxBytes) chunks.push(chunk);
|
||||
});
|
||||
req.on('end', () => (size > maxBytes ? reject(new DomainError(413, TOO_LARGE)) : resolve(Buffer.concat(chunks))));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
|
||||
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { logError, logWarn } from '../audit/audit-log.logger';
|
||||
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestRelayHost, type MailInboxRelay } from './mail-ingest-relay.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
|
||||
/** How long a run holds the relay before another process may take it over. */
|
||||
const LEASE_MS = 10 * 60 * 1000;
|
||||
/** The mails one run asks the relay for. */
|
||||
const PAGE = 10;
|
||||
/** The base64 length of the largest mail the relay hands on. */
|
||||
const MAX_RAW_CHARS = Math.ceil(MAIL_INGEST_LIMITS.cloudMailBytes / 3) * 4;
|
||||
|
||||
/** The relay's answer, checked before anything of it is used. */
|
||||
const relayInboxSchema = z.object({
|
||||
mails: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/),
|
||||
/** The addresses it was sent to, as the relay's mail server took them (RCPT TO). */
|
||||
recipients: z.array(z.string().min(3).max(320)).min(1).max(5),
|
||||
sha256: z.string().regex(/^[0-9a-f]{64}$/),
|
||||
/** The mail itself, base64. */
|
||||
raw: z.string().max(MAX_RAW_CHARS),
|
||||
}),
|
||||
)
|
||||
.max(PAGE),
|
||||
});
|
||||
type RelayedMail = z.infer<typeof relayInboxSchema>['mails'][number];
|
||||
|
||||
/**
|
||||
* Collects this instance's forwarded mails from the TREK Cloud relay every
|
||||
* two minutes: a page of at most ten, each checked against its hash, handed
|
||||
* to the intake under the address it was sent to, and acknowledged once it
|
||||
* is in (or once it is clear it never will be), so the relay drops it. A
|
||||
* mail that does not arrive whole is left for the next run. Idle wherever
|
||||
* no relay is attached, which is every self-hosted install.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestRelayJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly intake: MailIngestIntakeService,
|
||||
private readonly relays: MailIngestRelayHost,
|
||||
private readonly registrar: CronRegistrarService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('mail-ingest-relay', '*/2 * * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* One run; answers how many mails it took in.
|
||||
*
|
||||
* @txIndependent each mail is stored on its own and acknowledged after it, network I/O between them.
|
||||
*/
|
||||
async tick(now: Date = new Date()): Promise<number> {
|
||||
const relay = this.relays.current();
|
||||
if (!relay || !(await this.settings.isAvailable()) || (await this.settings.transport()) !== 'cloud') return 0;
|
||||
const until = dbNow(new Date(now.getTime() + LEASE_MS));
|
||||
if (!(await this.cursor.acquireLease('cloud', '', LEASE_OWNER, until, dbNow(now)))) return 0;
|
||||
try {
|
||||
const page = relayInboxSchema.safeParse(await relay.list(PAGE));
|
||||
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
|
||||
const taken = await page.data.mails.reduce(async (previous, mail) => {
|
||||
const count = await previous;
|
||||
return count + ((await this.takeIn(relay, mail)) ? 1 : 0);
|
||||
}, Promise.resolve(0));
|
||||
await this.cursor.recordOk('cloud', '', dbNow(new Date()));
|
||||
return taken;
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
logError(`[mail-ingest] Collecting mails from the relay failed: ${reason}`);
|
||||
await this.cursor.recordError('cloud', '', reason);
|
||||
return 0;
|
||||
} finally {
|
||||
await this.cursor.releaseLease('cloud', '', LEASE_OWNER);
|
||||
}
|
||||
}
|
||||
|
||||
/** One relayed mail into the intake; answers whether it was taken in. */
|
||||
private async takeIn(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
|
||||
const raw = Buffer.from(mail.raw, 'base64');
|
||||
if (createHash('sha256').update(raw).digest('hex') !== mail.sha256) {
|
||||
logWarn(`[mail-ingest] Relayed mail ${mail.id} did not arrive whole; left for the next run`);
|
||||
return false;
|
||||
}
|
||||
const result = await this.intake.receive(raw, 'cloud', mail.recipients);
|
||||
// Switched off in between: the relay keeps the mail until the import is back or the mail expires.
|
||||
if (result.status === 'refused' && result.reason === 'unavailable') return false;
|
||||
if (result.status === 'refused') {
|
||||
logWarn(`[mail-ingest] Relayed mail ${mail.id} was not taken in: ${result.reason}`);
|
||||
}
|
||||
await relay.ack(mail.id);
|
||||
return result.status === 'accepted';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* The TREK Cloud control plane's queue of mails for this instance, as the
|
||||
* managed install reaches it with its instance token. The control plane's
|
||||
* mail server takes a forwarded mail for a contract's route key, keeps it
|
||||
* encrypted, and hands it over here; nothing of a person reaches it.
|
||||
*/
|
||||
export interface MailInboxRelay {
|
||||
/** `GET /v1/instance/inbox?limit=n`: the waiting mails, the answer's JSON as parsed, checked by the caller. */
|
||||
list(limit: number): Promise<unknown>;
|
||||
/** `POST /v1/instance/inbox/:id/ack`: the mail is taken in (or never will be), and the relay drops it. */
|
||||
ack(id: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the managed install plugs its control-plane client in. This
|
||||
* repository has none: a self-hosted install has no relay, and the pull job
|
||||
* stays idle until a module of the managed build attaches one when it
|
||||
* starts, through the client it already talks to the control plane with.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestRelayHost {
|
||||
private relay: MailInboxRelay | null = null;
|
||||
|
||||
attach(relay: MailInboxRelay): void {
|
||||
this.relay = relay;
|
||||
}
|
||||
|
||||
current(): MailInboxRelay | null {
|
||||
return this.relay;
|
||||
}
|
||||
}
|
||||
@@ -79,7 +79,9 @@ export class MailIngestSettingsService {
|
||||
const user = await this.pick(this.config.imapUser, 'mail_ingest_imap_user');
|
||||
const pass = this.config.imapPass ?? decrypt_api_key(await this.settings.getAppSetting('mail_ingest_imap_pass'));
|
||||
if (!host || !user || !pass) return null;
|
||||
const tls = mailIngestImapTlsSchema.catch('tls').parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls'));
|
||||
const tls = mailIngestImapTlsSchema
|
||||
.catch('tls')
|
||||
.parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls'));
|
||||
const port = Number(await this.pick(this.config.imapPort, 'mail_ingest_imap_port'));
|
||||
const mailbox = await this.pick(this.config.imapMailbox, 'mail_ingest_imap_mailbox');
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow, utcSuffix } from '../../db/types';
|
||||
import { describeImapFailure, MailIngestImapClient } from './mail-ingest-imap.client';
|
||||
import { MailIngestRelayHost } from './mail-ingest-relay.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
mailIngestMessageStatusSchema,
|
||||
mailIngestReasonSchema,
|
||||
type MailIngestAdminStatusResponse,
|
||||
type MailIngestAdminTestResponse,
|
||||
type MailIngestTransportSetting,
|
||||
} from '@trek/shared';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* The admin's view of the import: how its transport runs and what became of
|
||||
* the mails of the last day, in counts only. No subject, sender, address or
|
||||
* content of any mail is part of it, and the backlog is a list of numbers,
|
||||
* not of people. The connection test reaches the transport the way the
|
||||
* import does and says what failed.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestStatusService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly imap: MailIngestImapClient,
|
||||
private readonly relays: MailIngestRelayHost,
|
||||
) {}
|
||||
|
||||
async status(now: Date = new Date()): Promise<MailIngestAdminStatusResponse> {
|
||||
const transport = await this.settings.transport();
|
||||
const reader = await this.readerCursor(transport);
|
||||
const since = dbNow(new Date(now.getTime() - DAY_MS));
|
||||
const byStatus: MailIngestAdminStatusResponse['last_day']['by_status'] = {};
|
||||
const byReason: MailIngestAdminStatusResponse['last_day']['by_reason'] = {};
|
||||
for (const { status, reason, n } of await this.messages.tallySince(since)) {
|
||||
const knownStatus = mailIngestMessageStatusSchema.safeParse(status);
|
||||
if (knownStatus.success) byStatus[knownStatus.data] = (byStatus[knownStatus.data] ?? 0) + n;
|
||||
const knownReason = mailIngestReasonSchema.safeParse(reason);
|
||||
if (knownReason.success) byReason[knownReason.data] = (byReason[knownReason.data] ?? 0) + n;
|
||||
}
|
||||
const resolutions = new Map(
|
||||
(await this.candidates.tallyResolutionsSince(since)).map(({ resolution, n }) => [resolution, n]),
|
||||
);
|
||||
const startOfDay = dbNow(new Date(`${now.toISOString().slice(0, 10)}T00:00:00Z`));
|
||||
return {
|
||||
transport,
|
||||
last_ok_at: utcSuffix(reader?.last_ok_at),
|
||||
last_error: reader?.last_error ?? null,
|
||||
lease_until: utcSuffix(reader?.lease_until),
|
||||
last_day: { by_status: byStatus, by_reason: byReason },
|
||||
auto_applied: resolutions.get('auto_applied') ?? 0,
|
||||
undone: resolutions.get('undone') ?? 0,
|
||||
ai_calls_today: await this.messages.countAllAiReadSince(startOfDay),
|
||||
backlog: await this.candidates.pendingPerUser(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether the configured transport can reach its mail now, and if not, why. */
|
||||
async test(): Promise<MailIngestAdminTestResponse> {
|
||||
const transport = await this.settings.transport();
|
||||
const error = await this.problemWith(transport);
|
||||
return { transport, ok: error === null, error };
|
||||
}
|
||||
|
||||
private async problemWith(transport: MailIngestTransportSetting): Promise<string | null> {
|
||||
switch (transport) {
|
||||
case 'off':
|
||||
return 'No transport is set; mails come in by upload only';
|
||||
case 'webhook':
|
||||
return (await this.settings.webhookSecret()) ? null : 'No webhook secret is set';
|
||||
case 'cloud':
|
||||
return this.relays.current() ? null : 'The TREK Cloud relay is not connected';
|
||||
case 'imap':
|
||||
return this.imapProblem();
|
||||
}
|
||||
}
|
||||
|
||||
/** Opens the box and its folder the way a run does, and lets go again. */
|
||||
private async imapProblem(): Promise<string | null> {
|
||||
const config = await this.settings.imapConfig();
|
||||
if (!config) return 'The IMAP host, user or password is missing';
|
||||
try {
|
||||
const session = await this.imap.connect(config);
|
||||
try {
|
||||
await session.open(config.mailbox);
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
return null;
|
||||
} catch (err) {
|
||||
return describeImapFailure(err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Where the reader of the transport stands: the IMAP box's or the relay's cursor, else the pipeline's own. */
|
||||
private async readerCursor(transport: MailIngestTransportSetting) {
|
||||
if (transport === 'imap') {
|
||||
const config = await this.settings.imapConfig();
|
||||
return config ? this.cursor.find('imap', config.mailbox) : null;
|
||||
}
|
||||
return this.cursor.find(transport === 'cloud' ? 'cloud' : 'pipeline', '');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { MailIngestIntakeService, type MailIntakeRefusal } from './mail-ingest-intake.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { MailIngestInboundResponse } from '@trek/shared';
|
||||
|
||||
import { createHmac, timingSafeEqual } from 'node:crypto';
|
||||
|
||||
/** The headers a webhook sender signs a mail with, and the address it was sent to (the RCPT TO). */
|
||||
export const WEBHOOK_TIMESTAMP_HEADER = 'x-trek-timestamp';
|
||||
export const WEBHOOK_SIGNATURE_HEADER = 'x-trek-signature';
|
||||
export const WEBHOOK_RECIPIENT_HEADER = 'x-trek-recipient';
|
||||
|
||||
/** How far a signed timestamp may lie from the server's clock, either way. */
|
||||
const WINDOW_MS = 5 * 60 * 1000;
|
||||
|
||||
/** How a mail the intake refused answers the sender, by why. */
|
||||
const REFUSALS: Record<MailIntakeRefusal, [status: number, error: string]> = {
|
||||
unavailable: [404, 'Not found'],
|
||||
too_large: [413, 'This mail is larger than the import takes'],
|
||||
not_a_mail: [422, 'This is not a mail'],
|
||||
unknown_address: [422, 'No address of this server takes this mail'],
|
||||
address_daily_limit: [429, 'This address took all the mails it takes today'],
|
||||
inbox_full: [429, 'The inbox of this address is full'],
|
||||
};
|
||||
|
||||
/** The signed parts of a webhook request, as the sender put them in its headers. */
|
||||
interface WebhookSignedParts {
|
||||
timestamp?: string;
|
||||
signature?: string;
|
||||
recipient?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The signed webhook: a mail service (a Cloudflare Email Worker, Postmark,
|
||||
* Mailgun) posts each mail to the install raw, signed with the admin's
|
||||
* secret. The signature covers the time it was sent, the address it was
|
||||
* sent to and every byte of the mail, so neither can be changed on the way,
|
||||
* and a request older than five minutes is refused. A request replayed
|
||||
* within that window stores nothing new: the intake keeps one copy of a mail
|
||||
* per address.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestWebhookService {
|
||||
constructor(
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly intake: MailIngestIntakeService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* The secret mails are checked against. While the import is off, the
|
||||
* webhook is not its transport or no secret is set, the route answers as
|
||||
* if it did not exist.
|
||||
*/
|
||||
async secret(): Promise<string> {
|
||||
const open = (await this.settings.isAvailable()) && (await this.settings.transport()) === 'webhook';
|
||||
const secret = open ? await this.settings.webhookSecret() : null;
|
||||
if (!secret) throw new DomainError(404, 'Not found');
|
||||
return secret;
|
||||
}
|
||||
|
||||
maxBytes(): number {
|
||||
return this.settings.maxMailBytes();
|
||||
}
|
||||
|
||||
async receive(
|
||||
secret: string,
|
||||
raw: Buffer,
|
||||
signed: WebhookSignedParts,
|
||||
now: number = Date.now(),
|
||||
): Promise<MailIngestInboundResponse> {
|
||||
const recipient = signed.recipient?.trim() ?? '';
|
||||
if (!verifyWebhook(secret, signed, recipient, raw, now)) throw new DomainError(401, 'The signature does not match');
|
||||
const result = await this.intake.receive(raw, 'webhook', recipient ? [recipient] : []);
|
||||
if (result.status === 'refused') {
|
||||
const [status, error] = REFUSALS[result.reason];
|
||||
throw new DomainError(status, error);
|
||||
}
|
||||
return { accepted: true, duplicate: result.status === 'duplicate' };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The signature a sender puts on a mail: hex HMAC-SHA256, keyed with the
|
||||
* secret, over the timestamp (Unix seconds), a newline, the recipient
|
||||
* address (empty when the sender names none), a newline, and the raw mail.
|
||||
*/
|
||||
export function signWebhookMail(secret: string, timestamp: string, recipient: string, raw: Buffer): string {
|
||||
return createHmac('sha256', secret).update(`${timestamp}\n${recipient}\n`).update(raw).digest('hex');
|
||||
}
|
||||
|
||||
function verifyWebhook(
|
||||
secret: string,
|
||||
signed: WebhookSignedParts,
|
||||
recipient: string,
|
||||
raw: Buffer,
|
||||
now: number,
|
||||
): boolean {
|
||||
const { timestamp, signature } = signed;
|
||||
if (!timestamp || !/^\d{1,12}$/.test(timestamp) || Math.abs(now - Number(timestamp) * 1000) > WINDOW_MS) {
|
||||
return false;
|
||||
}
|
||||
const given = (signature ?? '').trim().replace(/^sha256=/i, '').toLowerCase();
|
||||
if (!/^[0-9a-f]{64}$/.test(given) || recipient.length > 320) return false;
|
||||
const expected = signWebhookMail(secret, timestamp, recipient, raw);
|
||||
return timingSafeEqual(Buffer.from(given, 'hex'), Buffer.from(expected, 'hex'));
|
||||
}
|
||||
@@ -21,6 +21,9 @@ import { SettingsModule } from '../settings/settings.module';
|
||||
import { MailIngestActionsService } from './mail-ingest-actions.service';
|
||||
import { MailIngestAddressService } from './mail-ingest-address.service';
|
||||
import { MailIngestApplyService } from './mail-ingest-apply.service';
|
||||
import { MailIngestImapClient } from './mail-ingest-imap.client';
|
||||
import { MailIngestImapJob } from './mail-ingest-imap.job';
|
||||
import { MailIngestInboundController } from './mail-ingest-inbound.controller';
|
||||
import { MailIngestInboxService } from './mail-ingest-inbox.service';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestNotificationActions } from './mail-ingest-notification-actions';
|
||||
@@ -28,19 +31,25 @@ import { MailIngestNotifyService } from './mail-ingest-notify.service';
|
||||
import { MailIngestProcessJob } from './mail-ingest-process.job';
|
||||
import { MailIngestProcessService } from './mail-ingest-process.service';
|
||||
import { MailIngestReadingService } from './mail-ingest-reading.service';
|
||||
import { MailIngestRelayJob } from './mail-ingest-relay.job';
|
||||
import { MailIngestRelayHost } from './mail-ingest-relay.service';
|
||||
import { MailIngestRetentionJob } from './mail-ingest-retention.job';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MailIngestStatusService } from './mail-ingest-status.service';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { MailIngestWebhookService } from './mail-ingest-webhook.service';
|
||||
import { MikroOrmModule } from '@mikro-orm/nestjs';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* The booking mail import: a person forwards a booking mail to an address of
|
||||
* their own, and the booking lands in the matching trip, or waits in their
|
||||
* inbox with a reason. Transports (IMAP, the signed webhook, the TREK Cloud
|
||||
* relay, an .eml upload) only hand mail to MailIngestIntakeService; the
|
||||
* processing job reads it; everything a person does with an entry goes
|
||||
* through MailIngestActionsService.
|
||||
* inbox with a reason. Transports (the IMAP job, the signed webhook, the
|
||||
* TREK Cloud relay job, an .eml upload) only hand mail to
|
||||
* MailIngestIntakeService; the processing job reads it; everything a person
|
||||
* does with an entry goes through MailIngestActionsService, from REST and
|
||||
* MCP alike. MailIngestRelayHost is exported for the managed build, which
|
||||
* attaches its control-plane client to it.
|
||||
*/
|
||||
@Module({
|
||||
imports: [
|
||||
@@ -67,6 +76,7 @@ import { Module } from '@nestjs/common';
|
||||
MailIngestCandidates,
|
||||
]),
|
||||
],
|
||||
controllers: [MailIngestInboundController],
|
||||
providers: [
|
||||
MailIngestSettingsService,
|
||||
MailIngestTripsService,
|
||||
@@ -78,10 +88,22 @@ import { Module } from '@nestjs/common';
|
||||
MailIngestProcessService,
|
||||
MailIngestActionsService,
|
||||
MailIngestInboxService,
|
||||
MailIngestStatusService,
|
||||
MailIngestWebhookService,
|
||||
MailIngestImapClient,
|
||||
MailIngestRelayHost,
|
||||
MailIngestNotificationActions,
|
||||
MailIngestProcessJob,
|
||||
MailIngestRetentionJob,
|
||||
MailIngestImapJob,
|
||||
MailIngestRelayJob,
|
||||
],
|
||||
exports: [
|
||||
MailIngestSettingsService,
|
||||
MailIngestAddressService,
|
||||
MailIngestIntakeService,
|
||||
MailIngestActionsService,
|
||||
MailIngestRelayHost,
|
||||
],
|
||||
exports: [MailIngestSettingsService, MailIngestAddressService, MailIngestIntakeService, MailIngestActionsService],
|
||||
})
|
||||
export class MailIngestModule {}
|
||||
|
||||
Reference in New Issue
Block a user