fix(mail-ingest): guard the public webhook and the settings that open it

The webhook refuses headers that cannot be signed right before reading a
byte, limits each address to 120 requests a minute and reads four bodies at
once. The admin save refuses a webhook secret under 32 characters, and the
MCP settings tool registers its input strict like every other tool.
This commit is contained in:
Maurice
2026-11-13 17:05:59 +01:00
parent 32291ce8b4
commit 264f352502
8 changed files with 143 additions and 31 deletions
+6 -1
View File
@@ -770,7 +770,12 @@ export class AuthService {
if (key === 'places_provider' && !isPlacesProviderChoice(val)) continue;
// A masked key sent back unchanged keeps its stored value.
if (MASKED_ADMIN_SETTING_KEYS.has(key) && val === '••••••••') continue;
const encrypted: AppSettingDef['encrypted'] = (APP_SETTINGS[key] as AppSettingDef).encrypted;
const def = APP_SETTINGS[key] as AppSettingDef;
if (def.validated) {
const checked = def.schema.safeParse(val);
if (!checked.success) throw new DomainError(400, checked.error.issues[0]?.message ?? `${key} is not valid`);
}
const encrypted: AppSettingDef['encrypted'] = def.encrypted;
if (encrypted === 'always') val = encrypt_api_key(val);
else if (encrypted === 'if-plain' && val) val = maybe_encrypt_api_key(val) ?? val;
await this.appSettings.setValue(key, val);
@@ -46,6 +46,8 @@ export interface AppSettingDef {
readonly encrypted?: 'always' | 'if-plain';
readonly managedLocked?: true;
readonly publicConfig?: true;
/** The admin save refuses a value the schema does not take, with the schema's own message, instead of storing it. */
readonly validated?: true;
}
// The admin form's keys come first, in the order the form has always answered them.
@@ -192,12 +194,14 @@ export const APP_SETTINGS = {
mail_ingest_address_template: { schema: text, default: null, adminForm: true, managedLocked: true },
/** The key the webhook's sender signs every mail with (HMAC-SHA256). */
mail_ingest_webhook_secret: {
schema: text,
// As MAIL_INGEST_WEBHOOK_SECRET: a short secret's HMAC can be guessed. Empty switches the webhook off.
schema: z.union([z.literal(''), z.string().min(32, 'The webhook secret needs at least 32 characters')]),
default: null,
adminForm: true,
masked: true,
encrypted: 'always',
managedLocked: true,
validated: true,
},
/** How many forwarded mails a day an AI reads for one person. */
mail_ingest_ai_per_user_daily: { schema: intText, default: '10', adminForm: true, managedLocked: true },
@@ -1,11 +1,13 @@
import { Public } from '../auth-core/public.decorator';
import { DomainError } from '../common/domain-error';
import { RateLimitService } from '../common/rate-limit.service';
import { ResponseContract } from '../common/response-contract';
import {
MailIngestWebhookService,
WEBHOOK_RECIPIENT_HEADER,
WEBHOOK_SIGNATURE_HEADER,
WEBHOOK_TIMESTAMP_HEADER,
type WebhookSignedParts,
} from './mail-ingest-webhook.service';
import { Controller, HttpCode, Post, Req } from '@nestjs/common';
import { mailIngestInboundResponseSchema, type MailIngestInboundResponse } from '@trek/shared';
@@ -13,18 +15,30 @@ import { mailIngestInboundResponseSchema, type MailIngestInboundResponse } from
import type { Request } from 'express';
const TOO_LARGE = 'This mail is larger than the import takes';
/** Requests one address may post in a minute: a mail service delivers in bursts, never at this rate for long. */
const REQUESTS_PER_MINUTE = 120;
/** Bodies read at once, from all senders together: each may hold the import's largest mail in memory. */
const MAX_BODIES_IN_FLIGHT = 4;
/**
* `POST /api/mail-ingest/inbound`: the signed webhook a mail service posts
* every mail to, raw (message/rfc822). It does not exist (404) unless the
* import is on, the webhook is its transport and a secret is set; then the
* body is read up to the import's limit, and the signature over it decides.
* The JSON parsers leave a message/rfc822 body alone, so this route reads
* the stream itself and nothing of the mail reaches the request log.
* Before a byte of the body is read, a request whose headers cannot carry a
* valid signature is refused, each address has its share of requests a
* minute, and only a few bodies are read at once; a sender told 429 tries
* again. The JSON parsers leave a message/rfc822 body alone, so this route
* reads the stream itself and nothing of the mail reaches the request log.
*/
@Controller('api/mail-ingest/inbound')
export class MailIngestInboundController {
constructor(private readonly webhook: MailIngestWebhookService) {}
private reading = 0;
constructor(
private readonly webhook: MailIngestWebhookService,
private readonly rateLimit: RateLimitService,
) {}
@Post()
@Public(
@@ -37,12 +51,28 @@ export class MailIngestInboundController {
if (!req.is('message/rfc822')) {
throw new DomainError(415, 'Send the raw mail as message/rfc822');
}
const raw = await readBody(req, this.webhook.maxBytes());
return this.webhook.receive(secret, raw, {
const signed: WebhookSignedParts = {
timestamp: req.get(WEBHOOK_TIMESTAMP_HEADER),
signature: req.get(WEBHOOK_SIGNATURE_HEADER),
recipient: req.get(WEBHOOK_RECIPIENT_HEADER),
});
};
if (!this.webhook.plausiblySigned(signed)) throw new DomainError(401, 'The signature does not match');
const now = Date.now();
if (!(await this.rateLimit.check('mail-ingest-webhook', req.ip ?? 'unknown', REQUESTS_PER_MINUTE, 60_000, now))) {
throw new DomainError(429, 'Too many mails in a short time; try again later');
}
return this.webhook.receive(secret, await this.readLimited(req), signed);
}
/** The body, while fewer than {@link MAX_BODIES_IN_FLIGHT} others are being read. */
private async readLimited(req: Request): Promise<Buffer> {
if (this.reading >= MAX_BODIES_IN_FLIGHT) throw new DomainError(429, 'Too many mails at once; try again shortly');
this.reading += 1;
try {
return await readBody(req, this.webhook.maxBytes());
} finally {
this.reading -= 1;
}
}
}
@@ -25,7 +25,7 @@ const REFUSALS: Record<MailIntakeRefusal, [status: number, error: string]> = {
};
/** The signed parts of a webhook request, as the sender put them in its headers. */
interface WebhookSignedParts {
export interface WebhookSignedParts {
timestamp?: string;
signature?: string;
recipient?: string;
@@ -63,6 +63,20 @@ export class MailIngestWebhookService {
return this.settings.maxMailBytes();
}
/**
* Whether the headers could carry a valid signature at all: a fresh
* timestamp, a signature of the right form and a recipient within its
* length. Checked before the body is read, so a request that can never
* pass holds no memory.
*/
plausiblySigned(signed: WebhookSignedParts, now: number = Date.now()): boolean {
const { timestamp } = signed;
if (!timestamp || !/^\d{1,12}$/.test(timestamp) || Math.abs(now - Number(timestamp) * 1000) > WINDOW_MS) {
return false;
}
return /^[0-9a-f]{64}$/.test(signatureOf(signed)) && (signed.recipient?.trim() ?? '').length <= 320;
}
async receive(
secret: string,
raw: Buffer,
@@ -70,7 +84,9 @@ export class MailIngestWebhookService {
now: number = Date.now(),
): Promise<MailIngestInboundResponse> {
const recipient = signed.recipient?.trim() ?? '';
if (!verifyWebhook(secret, signed, recipient, raw, now)) throw new DomainError(401, 'The signature does not match');
if (!this.plausiblySigned(signed, now) || !verifyWebhook(secret, signed, recipient, raw)) {
throw new DomainError(401, 'The signature does not match');
}
const result = await this.intake.receive(raw, 'webhook', recipient ? [recipient] : []);
if (result.status === 'refused') {
const [status, error] = REFUSALS[result.reason];
@@ -89,19 +105,13 @@ export function signWebhookMail(secret: string, timestamp: string, recipient: st
return createHmac('sha256', secret).update(`${timestamp}\n${recipient}\n`).update(raw).digest('hex');
}
function verifyWebhook(
secret: string,
signed: WebhookSignedParts,
recipient: string,
raw: Buffer,
now: number,
): boolean {
const { timestamp, signature } = signed;
if (!timestamp || !/^\d{1,12}$/.test(timestamp) || Math.abs(now - Number(timestamp) * 1000) > WINDOW_MS) {
return false;
}
const given = (signature ?? '').trim().replace(/^sha256=/i, '').toLowerCase();
if (!/^[0-9a-f]{64}$/.test(given) || recipient.length > 320) return false;
const expected = signWebhookMail(secret, timestamp, recipient, raw);
return timingSafeEqual(Buffer.from(given, 'hex'), Buffer.from(expected, 'hex'));
/** The signature over a request whose headers were found plausible: compared in constant time. */
function verifyWebhook(secret: string, signed: WebhookSignedParts, recipient: string, raw: Buffer): boolean {
const expected = signWebhookMail(secret, signed.timestamp ?? '', recipient, raw);
return timingSafeEqual(Buffer.from(signatureOf(signed), 'hex'), Buffer.from(expected, 'hex'));
}
/** The hex signature as sent, with or without its `sha256=` prefix. */
function signatureOf(signed: WebhookSignedParts): string {
return (signed.signature ?? '').trim().replace(/^sha256=/i, '').toLowerCase();
}
@@ -9,6 +9,7 @@ import {
ok,
type McpContext,
} from '../../nest-mcp';
import { DomainError } from '../common/domain-error';
import { MailIngestActionsService } from './mail-ingest-actions.service';
import { MailIngestInboxService } from './mail-ingest-inbox.service';
import {
@@ -65,7 +66,7 @@ export class MailIngestMcp {
@Tool({
name: 'move_mail_booking',
description:
'Move a booking that was added from a mail into another trip (trip_id), while it can still be undone. The booking is written there anew and removed from the first trip. One whose cost was settled with, or that was edited since, is refused with needs_confirmation until confirm is true; the edit is lost.',
'Move a booking that was added from a mail into another trip (trip_id), while it can still be undone; only the person who forwarded the mail can move it. The booking is written there anew and removed from the first trip. One whose cost was settled with, or that was edited since, is refused with needs_confirmation until confirm is true; the edit is lost.',
inputSchema: { candidateId, ...mailIngestMoveRequestSchema.shape },
annotations: TOOL_ANNOTATIONS_NON_IDEMPOTENT,
access: { group: 'reservations', mode: 'write' },
@@ -116,12 +117,15 @@ export class MailIngestMcp {
name: 'update_mail_import_settings',
description:
"Change the user's own switches of the mail import: enabled (whether their address takes mail), ai_enabled (whether an AI reads their mails) and auto_add (add a booking that passes every check on its own, or ask first). Name at least one.",
inputSchema: mailIngestPrefsUpdateRequestSchema,
// The shape, registered strict like every other tool's; its "at least one" check runs below.
inputSchema: mailIngestPrefsUpdateRequestSchema.shape,
annotations: TOOL_ANNOTATIONS_WRITE,
access: { group: 'reservations', mode: 'write' },
})
async updateMailImportSettings(update: MailIngestPrefsUpdateRequest, ctx: McpContext) {
const { available, prefs, ai_available, pending } = await this.inbox.updatePrefs(ctx.userId, update);
const checked = mailIngestPrefsUpdateRequestSchema.safeParse(update);
if (!checked.success) throw new DomainError(400, 'Name at least one setting');
const { available, prefs, ai_available, pending } = await this.inbox.updatePrefs(ctx.userId, checked.data);
return ok({ available, prefs, ai_available, pending });
}
@@ -9,6 +9,7 @@ import { AppConfigModule } from '../app-config/app-config.module';
import { BookingImportModule } from '../booking-import/booking-import.module';
import { BudgetModule } from '../budget/budget.module';
import { DemoModule } from '../common/demo.module';
import { RateLimitModule } from '../common/rate-limit.module';
import { FilesModule } from '../files/files.module';
import { LlmParseModule } from '../llm-parse/llm-parse.module';
import { MapsModule } from '../maps/maps.module';
@@ -72,6 +73,7 @@ import { Module } from '@nestjs/common';
BookingImportModule,
LlmParseModule,
SchedulingModule,
RateLimitModule,
MikroOrmModule.forFeature([
MailIngestAddresses,
MailIngestPrefs,
+15 -2
View File
@@ -1128,18 +1128,20 @@ describe('updateAppSettings', () => {
await deleteRows(orm, AppSettings, { key: { $in: ['smtp_pass', 'notification_channels'] } });
});
const HOOK_SECRET = 'h'.repeat(40);
it('AUTH-DB-075b: the mail import secrets are stored encrypted and never reach the audit row', async () => {
const { user } = createAdmin(testDb);
const result = await asLegacyResult(
svc.updateAppSettings(user.id, {
mail_ingest_imap_host: 'imap.example.com',
mail_ingest_imap_pass: 'box-secret',
mail_ingest_webhook_secret: 'hook-secret',
mail_ingest_webhook_secret: HOOK_SECRET,
}),
);
expect(result.success).toBe(true);
expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('box-secret');
expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('hook-secret');
expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith(HOOK_SECRET);
expect(result.auditDebugDetails).toEqual({
mail_ingest_imap_host: 'imap.example.com',
mail_ingest_imap_pass: '***',
@@ -1149,6 +1151,17 @@ describe('updateAppSettings', () => {
const keys = ['mail_ingest_imap_host', 'mail_ingest_imap_pass', 'mail_ingest_webhook_secret'];
await deleteRows(orm, AppSettings, { key: { $in: keys } });
});
it('AUTH-DB-075c: a webhook secret shorter than 32 characters is refused, and nothing of the form lands', async () => {
const { user } = createAdmin(testDb);
await expect(
svc.updateAppSettings(user.id, { mail_ingest_imap_host: 'imap.example.com', mail_ingest_webhook_secret: 'abc' }),
).rejects.toThrow('The webhook secret needs at least 32 characters');
expect(await readAppSetting(orm, 'mail_ingest_imap_host')).toBeNull();
const cleared = await asLegacyResult(svc.updateAppSettings(user.id, { mail_ingest_webhook_secret: '' }));
expect(cleared.success).toBe(true);
await deleteRows(orm, AppSettings, { key: 'mail_ingest_webhook_secret' });
});
});
describe('MFA success flows', () => {
@@ -8,6 +8,7 @@
import type { MailIngestCandidateRow } from '../../../../src/db/repositories/MailIngestCandidates.repository';
import type { McpContext } from '../../../../src/nest-mcp';
import { DomainError } from '../../../../src/nest/common/domain-error';
import type { RateLimitService } from '../../../../src/nest/common/rate-limit.service';
import type { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service';
import { MailIngestAdminController } from '../../../../src/nest/mail-ingest/mail-ingest-admin.controller';
import { MailIngestInboundController } from '../../../../src/nest/mail-ingest/mail-ingest-inbound.controller';
@@ -189,6 +190,7 @@ describe('acting on an entry, over REST and MCP alike', () => {
pending: 0,
});
expect(JSON.stringify(answer)).not.toContain('secrettoken');
await expect(mcp().updateMailImportSettings({}, ctx)).rejects.toThrow('Name at least one setting');
});
it('list_mail_inbox reads the same page the REST inbox reads', async () => {
@@ -250,9 +252,15 @@ describe('POST /api/mail-ingest/inbound', () => {
const webhook = {
secret: vi.fn(),
maxBytes: vi.fn(() => 10),
plausiblySigned: vi.fn(() => true),
receive: vi.fn(),
};
const controller = () => new MailIngestInboundController(webhook as unknown as MailIngestWebhookService);
const rateLimit = { check: vi.fn(async () => true) };
const controller = () =>
new MailIngestInboundController(
webhook as unknown as MailIngestWebhookService,
rateLimit as unknown as RateLimitService,
);
/** A request whose body arrives as a stream, the way Express hands an unparsed one on. */
function request(body: string, headers: Record<string, string>): Request {
@@ -268,7 +276,9 @@ describe('POST /api/mail-ingest/inbound', () => {
beforeEach(() => {
webhook.secret.mockResolvedValue('s3cret');
webhook.plausiblySigned.mockReturnValue(true);
webhook.receive.mockResolvedValue({ accepted: true, duplicate: false });
rateLimit.check.mockResolvedValue(true);
});
it('reads the raw mail and hands it with the signed headers to the webhook service', async () => {
@@ -300,4 +310,38 @@ describe('POST /api/mail-ingest/inbound', () => {
expect((await thrown(() => controller().receive(declared))).status).toBe(413);
expect(webhook.receive).not.toHaveBeenCalled();
});
it('refuses before reading the body: headers that cannot be signed right, and an address over its share', async () => {
const mail = () => request('Subject: x', { 'content-type': 'message/rfc822' });
webhook.plausiblySigned.mockReturnValueOnce(false);
expect(await thrown(() => controller().receive(mail()))).toEqual({
status: 401,
body: { error: 'The signature does not match' },
});
expect(rateLimit.check).not.toHaveBeenCalled();
rateLimit.check.mockResolvedValueOnce(false);
expect((await thrown(() => controller().receive(mail()))).status).toBe(429);
expect(rateLimit.check).toHaveBeenCalledWith('mail-ingest-webhook', 'unknown', 120, 60_000, expect.any(Number));
expect(webhook.receive).not.toHaveBeenCalled();
});
it('reads only a few bodies at once, and tells the next sender to come back', async () => {
const inbound = controller();
const slow = () => {
const req = new PassThrough();
Object.assign(req, { get: () => undefined, is: (type: string) => type });
return req;
};
const held = [slow(), slow(), slow(), slow()];
const pending = held.map((req) => inbound.receive(req as unknown as Request));
await new Promise((resolve) => setImmediate(resolve));
const fifth = await thrown(() => inbound.receive(request('Subject: x', { 'content-type': 'message/rfc822' })));
expect(fifth).toEqual({ status: 429, body: { error: 'Too many mails at once; try again shortly' } });
held.forEach((req) => req.end('x'));
await Promise.all(pending);
expect(await inbound.receive(request('Subject: x', { 'content-type': 'message/rfc822' }))).toEqual({
accepted: true,
duplicate: false,
});
});
});