fix(mail-ingest): read the Cloud inbox as it is sent, and keep mail refused for now

The relay job reads the control plane's { messages, more, next } page, checks
each mail on its own, pages on while more wait and is idle on a 404. A mail
for several addresses is stored for each, and a passing refusal (inbox full,
daily share taken) leaves it with the relay or in IMAP's TREK/deferred.
This commit is contained in:
Maurice
2026-11-13 16:59:04 +01:00
parent 011cc9f11f
commit e7db110417
9 changed files with 445 additions and 150 deletions
@@ -3,7 +3,6 @@ import type {
MailIngestAddressesRepository,
MailIngestAddressRow,
} from '../../db/repositories/MailIngestAddresses.repository';
import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository';
import { dbNow, utcSuffix } from '../../db/types';
import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto';
import { DomainError } from '../common/domain-error';
@@ -20,12 +19,6 @@ import { createHash, randomBytes } from 'node:crypto';
/** RFC 4648 base32, lower case: a mail's local part keeps it as it is. */
const BASE32 = 'abcdefghijklmnopqrstuvwxyz234567';
/** A live address and the person it belongs to, as a mail that reached it is filed under. */
interface ResolvedAddress {
address: MailIngestAddressRow;
person: MailIngestPersonRow;
}
/**
* The forwarding addresses. A person has one of their own and may make one
* per trip they can add bookings to; each address is a random token behind
@@ -75,20 +68,20 @@ export class MailIngestAddressService {
}
/**
* The live address a mail was sent to, from the recipients in the order the
* receiving server names them (Delivered-To, X-Original-To, To). Null for a
* token nobody holds, a revoked one, a person who switched the import off,
* and a trip address whose owner may no longer add bookings to the trip.
* Every live address a mail was sent to, each once: a confirmation two
* co-travellers forward together reaches both. A token nobody holds, a
* revoked one, a person who switched the import off, and a trip address
* whose owner may no longer add bookings to the trip are left out.
*/
async resolve(recipients: readonly string[]): Promise<ResolvedAddress | null> {
async resolveAll(recipients: readonly string[]): Promise<MailIngestAddressRow[]> {
const template = await this.settings.addressTemplate();
const tokens = recipients
.map((recipient) => tokenOf(recipient, template))
.filter((token): token is string => token !== null);
const found = await Promise.all(tokens.map((token) => this.addresses.findByTokenHash(hashToken(token))));
const address = found.find((row): row is MailIngestAddressRow => row !== null);
if (!address || !(await this.isLive(address))) return null;
return this.withPerson(address);
const tokens = new Set(
recipients.map((recipient) => tokenOf(recipient, template)).filter((token): token is string => token !== null),
);
const found = await Promise.all([...tokens].map((token) => this.addresses.findByTokenHash(hashToken(token))));
const rows = found.filter((row): row is MailIngestAddressRow => row !== null);
const live = await Promise.all(rows.map((row) => this.isLive(row)));
return rows.filter((_, i) => live[i]);
}
/** The person's own live address, made now when they have none: an uploaded mail goes through it. */
@@ -100,9 +93,9 @@ export class MailIngestAddressService {
async isLive(address: MailIngestAddressRow): Promise<boolean> {
if (address.revoked_at) return false;
if (!(await this.settings.prefsOf(address.user_id)).enabled) return false;
if (address.trip_id === null) return true;
const person = await this.trips.person(address.user_id);
return person !== null && (await this.trips.editableTrip(person, address.trip_id)) !== null;
if (person === null) return false;
return address.trip_id === null || (await this.trips.editableTrip(person, address.trip_id)) !== null;
}
async findById(id: number): Promise<MailIngestAddressRow | null> {
@@ -114,11 +107,6 @@ export class MailIngestAddressService {
await this.addresses.touchUsed(addressId, dbNow());
}
private async withPerson(address: MailIngestAddressRow): Promise<ResolvedAddress | null> {
const person = await this.trips.person(address.user_id);
return person ? { address, person } : null;
}
private async rotate(userId: number, tripId: number | null): Promise<MailIngestAddressRow> {
const token = newToken();
const id = await this.uow.transactional(async () => {
@@ -9,10 +9,11 @@ import { isIP } from 'node:net';
const CONNECT_TIMEOUT_MS = 15_000;
const SOCKET_TIMEOUT_MS = 60_000;
/** A mail waiting in the box: its uid and its size, before anything of it is downloaded. */
/** A mail waiting in the box: its uid, its size and when the box got it, before anything of it is downloaded. */
export interface ImapMailRef {
uid: number;
size: number;
receivedAt: Date | null;
}
/** What the import asks of an IMAP box, one connection per run. */
@@ -84,8 +85,11 @@ class ImapFlowSession implements ImapSession {
if (this.exists === 0) return [];
const found: ImapMailRef[] = [];
// `n:*` always names the newest mail, even below n, so the range alone does not filter.
for await (const message of this.client.fetch(`${lastUid + 1}:*`, { uid: true, size: true }, { uid: true })) {
if (message.uid > lastUid) found.push({ uid: message.uid, size: message.size ?? 0 });
const query = { uid: true, size: true, internalDate: true };
for await (const message of this.client.fetch(`${lastUid + 1}:*`, query, { uid: true })) {
if (message.uid > lastUid) {
found.push({ uid: message.uid, size: message.size ?? 0, receivedAt: dateOf(message.internalDate) });
}
}
return found.sort((a, b) => a.uid - b.uid);
}
@@ -115,6 +119,13 @@ class ImapFlowSession implements ImapSession {
}
}
/** The server's arrival time of a mail, which imapflow hands over as a Date or as the server wrote it. */
function dateOf(value: Date | string | undefined): Date | null {
if (value === undefined) return null;
const date = value instanceof Date ? value : new Date(value);
return Number.isNaN(date.getTime()) ? null : date;
}
/** What an admin reads when the box cannot be read: the reason, never the password. */
export function describeImapFailure(err: unknown): string {
if (err instanceof SsrfBlockedError) return err.message;
@@ -9,7 +9,7 @@ import {
type ImapMailRef,
type ImapSession,
} from './mail-ingest-imap.client';
import { MailIngestIntakeService } from './mail-ingest-intake.service';
import { isPassingRefusal, MailIngestIntakeService } from './mail-ingest-intake.service';
import { MailIngestSettingsService, type MailIngestImapConfig } from './mail-ingest-settings.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
@@ -18,16 +18,30 @@ import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
const LEASE_MS = 10 * 60 * 1000;
/** The mails one run takes from the box; the rest wait for the next run. */
const MAILS_PER_RUN = 20;
/** The mails one run offers the intake again from the deferred folder, oldest first. */
const DEFERRED_PER_RUN = 5;
/** How long a deferred mail is offered again before it is given up. */
const DEFER_DAYS = 3;
const DAY_MS = 24 * 60 * 60 * 1000;
/** Where read mails go: kept, never deleted, out of the folder the next run reads. */
const PROCESSED_FOLDER = ['TREK', 'processed'] as const;
/** Where a mail the import cannot take for now waits to be offered again. */
const DEFERRED_FOLDER = ['TREK', 'deferred'] as const;
/** What became of one mail: taken in, done with otherwise, or kept to be offered again. */
type ImapOutcome = 'taken' | 'done' | 'deferred';
/**
* Reads the admin's IMAP box every two minutes, with a client of its own for
* every run. Each new mail is handed to the intake, which stores it once,
* and then moved to TREK/processed. The cursor (uid validity, last uid)
* keeps a mail the move missed from being read twice, and the intake's
* dedupe keeps it so when the server renumbers the box and the run starts
* over. One process holds the box at a time, by the cursor's lease.
* and then moved to TREK/processed. A mail the import cannot take for now
* (an address's share for the day taken, an inbox full) goes to
* TREK/deferred instead, so it holds up no mail behind it, and a few of
* those are offered again on every run until they are taken or
* {@link DEFER_DAYS} old. The cursor (uid validity, last uid) keeps a mail
* a move missed from being read twice, and the intake's dedupe keeps it so
* when the server renumbers the box and the run starts over. One process
* holds the box at a time, by the cursor's lease.
*/
@Injectable()
export class MailIngestImapJob implements OnApplicationBootstrap {
@@ -45,8 +59,8 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
}
/**
* One run; answers how many mails it took from the box. Nothing happens
* unless the import is on, IMAP is its transport and the box is complete.
* One run; answers how many mails it took in. Nothing happens unless the
* import is on, IMAP is its transport and the box is complete.
*
* @txIndependent each mail is stored on its own as it is read; the cursor
* notes where the run got to once the box is done with.
@@ -58,7 +72,7 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
const until = dbNow(new Date(now.getTime() + LEASE_MS));
if (!(await this.cursor.acquireLease('imap', config.mailbox, LEASE_OWNER, until, dbNow(now)))) return 0;
try {
const { taken, uidValidity, lastUid } = await this.poll(config);
const { taken, uidValidity, lastUid } = await this.poll(config, now);
await this.cursor.recordOk('imap', config.mailbox, dbNow(new Date()), { uidValidity, lastUid });
return taken;
} catch (err) {
@@ -71,7 +85,10 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
}
}
private async poll(config: MailIngestImapConfig): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
private async poll(
config: MailIngestImapConfig,
now: Date,
): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
const session = await this.imap.connect(config);
try {
const { uidValidity } = await session.open(config.mailbox);
@@ -79,28 +96,71 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
// A changed uid validity means the server renumbered the box: start over, the dedupe keeps each mail once.
const after = position?.uid_validity === uidValidity ? (position.last_uid ?? 0) : 0;
const due = (await session.listAfter(after)).slice(0, MAILS_PER_RUN);
if (due.length === 0) return { taken: 0, uidValidity, lastUid: after };
const folder = await session.ensureFolder(PROCESSED_FOLDER);
await due.reduce(async (previous, mail) => {
await previous;
await this.takeIn(session, mail);
}, Promise.resolve());
await session.move(due.map((mail) => mail.uid), folder);
return { taken: due.length, uidValidity, lastUid: Math.max(...due.map((mail) => mail.uid)) };
const fresh = due.length > 0 ? await this.readNew(session, due) : 0;
const lastUid = due.length > 0 ? Math.max(...due.map((mail) => mail.uid)) : after;
const retried = await this.retryDeferred(session, now);
return { taken: fresh + retried, uidValidity, lastUid };
} finally {
await session.close();
}
}
/** The new mails of the box into the intake, each moved on by what became of it; answers how many were taken. */
private async readNew(session: ImapSession, due: readonly ImapMailRef[]): Promise<number> {
const outcomes = await this.takeAll(session, due);
const deferred = due.filter((_, i) => outcomes[i] === 'deferred').map((mail) => mail.uid);
const settled = due.filter((_, i) => outcomes[i] !== 'deferred').map((mail) => mail.uid);
if (settled.length > 0) await session.move(settled, await session.ensureFolder(PROCESSED_FOLDER));
if (deferred.length > 0) await session.move(deferred, await session.ensureFolder(DEFERRED_FOLDER));
return outcomes.filter((outcome) => outcome === 'taken').length;
}
/**
* Offers the oldest deferred mails again. One taken now, or done with
* otherwise, moves on to TREK/processed; one still refused stays, unless
* it waited {@link DEFER_DAYS} already, and is then given up the same way.
*/
private async retryDeferred(session: ImapSession, now: Date): Promise<number> {
await session.open(await session.ensureFolder(DEFERRED_FOLDER));
const due = (await session.listAfter(0)).slice(0, DEFERRED_PER_RUN);
if (due.length === 0) return 0;
const outcomes = await this.takeAll(session, due);
// A mail whose arrival the server does not name cannot be aged, so it gets this one retry only.
const expired = (mail: ImapMailRef) =>
!mail.receivedAt || now.getTime() - mail.receivedAt.getTime() > DEFER_DAYS * DAY_MS;
const settled = due.filter((mail, i) => outcomes[i] !== 'deferred' || expired(mail));
const givenUp = due.filter((mail, i) => outcomes[i] === 'deferred' && expired(mail)).length;
if (givenUp > 0) {
logWarn(`[mail-ingest] Deferred IMAP mails given up after waiting ${DEFER_DAYS} days: ${givenUp}`);
}
if (settled.length > 0) {
await session.move(
settled.map((mail) => mail.uid),
await session.ensureFolder(PROCESSED_FOLDER),
);
}
return outcomes.filter((outcome) => outcome === 'taken').length;
}
private takeAll(session: ImapSession, due: readonly ImapMailRef[]): Promise<ImapOutcome[]> {
return due.reduce<Promise<ImapOutcome[]>>(async (previous, mail) => {
const done = await previous;
return [...done, await this.takeIn(session, mail)];
}, Promise.resolve([]));
}
/** One mail into the intake. One larger than the import takes is left unread, and moved on with the rest. */
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<void> {
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<ImapOutcome> {
if (mail.size > this.settings.maxMailBytes()) {
logWarn(`[mail-ingest] An IMAP mail of ${mail.size} bytes is larger than the import takes; left unread`);
return;
return 'done';
}
const raw = await session.source(mail.uid);
if (!raw) return;
if (!raw) return 'done';
const result = await this.intake.receive(raw, 'imap');
if (result.status === 'refused') logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
if (result.status !== 'refused') return result.status === 'accepted' ? 'taken' : 'done';
if (isPassingRefusal(result.reason)) return 'deferred';
logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
return 'done';
}
}
@@ -5,6 +5,7 @@ import type { MailIngestCandidatesRepository } from '../../db/repositories/MailI
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
import { dbNow } from '../../db/types';
import { MailReadService } from '../llm-parse/mail-read.service';
import type { WalkedMail } from '../llm-parse/mail-read.types';
import { MailIngestAddressService } from './mail-ingest-address.service';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { sealMail } from './mail-seal';
@@ -24,10 +25,24 @@ export type MailIntakeRefusal =
| 'address_daily_limit'
| 'inbox_full';
/**
* Refusals that pass: the import is switched off for now, the address took
* its share for the day, or its person's inbox is full. A transport keeps
* such a mail and offers it again later instead of dropping it.
*/
const PASSING_REFUSALS: ReadonlySet<MailIntakeRefusal> = new Set(['unavailable', 'address_daily_limit', 'inbox_full']);
export function isPassingRefusal(reason: MailIntakeRefusal): boolean {
return PASSING_REFUSALS.has(reason);
}
type MailIntakeResult =
| { status: 'accepted' | 'duplicate'; messageId: number }
| { status: 'refused'; reason: MailIntakeRefusal };
/** A received mail's outcome over every address it reached. */
type MailReceiveResult = { status: 'accepted' | 'duplicate' } | { status: 'refused'; reason: MailIntakeRefusal };
const DAY_MS = 24 * 60 * 60 * 1000;
/**
@@ -48,26 +63,32 @@ export class MailIngestIntakeService {
) {}
/**
* A mail a transport received. `envelopeRecipients` are the RCPT TO
* addresses where the transport knows them; otherwise the headers name the
* address, in the order the receiving server adds them.
* A mail a transport received, stored once for every live address it was
* sent to. `envelopeRecipients` are the RCPT TO addresses where the
* transport knows them, and then the only ones read; otherwise the delivery
* header the receiving server put on top names the address.
*
* When one of its addresses refuses it for now, the whole mail answers that
* refusal: the transport offers it again, and the copies stored already
* come back as duplicates.
*/
async receive(
raw: Buffer,
transport: Exclude<MailIngestTransport, 'upload'>,
envelopeRecipients: readonly string[] = [],
): Promise<MailIntakeResult> {
): Promise<MailReceiveResult> {
const refused = await this.precheck(raw);
if (refused) return { status: 'refused', reason: refused };
const mail = this.reader.walk(raw);
if (!mail) return { status: 'refused', reason: 'not_a_mail' };
const { deliveredTo, xOriginalTo, to } = mail.envelope;
const headerRecipients = [deliveredTo, xOriginalTo, ...(to ?? '').split(',')].filter(
(value): value is string => !!value && value.trim() !== '',
);
const resolved = await this.addresses.resolve([...envelopeRecipients, ...headerRecipients]);
if (!resolved) return { status: 'refused', reason: 'unknown_address' };
return this.store(raw, transport, resolved.address, mail.envelope.messageId);
const recipients = envelopeRecipients.length > 0 ? envelopeRecipients : headerRecipients(mail.envelope);
const addresses = await this.addresses.resolveAll(recipients);
if (addresses.length === 0) return { status: 'refused', reason: 'unknown_address' };
const results = await addresses.reduce<Promise<MailIntakeResult[]>>(async (previous, address) => {
const done = await previous;
return [...done, await this.store(raw, transport, address, mail.envelope.messageId)];
}, Promise.resolve([]));
return combined(results);
}
/** An .eml the person uploaded, read as if they had forwarded it to their own address. */
@@ -126,3 +147,25 @@ export class MailIngestIntakeService {
return { status: 'accepted', messageId: id };
}
}
/**
* The addresses the headers name when the transport knows no envelope: the
* topmost Delivered-To, else the topmost X-Original-To, which the receiving
* server adds above whatever the mail brought along, and only without either
* the To line. A sender who names another token there gains nothing over
* sending to that token directly.
*/
function headerRecipients(envelope: WalkedMail['envelope']): string[] {
const delivered = [envelope.deliveredTo, envelope.xOriginalTo].find((value) => !!value?.trim());
if (delivered) return [delivered];
return (envelope.to ?? '').split(',').filter((value) => value.trim() !== '');
}
/** One answer for a mail stored per address: a passing refusal first, then accepted, then duplicate. */
function combined(results: readonly MailIntakeResult[]): MailReceiveResult {
const passing = results.find((result) => result.status === 'refused' && isPassingRefusal(result.reason));
if (passing) return passing;
if (results.some((result) => result.status === 'accepted')) return { status: 'accepted' };
if (results.some((result) => result.status === 'duplicate')) return { status: 'duplicate' };
return results[0] ?? { status: 'refused', reason: 'unknown_address' };
}
@@ -3,7 +3,7 @@ import type { MailIngestCursorRepository } from '../../db/repositories/MailInges
import { dbNow } from '../../db/types';
import { logError, logWarn } from '../audit/audit-log.logger';
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
import { MailIngestIntakeService } from './mail-ingest-intake.service';
import { isPassingRefusal, MailIngestIntakeService } from './mail-ingest-intake.service';
import { MailIngestRelayHost, type MailInboxRelay } from './mail-ingest-relay.service';
import { MailIngestSettingsService } from './mail-ingest-settings.service';
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
@@ -15,35 +15,51 @@ import { z } from 'zod';
/** How long a run holds the relay before another process may take it over. */
const LEASE_MS = 10 * 60 * 1000;
/** The mails one run asks the relay for. */
/** The mails one page of the relay's queue holds, and the pages one run reads. */
const PAGE = 10;
const PAGES_PER_RUN = 5;
/** The most recipients one mail names on this instance; the relay sends no more. */
const MAX_RELAYED_RECIPIENTS = 100;
/** The base64 length of the largest mail the relay hands on. */
const MAX_RAW_CHARS = Math.ceil(MAIL_INGEST_LIMITS.cloudMailBytes / 3) * 4;
/** The relay's mail ids are UUIDs. */
const MAIL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
/** The relay's answer, checked before anything of it is used. */
const relayInboxSchema = z.object({
mails: z
.array(
z.object({
id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/),
/** The addresses it was sent to, as the relay's mail server took them (RCPT TO). */
recipients: z.array(z.string().min(3).max(320)).min(1).max(5),
sha256: z.string().regex(/^[0-9a-f]{64}$/),
/** The mail itself, base64. */
raw: z.string().max(MAX_RAW_CHARS),
}),
)
.max(PAGE),
/** One page of the relay's queue; each mail in it is checked on its own, so an unreadable one blocks no other. */
const relayPageSchema = z.object({
messages: z.array(z.unknown()).max(PAGE),
/** Further mails wait beyond `next`. */
more: z.boolean(),
/** Where the next page starts: after the last mail the relay looked at for this one. */
next: z
.string()
.regex(/^[A-Za-z0-9_-]{1,200}$/)
.nullable(),
});
type RelayedMail = z.infer<typeof relayInboxSchema>['mails'][number];
const relayedMailSchema = z.object({
id: z.string().regex(MAIL_ID),
receivedAt: z.string().max(40),
size: z.number().int().nonnegative(),
sha256: z.string().regex(/^[0-9a-f]{64}$/),
/** The addresses on this instance it was sent to, as the relay's mail server took them (RCPT TO). */
recipients: z.array(z.string().min(3).max(320)).min(1).max(MAX_RELAYED_RECIPIENTS),
/** The mail itself, base64. */
raw: z.string().max(MAX_RAW_CHARS),
});
type RelayedMail = z.infer<typeof relayedMailSchema>;
const relayedIdSchema = z.object({ id: z.string().regex(MAIL_ID) });
/**
* Collects this instance's forwarded mails from the TREK Cloud relay every
* two minutes: a page of at most ten, each checked against its hash, handed
* to the intake under the address it was sent to, and acknowledged once it
* is in (or once it is clear it never will be), so the relay drops it. A
* mail that does not arrive whole is left for the next run. Idle wherever
* no relay is attached, which is every self-hosted install.
* two minutes, page after page while more wait, up to five pages a run. Each
* mail is checked against its hash, handed to the intake under the addresses
* it was sent to, and acknowledged once it is in (or once it is clear it
* never will be), so the relay drops it. A mail that did not arrive whole,
* or that the import cannot take for now (switched off, an address's share
* for the day taken, an inbox full), stays with the relay for a later run,
* and the pages after it are still read. Idle wherever no relay is attached,
* which is every self-hosted install, and where the control plane runs no
* import for this instance.
*/
@Injectable()
export class MailIngestRelayJob implements OnApplicationBootstrap {
@@ -71,12 +87,7 @@ export class MailIngestRelayJob implements OnApplicationBootstrap {
const until = dbNow(new Date(now.getTime() + LEASE_MS));
if (!(await this.cursor.acquireLease('cloud', '', LEASE_OWNER, until, dbNow(now)))) return 0;
try {
const page = relayInboxSchema.safeParse(await relay.list(PAGE));
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
const taken = await page.data.mails.reduce(async (previous, mail) => {
const count = await previous;
return count + ((await this.takeIn(relay, mail)) ? 1 : 0);
}, Promise.resolve(0));
const taken = await this.drain(relay, null, PAGES_PER_RUN);
await this.cursor.recordOk('cloud', '', dbNow(new Date()));
return taken;
} catch (err) {
@@ -89,16 +100,43 @@ export class MailIngestRelayJob implements OnApplicationBootstrap {
}
}
/** One relayed mail into the intake; answers whether it was taken in. */
private async takeIn(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
/** The pages from `after` on, as many as are left to the run; answers the mails taken in. */
private async drain(relay: MailInboxRelay, after: string | null, pagesLeft: number): Promise<number> {
const answer = await relay.list(PAGE, after);
if (answer === null) return 0;
const page = relayPageSchema.safeParse(answer);
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
const taken = await page.data.messages.reduce(async (previous, item) => {
const count = await previous;
return count + ((await this.takeIn(relay, item)) ? 1 : 0);
}, Promise.resolve(0));
const { more, next } = page.data;
if (!more || next === null || next === after || pagesLeft <= 1) return taken;
return taken + (await this.drain(relay, next, pagesLeft - 1));
}
/** One mail of a page into the intake; answers whether it was taken in. */
private async takeIn(relay: MailInboxRelay, item: unknown): Promise<boolean> {
const parsed = relayedMailSchema.safeParse(item);
if (parsed.success) return this.store(relay, parsed.data);
// It would read the same on every run: dropped, where it names an id to drop it by.
const id = relayedIdSchema.safeParse(item);
logWarn(`[mail-ingest] The relay sent a mail this server cannot read${id.success ? `: ${id.data.id}` : ''}`);
if (id.success) await relay.ack(id.data.id);
return false;
}
private async store(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
const raw = Buffer.from(mail.raw, 'base64');
if (createHash('sha256').update(raw).digest('hex') !== mail.sha256) {
logWarn(`[mail-ingest] Relayed mail ${mail.id} did not arrive whole; left for the next run`);
return false;
}
const result = await this.intake.receive(raw, 'cloud', mail.recipients);
// Switched off in between: the relay keeps the mail until the import is back or the mail expires.
if (result.status === 'refused' && result.reason === 'unavailable') return false;
if (result.status === 'refused' && isPassingRefusal(result.reason)) {
logWarn(`[mail-ingest] Relayed mail ${mail.id} waits with the relay: ${result.reason}`);
return false;
}
if (result.status === 'refused') {
logWarn(`[mail-ingest] Relayed mail ${mail.id} was not taken in: ${result.reason}`);
}
@@ -7,8 +7,13 @@ import { Injectable } from '@nestjs/common';
* encrypted, and hands it over here; nothing of a person reaches it.
*/
export interface MailInboxRelay {
/** `GET /v1/instance/inbox?limit=n`: the waiting mails, the answer's JSON as parsed, checked by the caller. */
list(limit: number): Promise<unknown>;
/**
* `GET /v1/instance/inbox?limit=n&after=<cursor>`: the waiting mails after
* the cursor a previous page named (from the oldest without one), the
* answer's JSON as parsed, checked by the caller. Null when the control
* plane has no import for this instance (its 404), which is no failure.
*/
list(limit: number, after: string | null): Promise<unknown>;
/** `POST /v1/instance/inbox/:id/ack`: the mail is taken in (or never will be), and the relay drops it. */
ack(id: string): Promise<void>;
}