mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 19:40:34 +00:00
fix(mail-ingest): read the Cloud inbox as it is sent, and keep mail refused for now
The relay job reads the control plane's { messages, more, next } page, checks
each mail on its own, pages on while more wait and is idle on a 404. A mail
for several addresses is stored for each, and a passing refusal (inbox full,
daily share taken) leaves it with the relay or in IMAP's TREK/deferred.
This commit is contained in:
@@ -3,7 +3,6 @@ import type {
|
||||
MailIngestAddressesRepository,
|
||||
MailIngestAddressRow,
|
||||
} from '../../db/repositories/MailIngestAddresses.repository';
|
||||
import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import { dbNow, utcSuffix } from '../../db/types';
|
||||
import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
@@ -20,12 +19,6 @@ import { createHash, randomBytes } from 'node:crypto';
|
||||
/** RFC 4648 base32, lower case: a mail's local part keeps it as it is. */
|
||||
const BASE32 = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
|
||||
/** A live address and the person it belongs to, as a mail that reached it is filed under. */
|
||||
interface ResolvedAddress {
|
||||
address: MailIngestAddressRow;
|
||||
person: MailIngestPersonRow;
|
||||
}
|
||||
|
||||
/**
|
||||
* The forwarding addresses. A person has one of their own and may make one
|
||||
* per trip they can add bookings to; each address is a random token behind
|
||||
@@ -75,20 +68,20 @@ export class MailIngestAddressService {
|
||||
}
|
||||
|
||||
/**
|
||||
* The live address a mail was sent to, from the recipients in the order the
|
||||
* receiving server names them (Delivered-To, X-Original-To, To). Null for a
|
||||
* token nobody holds, a revoked one, a person who switched the import off,
|
||||
* and a trip address whose owner may no longer add bookings to the trip.
|
||||
* Every live address a mail was sent to, each once: a confirmation two
|
||||
* co-travellers forward together reaches both. A token nobody holds, a
|
||||
* revoked one, a person who switched the import off, and a trip address
|
||||
* whose owner may no longer add bookings to the trip are left out.
|
||||
*/
|
||||
async resolve(recipients: readonly string[]): Promise<ResolvedAddress | null> {
|
||||
async resolveAll(recipients: readonly string[]): Promise<MailIngestAddressRow[]> {
|
||||
const template = await this.settings.addressTemplate();
|
||||
const tokens = recipients
|
||||
.map((recipient) => tokenOf(recipient, template))
|
||||
.filter((token): token is string => token !== null);
|
||||
const found = await Promise.all(tokens.map((token) => this.addresses.findByTokenHash(hashToken(token))));
|
||||
const address = found.find((row): row is MailIngestAddressRow => row !== null);
|
||||
if (!address || !(await this.isLive(address))) return null;
|
||||
return this.withPerson(address);
|
||||
const tokens = new Set(
|
||||
recipients.map((recipient) => tokenOf(recipient, template)).filter((token): token is string => token !== null),
|
||||
);
|
||||
const found = await Promise.all([...tokens].map((token) => this.addresses.findByTokenHash(hashToken(token))));
|
||||
const rows = found.filter((row): row is MailIngestAddressRow => row !== null);
|
||||
const live = await Promise.all(rows.map((row) => this.isLive(row)));
|
||||
return rows.filter((_, i) => live[i]);
|
||||
}
|
||||
|
||||
/** The person's own live address, made now when they have none: an uploaded mail goes through it. */
|
||||
@@ -100,9 +93,9 @@ export class MailIngestAddressService {
|
||||
async isLive(address: MailIngestAddressRow): Promise<boolean> {
|
||||
if (address.revoked_at) return false;
|
||||
if (!(await this.settings.prefsOf(address.user_id)).enabled) return false;
|
||||
if (address.trip_id === null) return true;
|
||||
const person = await this.trips.person(address.user_id);
|
||||
return person !== null && (await this.trips.editableTrip(person, address.trip_id)) !== null;
|
||||
if (person === null) return false;
|
||||
return address.trip_id === null || (await this.trips.editableTrip(person, address.trip_id)) !== null;
|
||||
}
|
||||
|
||||
async findById(id: number): Promise<MailIngestAddressRow | null> {
|
||||
@@ -114,11 +107,6 @@ export class MailIngestAddressService {
|
||||
await this.addresses.touchUsed(addressId, dbNow());
|
||||
}
|
||||
|
||||
private async withPerson(address: MailIngestAddressRow): Promise<ResolvedAddress | null> {
|
||||
const person = await this.trips.person(address.user_id);
|
||||
return person ? { address, person } : null;
|
||||
}
|
||||
|
||||
private async rotate(userId: number, tripId: number | null): Promise<MailIngestAddressRow> {
|
||||
const token = newToken();
|
||||
const id = await this.uow.transactional(async () => {
|
||||
|
||||
@@ -9,10 +9,11 @@ import { isIP } from 'node:net';
|
||||
const CONNECT_TIMEOUT_MS = 15_000;
|
||||
const SOCKET_TIMEOUT_MS = 60_000;
|
||||
|
||||
/** A mail waiting in the box: its uid and its size, before anything of it is downloaded. */
|
||||
/** A mail waiting in the box: its uid, its size and when the box got it, before anything of it is downloaded. */
|
||||
export interface ImapMailRef {
|
||||
uid: number;
|
||||
size: number;
|
||||
receivedAt: Date | null;
|
||||
}
|
||||
|
||||
/** What the import asks of an IMAP box, one connection per run. */
|
||||
@@ -84,8 +85,11 @@ class ImapFlowSession implements ImapSession {
|
||||
if (this.exists === 0) return [];
|
||||
const found: ImapMailRef[] = [];
|
||||
// `n:*` always names the newest mail, even below n, so the range alone does not filter.
|
||||
for await (const message of this.client.fetch(`${lastUid + 1}:*`, { uid: true, size: true }, { uid: true })) {
|
||||
if (message.uid > lastUid) found.push({ uid: message.uid, size: message.size ?? 0 });
|
||||
const query = { uid: true, size: true, internalDate: true };
|
||||
for await (const message of this.client.fetch(`${lastUid + 1}:*`, query, { uid: true })) {
|
||||
if (message.uid > lastUid) {
|
||||
found.push({ uid: message.uid, size: message.size ?? 0, receivedAt: dateOf(message.internalDate) });
|
||||
}
|
||||
}
|
||||
return found.sort((a, b) => a.uid - b.uid);
|
||||
}
|
||||
@@ -115,6 +119,13 @@ class ImapFlowSession implements ImapSession {
|
||||
}
|
||||
}
|
||||
|
||||
/** The server's arrival time of a mail, which imapflow hands over as a Date or as the server wrote it. */
|
||||
function dateOf(value: Date | string | undefined): Date | null {
|
||||
if (value === undefined) return null;
|
||||
const date = value instanceof Date ? value : new Date(value);
|
||||
return Number.isNaN(date.getTime()) ? null : date;
|
||||
}
|
||||
|
||||
/** What an admin reads when the box cannot be read: the reason, never the password. */
|
||||
export function describeImapFailure(err: unknown): string {
|
||||
if (err instanceof SsrfBlockedError) return err.message;
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
type ImapMailRef,
|
||||
type ImapSession,
|
||||
} from './mail-ingest-imap.client';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { isPassingRefusal, MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestSettingsService, type MailIngestImapConfig } from './mail-ingest-settings.service';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
@@ -18,16 +18,30 @@ import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
const LEASE_MS = 10 * 60 * 1000;
|
||||
/** The mails one run takes from the box; the rest wait for the next run. */
|
||||
const MAILS_PER_RUN = 20;
|
||||
/** The mails one run offers the intake again from the deferred folder, oldest first. */
|
||||
const DEFERRED_PER_RUN = 5;
|
||||
/** How long a deferred mail is offered again before it is given up. */
|
||||
const DEFER_DAYS = 3;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
/** Where read mails go: kept, never deleted, out of the folder the next run reads. */
|
||||
const PROCESSED_FOLDER = ['TREK', 'processed'] as const;
|
||||
/** Where a mail the import cannot take for now waits to be offered again. */
|
||||
const DEFERRED_FOLDER = ['TREK', 'deferred'] as const;
|
||||
|
||||
/** What became of one mail: taken in, done with otherwise, or kept to be offered again. */
|
||||
type ImapOutcome = 'taken' | 'done' | 'deferred';
|
||||
|
||||
/**
|
||||
* Reads the admin's IMAP box every two minutes, with a client of its own for
|
||||
* every run. Each new mail is handed to the intake, which stores it once,
|
||||
* and then moved to TREK/processed. The cursor (uid validity, last uid)
|
||||
* keeps a mail the move missed from being read twice, and the intake's
|
||||
* dedupe keeps it so when the server renumbers the box and the run starts
|
||||
* over. One process holds the box at a time, by the cursor's lease.
|
||||
* and then moved to TREK/processed. A mail the import cannot take for now
|
||||
* (an address's share for the day taken, an inbox full) goes to
|
||||
* TREK/deferred instead, so it holds up no mail behind it, and a few of
|
||||
* those are offered again on every run until they are taken or
|
||||
* {@link DEFER_DAYS} old. The cursor (uid validity, last uid) keeps a mail
|
||||
* a move missed from being read twice, and the intake's dedupe keeps it so
|
||||
* when the server renumbers the box and the run starts over. One process
|
||||
* holds the box at a time, by the cursor's lease.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
@@ -45,8 +59,8 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
}
|
||||
|
||||
/**
|
||||
* One run; answers how many mails it took from the box. Nothing happens
|
||||
* unless the import is on, IMAP is its transport and the box is complete.
|
||||
* One run; answers how many mails it took in. Nothing happens unless the
|
||||
* import is on, IMAP is its transport and the box is complete.
|
||||
*
|
||||
* @txIndependent each mail is stored on its own as it is read; the cursor
|
||||
* notes where the run got to once the box is done with.
|
||||
@@ -58,7 +72,7 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
const until = dbNow(new Date(now.getTime() + LEASE_MS));
|
||||
if (!(await this.cursor.acquireLease('imap', config.mailbox, LEASE_OWNER, until, dbNow(now)))) return 0;
|
||||
try {
|
||||
const { taken, uidValidity, lastUid } = await this.poll(config);
|
||||
const { taken, uidValidity, lastUid } = await this.poll(config, now);
|
||||
await this.cursor.recordOk('imap', config.mailbox, dbNow(new Date()), { uidValidity, lastUid });
|
||||
return taken;
|
||||
} catch (err) {
|
||||
@@ -71,7 +85,10 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
}
|
||||
}
|
||||
|
||||
private async poll(config: MailIngestImapConfig): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
|
||||
private async poll(
|
||||
config: MailIngestImapConfig,
|
||||
now: Date,
|
||||
): Promise<{ taken: number; uidValidity: number; lastUid: number }> {
|
||||
const session = await this.imap.connect(config);
|
||||
try {
|
||||
const { uidValidity } = await session.open(config.mailbox);
|
||||
@@ -79,28 +96,71 @@ export class MailIngestImapJob implements OnApplicationBootstrap {
|
||||
// A changed uid validity means the server renumbered the box: start over, the dedupe keeps each mail once.
|
||||
const after = position?.uid_validity === uidValidity ? (position.last_uid ?? 0) : 0;
|
||||
const due = (await session.listAfter(after)).slice(0, MAILS_PER_RUN);
|
||||
if (due.length === 0) return { taken: 0, uidValidity, lastUid: after };
|
||||
const folder = await session.ensureFolder(PROCESSED_FOLDER);
|
||||
await due.reduce(async (previous, mail) => {
|
||||
await previous;
|
||||
await this.takeIn(session, mail);
|
||||
}, Promise.resolve());
|
||||
await session.move(due.map((mail) => mail.uid), folder);
|
||||
return { taken: due.length, uidValidity, lastUid: Math.max(...due.map((mail) => mail.uid)) };
|
||||
const fresh = due.length > 0 ? await this.readNew(session, due) : 0;
|
||||
const lastUid = due.length > 0 ? Math.max(...due.map((mail) => mail.uid)) : after;
|
||||
const retried = await this.retryDeferred(session, now);
|
||||
return { taken: fresh + retried, uidValidity, lastUid };
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** The new mails of the box into the intake, each moved on by what became of it; answers how many were taken. */
|
||||
private async readNew(session: ImapSession, due: readonly ImapMailRef[]): Promise<number> {
|
||||
const outcomes = await this.takeAll(session, due);
|
||||
const deferred = due.filter((_, i) => outcomes[i] === 'deferred').map((mail) => mail.uid);
|
||||
const settled = due.filter((_, i) => outcomes[i] !== 'deferred').map((mail) => mail.uid);
|
||||
if (settled.length > 0) await session.move(settled, await session.ensureFolder(PROCESSED_FOLDER));
|
||||
if (deferred.length > 0) await session.move(deferred, await session.ensureFolder(DEFERRED_FOLDER));
|
||||
return outcomes.filter((outcome) => outcome === 'taken').length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Offers the oldest deferred mails again. One taken now, or done with
|
||||
* otherwise, moves on to TREK/processed; one still refused stays, unless
|
||||
* it waited {@link DEFER_DAYS} already, and is then given up the same way.
|
||||
*/
|
||||
private async retryDeferred(session: ImapSession, now: Date): Promise<number> {
|
||||
await session.open(await session.ensureFolder(DEFERRED_FOLDER));
|
||||
const due = (await session.listAfter(0)).slice(0, DEFERRED_PER_RUN);
|
||||
if (due.length === 0) return 0;
|
||||
const outcomes = await this.takeAll(session, due);
|
||||
// A mail whose arrival the server does not name cannot be aged, so it gets this one retry only.
|
||||
const expired = (mail: ImapMailRef) =>
|
||||
!mail.receivedAt || now.getTime() - mail.receivedAt.getTime() > DEFER_DAYS * DAY_MS;
|
||||
const settled = due.filter((mail, i) => outcomes[i] !== 'deferred' || expired(mail));
|
||||
const givenUp = due.filter((mail, i) => outcomes[i] === 'deferred' && expired(mail)).length;
|
||||
if (givenUp > 0) {
|
||||
logWarn(`[mail-ingest] Deferred IMAP mails given up after waiting ${DEFER_DAYS} days: ${givenUp}`);
|
||||
}
|
||||
if (settled.length > 0) {
|
||||
await session.move(
|
||||
settled.map((mail) => mail.uid),
|
||||
await session.ensureFolder(PROCESSED_FOLDER),
|
||||
);
|
||||
}
|
||||
return outcomes.filter((outcome) => outcome === 'taken').length;
|
||||
}
|
||||
|
||||
private takeAll(session: ImapSession, due: readonly ImapMailRef[]): Promise<ImapOutcome[]> {
|
||||
return due.reduce<Promise<ImapOutcome[]>>(async (previous, mail) => {
|
||||
const done = await previous;
|
||||
return [...done, await this.takeIn(session, mail)];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/** One mail into the intake. One larger than the import takes is left unread, and moved on with the rest. */
|
||||
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<void> {
|
||||
private async takeIn(session: ImapSession, mail: ImapMailRef): Promise<ImapOutcome> {
|
||||
if (mail.size > this.settings.maxMailBytes()) {
|
||||
logWarn(`[mail-ingest] An IMAP mail of ${mail.size} bytes is larger than the import takes; left unread`);
|
||||
return;
|
||||
return 'done';
|
||||
}
|
||||
const raw = await session.source(mail.uid);
|
||||
if (!raw) return;
|
||||
if (!raw) return 'done';
|
||||
const result = await this.intake.receive(raw, 'imap');
|
||||
if (result.status === 'refused') logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
|
||||
if (result.status !== 'refused') return result.status === 'accepted' ? 'taken' : 'done';
|
||||
if (isPassingRefusal(result.reason)) return 'deferred';
|
||||
logWarn(`[mail-ingest] An IMAP mail was not taken in: ${result.reason}`);
|
||||
return 'done';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { MailIngestCandidatesRepository } from '../../db/repositories/MailI
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { MailReadService } from '../llm-parse/mail-read.service';
|
||||
import type { WalkedMail } from '../llm-parse/mail-read.types';
|
||||
import { MailIngestAddressService } from './mail-ingest-address.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { sealMail } from './mail-seal';
|
||||
@@ -24,10 +25,24 @@ export type MailIntakeRefusal =
|
||||
| 'address_daily_limit'
|
||||
| 'inbox_full';
|
||||
|
||||
/**
|
||||
* Refusals that pass: the import is switched off for now, the address took
|
||||
* its share for the day, or its person's inbox is full. A transport keeps
|
||||
* such a mail and offers it again later instead of dropping it.
|
||||
*/
|
||||
const PASSING_REFUSALS: ReadonlySet<MailIntakeRefusal> = new Set(['unavailable', 'address_daily_limit', 'inbox_full']);
|
||||
|
||||
export function isPassingRefusal(reason: MailIntakeRefusal): boolean {
|
||||
return PASSING_REFUSALS.has(reason);
|
||||
}
|
||||
|
||||
type MailIntakeResult =
|
||||
| { status: 'accepted' | 'duplicate'; messageId: number }
|
||||
| { status: 'refused'; reason: MailIntakeRefusal };
|
||||
|
||||
/** A received mail's outcome over every address it reached. */
|
||||
type MailReceiveResult = { status: 'accepted' | 'duplicate' } | { status: 'refused'; reason: MailIntakeRefusal };
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
@@ -48,26 +63,32 @@ export class MailIngestIntakeService {
|
||||
) {}
|
||||
|
||||
/**
|
||||
* A mail a transport received. `envelopeRecipients` are the RCPT TO
|
||||
* addresses where the transport knows them; otherwise the headers name the
|
||||
* address, in the order the receiving server adds them.
|
||||
* A mail a transport received, stored once for every live address it was
|
||||
* sent to. `envelopeRecipients` are the RCPT TO addresses where the
|
||||
* transport knows them, and then the only ones read; otherwise the delivery
|
||||
* header the receiving server put on top names the address.
|
||||
*
|
||||
* When one of its addresses refuses it for now, the whole mail answers that
|
||||
* refusal: the transport offers it again, and the copies stored already
|
||||
* come back as duplicates.
|
||||
*/
|
||||
async receive(
|
||||
raw: Buffer,
|
||||
transport: Exclude<MailIngestTransport, 'upload'>,
|
||||
envelopeRecipients: readonly string[] = [],
|
||||
): Promise<MailIntakeResult> {
|
||||
): Promise<MailReceiveResult> {
|
||||
const refused = await this.precheck(raw);
|
||||
if (refused) return { status: 'refused', reason: refused };
|
||||
const mail = this.reader.walk(raw);
|
||||
if (!mail) return { status: 'refused', reason: 'not_a_mail' };
|
||||
const { deliveredTo, xOriginalTo, to } = mail.envelope;
|
||||
const headerRecipients = [deliveredTo, xOriginalTo, ...(to ?? '').split(',')].filter(
|
||||
(value): value is string => !!value && value.trim() !== '',
|
||||
);
|
||||
const resolved = await this.addresses.resolve([...envelopeRecipients, ...headerRecipients]);
|
||||
if (!resolved) return { status: 'refused', reason: 'unknown_address' };
|
||||
return this.store(raw, transport, resolved.address, mail.envelope.messageId);
|
||||
const recipients = envelopeRecipients.length > 0 ? envelopeRecipients : headerRecipients(mail.envelope);
|
||||
const addresses = await this.addresses.resolveAll(recipients);
|
||||
if (addresses.length === 0) return { status: 'refused', reason: 'unknown_address' };
|
||||
const results = await addresses.reduce<Promise<MailIntakeResult[]>>(async (previous, address) => {
|
||||
const done = await previous;
|
||||
return [...done, await this.store(raw, transport, address, mail.envelope.messageId)];
|
||||
}, Promise.resolve([]));
|
||||
return combined(results);
|
||||
}
|
||||
|
||||
/** An .eml the person uploaded, read as if they had forwarded it to their own address. */
|
||||
@@ -126,3 +147,25 @@ export class MailIngestIntakeService {
|
||||
return { status: 'accepted', messageId: id };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The addresses the headers name when the transport knows no envelope: the
|
||||
* topmost Delivered-To, else the topmost X-Original-To, which the receiving
|
||||
* server adds above whatever the mail brought along, and only without either
|
||||
* the To line. A sender who names another token there gains nothing over
|
||||
* sending to that token directly.
|
||||
*/
|
||||
function headerRecipients(envelope: WalkedMail['envelope']): string[] {
|
||||
const delivered = [envelope.deliveredTo, envelope.xOriginalTo].find((value) => !!value?.trim());
|
||||
if (delivered) return [delivered];
|
||||
return (envelope.to ?? '').split(',').filter((value) => value.trim() !== '');
|
||||
}
|
||||
|
||||
/** One answer for a mail stored per address: a passing refusal first, then accepted, then duplicate. */
|
||||
function combined(results: readonly MailIntakeResult[]): MailReceiveResult {
|
||||
const passing = results.find((result) => result.status === 'refused' && isPassingRefusal(result.reason));
|
||||
if (passing) return passing;
|
||||
if (results.some((result) => result.status === 'accepted')) return { status: 'accepted' };
|
||||
if (results.some((result) => result.status === 'duplicate')) return { status: 'duplicate' };
|
||||
return results[0] ?? { status: 'refused', reason: 'unknown_address' };
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { MailIngestCursorRepository } from '../../db/repositories/MailInges
|
||||
import { dbNow } from '../../db/types';
|
||||
import { logError, logWarn } from '../audit/audit-log.logger';
|
||||
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { isPassingRefusal, MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestRelayHost, type MailInboxRelay } from './mail-ingest-relay.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
@@ -15,35 +15,51 @@ import { z } from 'zod';
|
||||
|
||||
/** How long a run holds the relay before another process may take it over. */
|
||||
const LEASE_MS = 10 * 60 * 1000;
|
||||
/** The mails one run asks the relay for. */
|
||||
/** The mails one page of the relay's queue holds, and the pages one run reads. */
|
||||
const PAGE = 10;
|
||||
const PAGES_PER_RUN = 5;
|
||||
/** The most recipients one mail names on this instance; the relay sends no more. */
|
||||
const MAX_RELAYED_RECIPIENTS = 100;
|
||||
/** The base64 length of the largest mail the relay hands on. */
|
||||
const MAX_RAW_CHARS = Math.ceil(MAIL_INGEST_LIMITS.cloudMailBytes / 3) * 4;
|
||||
/** The relay's mail ids are UUIDs. */
|
||||
const MAIL_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
/** The relay's answer, checked before anything of it is used. */
|
||||
const relayInboxSchema = z.object({
|
||||
mails: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/),
|
||||
/** The addresses it was sent to, as the relay's mail server took them (RCPT TO). */
|
||||
recipients: z.array(z.string().min(3).max(320)).min(1).max(5),
|
||||
sha256: z.string().regex(/^[0-9a-f]{64}$/),
|
||||
/** The mail itself, base64. */
|
||||
raw: z.string().max(MAX_RAW_CHARS),
|
||||
}),
|
||||
)
|
||||
.max(PAGE),
|
||||
/** One page of the relay's queue; each mail in it is checked on its own, so an unreadable one blocks no other. */
|
||||
const relayPageSchema = z.object({
|
||||
messages: z.array(z.unknown()).max(PAGE),
|
||||
/** Further mails wait beyond `next`. */
|
||||
more: z.boolean(),
|
||||
/** Where the next page starts: after the last mail the relay looked at for this one. */
|
||||
next: z
|
||||
.string()
|
||||
.regex(/^[A-Za-z0-9_-]{1,200}$/)
|
||||
.nullable(),
|
||||
});
|
||||
type RelayedMail = z.infer<typeof relayInboxSchema>['mails'][number];
|
||||
const relayedMailSchema = z.object({
|
||||
id: z.string().regex(MAIL_ID),
|
||||
receivedAt: z.string().max(40),
|
||||
size: z.number().int().nonnegative(),
|
||||
sha256: z.string().regex(/^[0-9a-f]{64}$/),
|
||||
/** The addresses on this instance it was sent to, as the relay's mail server took them (RCPT TO). */
|
||||
recipients: z.array(z.string().min(3).max(320)).min(1).max(MAX_RELAYED_RECIPIENTS),
|
||||
/** The mail itself, base64. */
|
||||
raw: z.string().max(MAX_RAW_CHARS),
|
||||
});
|
||||
type RelayedMail = z.infer<typeof relayedMailSchema>;
|
||||
const relayedIdSchema = z.object({ id: z.string().regex(MAIL_ID) });
|
||||
|
||||
/**
|
||||
* Collects this instance's forwarded mails from the TREK Cloud relay every
|
||||
* two minutes: a page of at most ten, each checked against its hash, handed
|
||||
* to the intake under the address it was sent to, and acknowledged once it
|
||||
* is in (or once it is clear it never will be), so the relay drops it. A
|
||||
* mail that does not arrive whole is left for the next run. Idle wherever
|
||||
* no relay is attached, which is every self-hosted install.
|
||||
* two minutes, page after page while more wait, up to five pages a run. Each
|
||||
* mail is checked against its hash, handed to the intake under the addresses
|
||||
* it was sent to, and acknowledged once it is in (or once it is clear it
|
||||
* never will be), so the relay drops it. A mail that did not arrive whole,
|
||||
* or that the import cannot take for now (switched off, an address's share
|
||||
* for the day taken, an inbox full), stays with the relay for a later run,
|
||||
* and the pages after it are still read. Idle wherever no relay is attached,
|
||||
* which is every self-hosted install, and where the control plane runs no
|
||||
* import for this instance.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestRelayJob implements OnApplicationBootstrap {
|
||||
@@ -71,12 +87,7 @@ export class MailIngestRelayJob implements OnApplicationBootstrap {
|
||||
const until = dbNow(new Date(now.getTime() + LEASE_MS));
|
||||
if (!(await this.cursor.acquireLease('cloud', '', LEASE_OWNER, until, dbNow(now)))) return 0;
|
||||
try {
|
||||
const page = relayInboxSchema.safeParse(await relay.list(PAGE));
|
||||
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
|
||||
const taken = await page.data.mails.reduce(async (previous, mail) => {
|
||||
const count = await previous;
|
||||
return count + ((await this.takeIn(relay, mail)) ? 1 : 0);
|
||||
}, Promise.resolve(0));
|
||||
const taken = await this.drain(relay, null, PAGES_PER_RUN);
|
||||
await this.cursor.recordOk('cloud', '', dbNow(new Date()));
|
||||
return taken;
|
||||
} catch (err) {
|
||||
@@ -89,16 +100,43 @@ export class MailIngestRelayJob implements OnApplicationBootstrap {
|
||||
}
|
||||
}
|
||||
|
||||
/** One relayed mail into the intake; answers whether it was taken in. */
|
||||
private async takeIn(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
|
||||
/** The pages from `after` on, as many as are left to the run; answers the mails taken in. */
|
||||
private async drain(relay: MailInboxRelay, after: string | null, pagesLeft: number): Promise<number> {
|
||||
const answer = await relay.list(PAGE, after);
|
||||
if (answer === null) return 0;
|
||||
const page = relayPageSchema.safeParse(answer);
|
||||
if (!page.success) throw new Error('The relay answered with an inbox this server cannot read');
|
||||
const taken = await page.data.messages.reduce(async (previous, item) => {
|
||||
const count = await previous;
|
||||
return count + ((await this.takeIn(relay, item)) ? 1 : 0);
|
||||
}, Promise.resolve(0));
|
||||
const { more, next } = page.data;
|
||||
if (!more || next === null || next === after || pagesLeft <= 1) return taken;
|
||||
return taken + (await this.drain(relay, next, pagesLeft - 1));
|
||||
}
|
||||
|
||||
/** One mail of a page into the intake; answers whether it was taken in. */
|
||||
private async takeIn(relay: MailInboxRelay, item: unknown): Promise<boolean> {
|
||||
const parsed = relayedMailSchema.safeParse(item);
|
||||
if (parsed.success) return this.store(relay, parsed.data);
|
||||
// It would read the same on every run: dropped, where it names an id to drop it by.
|
||||
const id = relayedIdSchema.safeParse(item);
|
||||
logWarn(`[mail-ingest] The relay sent a mail this server cannot read${id.success ? `: ${id.data.id}` : ''}`);
|
||||
if (id.success) await relay.ack(id.data.id);
|
||||
return false;
|
||||
}
|
||||
|
||||
private async store(relay: MailInboxRelay, mail: RelayedMail): Promise<boolean> {
|
||||
const raw = Buffer.from(mail.raw, 'base64');
|
||||
if (createHash('sha256').update(raw).digest('hex') !== mail.sha256) {
|
||||
logWarn(`[mail-ingest] Relayed mail ${mail.id} did not arrive whole; left for the next run`);
|
||||
return false;
|
||||
}
|
||||
const result = await this.intake.receive(raw, 'cloud', mail.recipients);
|
||||
// Switched off in between: the relay keeps the mail until the import is back or the mail expires.
|
||||
if (result.status === 'refused' && result.reason === 'unavailable') return false;
|
||||
if (result.status === 'refused' && isPassingRefusal(result.reason)) {
|
||||
logWarn(`[mail-ingest] Relayed mail ${mail.id} waits with the relay: ${result.reason}`);
|
||||
return false;
|
||||
}
|
||||
if (result.status === 'refused') {
|
||||
logWarn(`[mail-ingest] Relayed mail ${mail.id} was not taken in: ${result.reason}`);
|
||||
}
|
||||
|
||||
@@ -7,8 +7,13 @@ import { Injectable } from '@nestjs/common';
|
||||
* encrypted, and hands it over here; nothing of a person reaches it.
|
||||
*/
|
||||
export interface MailInboxRelay {
|
||||
/** `GET /v1/instance/inbox?limit=n`: the waiting mails, the answer's JSON as parsed, checked by the caller. */
|
||||
list(limit: number): Promise<unknown>;
|
||||
/**
|
||||
* `GET /v1/instance/inbox?limit=n&after=<cursor>`: the waiting mails after
|
||||
* the cursor a previous page named (from the oldest without one), the
|
||||
* answer's JSON as parsed, checked by the caller. Null when the control
|
||||
* plane has no import for this instance (its 404), which is no failure.
|
||||
*/
|
||||
list(limit: number, after: string | null): Promise<unknown>;
|
||||
/** `POST /v1/instance/inbox/:id/ack`: the mail is taken in (or never will be), and the relay drops it. */
|
||||
ack(id: string): Promise<void>;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user