mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
5b0412bc2c
isBareHeaderBlock rescanned the header run below every sender line, so a plain-text mail of nothing but "From:" lines without a subject took quadratic time and held the event loop for seconds before the address was even checked. The block starts are now worked out for all lines in one pass from the bottom up, with the same rule.
370 lines
14 KiB
TypeScript
370 lines
14 KiB
TypeScript
import {
|
|
MAX_MAIL_ATTACHMENTS,
|
|
MAX_MAIL_ATTACHMENT_BYTES,
|
|
walkMail,
|
|
type WalkedMail,
|
|
} from '../../../../src/nest/llm-parse/mime-walker';
|
|
|
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
|
|
/** Mail is CRLF-delimited on the wire: the fixtures are built the same way. */
|
|
const mail = (...lines: string[]) => Buffer.from(lines.join('\r\n'), 'latin1');
|
|
|
|
/** Base64 in 76-character lines, as mailers write it. */
|
|
const b64 = (data: Buffer | string) =>
|
|
(typeof data === 'string' ? Buffer.from(data, 'utf8') : data).toString('base64').replace(/.{76}/g, '$&\r\n');
|
|
|
|
const PDF = Buffer.from('%PDF-1.4\n1 0 obj << >> endobj\n%%EOF\n', 'latin1');
|
|
const PKPASS = Buffer.concat([Buffer.from([0x50, 0x4b, 0x03, 0x04]), Buffer.from('pass.json')]);
|
|
const PNG = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.from('IHDR')]);
|
|
const JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
|
|
const WEBP = Buffer.concat([Buffer.from('RIFF'), Buffer.from([0, 0, 0, 0]), Buffer.from('WEBPVP8 ')]);
|
|
const ICS = 'BEGIN:VCALENDAR\r\nBEGIN:VEVENT\r\nSUMMARY:LH 716\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n';
|
|
|
|
/** One attachment part of a multipart/mixed body. */
|
|
const part = (type: string, name: string | null, data: Buffer | string, extra: string[] = []) => [
|
|
'--mix',
|
|
`Content-Type: ${type}${name ? `; name="${name}"` : ''}`,
|
|
'Content-Transfer-Encoding: base64',
|
|
...(name ? [`Content-Disposition: attachment; filename="${name}"`] : []),
|
|
...extra,
|
|
'',
|
|
b64(data),
|
|
];
|
|
|
|
/** A forwarded booking with the given parts after its text. */
|
|
const mixed = (text: string, ...parts: string[][]) =>
|
|
mail(
|
|
'From: Ana <ana@example.org>',
|
|
'To: in+token@trek.example',
|
|
'Subject: Fwd: Your booking',
|
|
'MIME-Version: 1.0',
|
|
'Content-Type: multipart/mixed; boundary="mix"',
|
|
'',
|
|
'--mix',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
text,
|
|
...parts.flat(),
|
|
'--mix--',
|
|
'',
|
|
);
|
|
|
|
afterEach(() => vi.restoreAllMocks());
|
|
|
|
/** The walk of a mail the case expects to read: one the walker refuses fails here, by name. */
|
|
function walk(raw: Buffer): WalkedMail {
|
|
const walked = walkMail(raw);
|
|
if (!walked) throw new Error('the walker read no mail');
|
|
return walked;
|
|
}
|
|
|
|
describe('walkMail: the envelope and the body', () => {
|
|
it('MIME-WALK-001: refuses bytes that are not a mail', () => {
|
|
expect(walkMail(Buffer.from('<html><body>Flight AB123</body></html>'))).toBeNull();
|
|
expect(walkMail(Buffer.from(''))).toBeNull();
|
|
});
|
|
|
|
it('MIME-WALK-002: keeps the envelope apart, decoded, and the body free of it', () => {
|
|
const walked: WalkedMail = walk(
|
|
mail(
|
|
'Authentication-Results: mx.trek.example; dkim=pass header.d=example.org',
|
|
'Authentication-Results: relay.example; dkim=fail',
|
|
'Delivered-To: in+token@trek.example',
|
|
'X-Original-To: in+token@trek.example',
|
|
'From: =?utf-8?B?QW5hIE3DvGxsZXI=?= <ana@example.org>',
|
|
'To: in+token@trek.example',
|
|
'Cc: ben@example.org',
|
|
'Subject: Fwd: WG: =?utf-8?Q?Ihre_Buchung_f=C3=BCr_Tokio?=',
|
|
'Date: Mon, 12 May 2025 09:00:00 +0200',
|
|
'Message-ID: <abc@example.org>',
|
|
'List-Unsubscribe: <https://airline.example/unsubscribe>',
|
|
'Precedence: bulk',
|
|
'Auto-Submitted: auto-generated',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
'Flight LH 716, booking ABC123',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.envelope).toEqual({
|
|
from: 'Ana Müller <ana@example.org>',
|
|
to: 'in+token@trek.example',
|
|
cc: 'ben@example.org',
|
|
subject: 'Fwd: WG: Ihre Buchung für Tokio',
|
|
date: 'Mon, 12 May 2025 09:00:00 +0200',
|
|
messageId: '<abc@example.org>',
|
|
deliveredTo: 'in+token@trek.example',
|
|
xOriginalTo: 'in+token@trek.example',
|
|
listUnsubscribe: '<https://airline.example/unsubscribe>',
|
|
precedence: 'bulk',
|
|
autoSubmitted: 'auto-generated',
|
|
// The receiving server's own verdict is the topmost header.
|
|
authenticationResults: 'mx.trek.example; dkim=pass header.d=example.org',
|
|
});
|
|
expect(walked.subject).toBe('Ihre Buchung für Tokio');
|
|
expect(walked.body).toBe('Flight LH 716, booking ABC123');
|
|
expect(walked.body).not.toContain('example.org');
|
|
expect(walked.forwardLayers).toBe(0);
|
|
});
|
|
|
|
it('MIME-WALK-003: a missing header is null and a mail without a subject has none', () => {
|
|
const walked = walk(mail('From: a@example.org', 'Content-Type: text/plain', '', 'Hotel', ''));
|
|
expect(walked.subject).toBeNull();
|
|
expect(walked.envelope.to).toBeNull();
|
|
expect(walked.envelope.authenticationResults).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('walkMail: forwarded and quoted messages', () => {
|
|
it('MIME-WALK-010: a Gmail forward loses its marker and the header block under it', () => {
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
'Subject: Fwd: Booking',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
'See below',
|
|
'',
|
|
'---------- Forwarded message ---------',
|
|
'From: Airline <noreply@airline.example>',
|
|
'Date: Mon, 12 May 2025 at 09:00',
|
|
'Subject: Your booking ABC123',
|
|
'To: <ana@example.org>',
|
|
'',
|
|
'Flight LH 716 Frankfurt to Tokyo, booking ABC123',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('See below\n\nFlight LH 716 Frankfurt to Tokyo, booking ABC123');
|
|
expect(walked.body).not.toContain('airline.example');
|
|
expect(walked.forwardLayers).toBe(1);
|
|
});
|
|
|
|
it('MIME-WALK-011: an Outlook forward in HTML, its rule and bold fields on their own lines', () => {
|
|
const html = [
|
|
'<div>FYI</div>',
|
|
'<hr><div><b>From:</b> Hotel <stay@hotel.example><br><b>Sent:</b> Monday<br>',
|
|
'<b>To:</b> Ana<br><b>Subject:</b> Reservation H-77</div>',
|
|
'<p>Check-in 2 September, reservation H-77</p>',
|
|
'<img src="https://tracker.example/open.png">',
|
|
].join('');
|
|
const walked = walk(
|
|
mail('From: ana@example.org', 'Subject: WG: Reservation', 'Content-Type: text/html; charset=utf-8', '', html, ''),
|
|
);
|
|
expect(walked.body).toContain('Check-in 2 September, reservation H-77');
|
|
expect(walked.body).not.toContain('hotel.example');
|
|
expect(walked.body).not.toContain('Subject:');
|
|
});
|
|
|
|
it('MIME-WALK-012: a bare Outlook block goes, a French ticket that reads like one stays', () => {
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
// The fixture is written byte for byte in latin1, so the "À" is declared as such.
|
|
'Content-Type: text/plain; charset=iso-8859-1',
|
|
'',
|
|
'From: Rail <tickets@rail.example>',
|
|
'Sent: Monday',
|
|
'To: Ana',
|
|
'Subject: Votre billet',
|
|
'',
|
|
'De : Paris',
|
|
'À : Lyon',
|
|
'Date : 12/05/2026',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('De : Paris\nÀ : Lyon\nDate : 12/05/2026');
|
|
});
|
|
|
|
it('MIME-WALK-013: quoted lines are unquoted level by level, the attribution goes', () => {
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
'On Mon, 12 May 2025, Ben <ben@example.org> wrote:',
|
|
'> Begin forwarded message:',
|
|
'>',
|
|
'> From: Ferry <info@ferry.example>',
|
|
'> Subject: Ticket',
|
|
'>',
|
|
'>> Ferry Piraeus to Naxos, ticket F-12',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('Ferry Piraeus to Naxos, ticket F-12');
|
|
expect(walked.body).not.toMatch(/example\.org|ferry\.example/);
|
|
expect(walked.forwardLayers).toBeGreaterThanOrEqual(2);
|
|
});
|
|
|
|
it('MIME-WALK-014: a mail forwarded as an attachment is read as a message of its own', () => {
|
|
const inner = [
|
|
'From: Airline <noreply@airline.example>',
|
|
'Subject: Your booking',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
'Flight LH 716, booking ABC123',
|
|
].join('\r\n');
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
'Subject: Fwd: Your booking',
|
|
'Content-Type: multipart/mixed; boundary="mix"',
|
|
'',
|
|
'--mix',
|
|
'Content-Type: text/plain',
|
|
'',
|
|
'Forwarding this',
|
|
'--mix',
|
|
'Content-Type: message/rfc822',
|
|
'Content-Disposition: attachment; filename="booking.eml"',
|
|
'',
|
|
inner,
|
|
'--mix--',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('Forwarding this\n\nFlight LH 716, booking ABC123');
|
|
expect(walked.body).not.toContain('airline.example');
|
|
expect(walked.forwardLayers).toBe(1);
|
|
});
|
|
|
|
it('MIME-WALK-015: a message nested past the depth limit stops there instead of recursing on', () => {
|
|
let message = ['From: deep@example.org', 'Content-Type: text/plain', '', 'Hotel H-77'].join('\r\n');
|
|
for (let level = 0; level < 15; level++) {
|
|
message = ['From: x@example.org', 'Content-Type: message/rfc822', '', message].join('\r\n');
|
|
}
|
|
const walked = walk(Buffer.from(message, 'latin1'));
|
|
expect(walked.body).toBe('');
|
|
expect(walked.forwardLayers).toBe(10);
|
|
});
|
|
|
|
it('MIME-WALK-016: a run of sender lines ending in a subject goes, the run without one under it stays', () => {
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
'Content-Type: text/plain; charset=utf-8',
|
|
'',
|
|
'From: a',
|
|
'From: b',
|
|
'From: Ferry <info@ferry.example>',
|
|
'Subject: Ticket',
|
|
'',
|
|
'From: Naxos',
|
|
'To: Piraeus',
|
|
'',
|
|
'Ferry ticket F-12',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('From: Naxos\nTo: Piraeus\n\nFerry ticket F-12');
|
|
expect(walked.forwardLayers).toBe(1);
|
|
});
|
|
|
|
it(
|
|
'MIME-WALK-017: a mail of nothing but sender lines is read in one pass instead of rescanning them per line',
|
|
{ timeout: 5_000 },
|
|
() => {
|
|
const lines = 30_000;
|
|
const walked = walk(
|
|
mail('From: ana@example.org', 'Content-Type: text/plain', '', ...Array<string>(lines).fill('From: a'), ''),
|
|
);
|
|
expect(walked.body.split('\n')).toHaveLength(lines);
|
|
expect(walked.forwardLayers).toBe(0);
|
|
},
|
|
);
|
|
});
|
|
|
|
describe('walkMail: attachments', () => {
|
|
it('MIME-WALK-020: keeps a PDF, a pass, a calendar and the three photo types, by type or by name', () => {
|
|
const walked = walk(
|
|
mixed(
|
|
'Booking ABC123',
|
|
part('application/pdf', 'ticket.pdf', PDF),
|
|
part('application/octet-stream', 'boarding.pkpass', PKPASS),
|
|
part('text/calendar', null, ICS),
|
|
part('image/jpeg', 'scan.JPG', JPEG),
|
|
part('image/png', 'pass.png', PNG),
|
|
),
|
|
);
|
|
expect(walked.attachments.map(({ fileName, kind, mimeType }) => ({ fileName, kind, mimeType }))).toEqual([
|
|
{ fileName: 'ticket.pdf', kind: 'pdf', mimeType: 'application/pdf' },
|
|
{ fileName: 'boarding.pkpass', kind: 'pkpass', mimeType: 'application/vnd.apple.pkpass' },
|
|
{ fileName: 'attachment-3.ics', kind: 'ics', mimeType: 'text/calendar' },
|
|
{ fileName: 'scan.JPG', kind: 'jpeg', mimeType: 'image/jpeg' },
|
|
{ fileName: 'pass.png', kind: 'png', mimeType: 'image/png' },
|
|
]);
|
|
expect(walked.attachments[0].data.equals(PDF)).toBe(true);
|
|
expect(walked.attachments[2].data.toString('utf8')).toBe(ICS);
|
|
expect(walked.skippedAttachments).toBe(0);
|
|
// The calendar is an attachment, not body text.
|
|
expect(walked.body).toBe('Booking ABC123');
|
|
});
|
|
|
|
it('MIME-WALK-021: a layout image goes, a WebP the person attached stays', () => {
|
|
const walked = walk(
|
|
mixed(
|
|
'Booking',
|
|
part('image/png', null, PNG, ['Content-ID: <logo@airline>', 'Content-Disposition: inline']),
|
|
part('image/webp', 'photo.webp', WEBP),
|
|
),
|
|
);
|
|
expect(walked.attachments.map((a) => a.kind)).toEqual(['webp']);
|
|
expect(walked.skippedAttachments).toBe(0);
|
|
});
|
|
|
|
it('MIME-WALK-022: a file that is not what it claims is skipped, a type no booking travels in ignored', () => {
|
|
const walked = walk(
|
|
mixed(
|
|
'Booking',
|
|
part('application/pdf', 'ticket.pdf', Buffer.from('MZ not a pdf')),
|
|
part('image/png', 'pass.png', JPEG),
|
|
part('text/calendar', 'invite.ics', 'just text'),
|
|
part('application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'terms.docx', 'PK'),
|
|
),
|
|
);
|
|
expect(walked.attachments).toEqual([]);
|
|
expect(walked.skippedAttachments).toBe(3);
|
|
});
|
|
|
|
it('MIME-WALK-023: past the count limit the rest are skipped and counted', () => {
|
|
const pdfs = Array.from({ length: MAX_MAIL_ATTACHMENTS + 2 }, (_, i) => part('application/pdf', `t${i}.pdf`, PDF));
|
|
const walked = walk(mixed('Booking', ...pdfs));
|
|
expect(walked.attachments).toHaveLength(MAX_MAIL_ATTACHMENTS);
|
|
expect(walked.skippedAttachments).toBe(2);
|
|
});
|
|
|
|
it('MIME-WALK-024: an attachment past the size limit is left out', () => {
|
|
const big = Buffer.concat([PDF, Buffer.alloc(MAX_MAIL_ATTACHMENT_BYTES)]);
|
|
const walked = walk(mixed('Booking', part('application/pdf', 'big.pdf', big)));
|
|
expect(walked.attachments).toEqual([]);
|
|
expect(walked.skippedAttachments).toBe(1);
|
|
});
|
|
|
|
it('MIME-WALK-025: a file name loses its path and control characters, an empty one is made up', () => {
|
|
const walked = walk(
|
|
mixed('Booking', part('application/pdf', '../../etc/ticket\u0007.pdf', PDF), part('application/pdf', '..', PDF)),
|
|
);
|
|
expect(walked.attachments.map((a) => a.fileName)).toEqual(['ticket.pdf', 'attachment-2.pdf']);
|
|
});
|
|
|
|
it('MIME-WALK-026: nothing is fetched, a remote image stays a string in the markup that is dropped', () => {
|
|
const fetchSpy = vi.spyOn(globalThis, 'fetch');
|
|
const walked = walk(
|
|
mail(
|
|
'From: ana@example.org',
|
|
'Content-Type: text/html; charset=utf-8',
|
|
'',
|
|
'<p>Booking ABC123</p><img src="https://tracker.example/pixel.gif">',
|
|
'<link rel="stylesheet" href="https://x.example/a.css">',
|
|
'',
|
|
),
|
|
);
|
|
expect(walked.body).toBe('Booking ABC123');
|
|
expect(walked.attachments).toEqual([]);
|
|
expect(fetchSpy).not.toHaveBeenCalled();
|
|
});
|
|
});
|