fix(llm-parse): decode .eml as MIME before extracting text (#1724)

Attachments were treated as ready-made HTML, but a real mail is multipart with base64 or quoted-printable transfer encoding. What reached the model was the plaintext headers followed by a wall of base64, truncated before any content — which is why a sender name from the Subject line still showed up while the booking never did. The html part is preferred, plain text is the fallback, transfer encoding and charset are decoded and entities resolved before the text goes out. No new dependency. If no text part is found the previous raw path still runs.
2026-08-14 08:26:39 +00:00 · 2026-07-29 14:37:26 +02:00
parent 50600b74cd
commit 9ab9665e99
4 changed files with 803 additions and 6 deletions
@@ -0,0 +1,254 @@
+/**
+ * Minimal MIME reader for uploaded `.eml` files.
+ *
+ * A real mail keeps its body inside multipart containers and encodes it as
+ * base64 or quoted-printable, so the raw bytes carry no readable booking data —
+ * reading the file as text yields the plaintext headers plus an encoded blob.
+ * Only the body parts matter for the LLM prompt, so the part tree is walked
+ * here instead of pulling in a full mail-parser dependency: unfold headers,
+ * split on the boundary, decode transfer encoding, then the charset.
+ * Attachments are skipped.
+ */
+
+/** Guards against a hand-crafted mail nesting multiparts without end. */
+const MAX_DEPTH = 10;
+
+type Headers = Map<string, string>;
+
+interface MessagePart {
+  headers: Headers;
+  body: string;
+}
+
+interface ContentType {
+  type: string;
+  params: Record<string, string>;
+}
+
+export interface ParsedMail {
+  /** Decoded `text/html` body part, `null` when the mail has none. */
+  html: string | null;
+  /** Decoded `text/plain` body part, `null` when the mail has none. */
+  text: string | null;
+  /** `From`/`To`/`Subject`/`Date` as decoded `Name: value` lines. */
+  headerLines: string[];
+}
+
+/** Header names that mark a buffer as an actual mail rather than a stray text file. */
+const MAIL_HEADERS = [
+  'from',
+  'to',
+  'subject',
+  'date',
+  'message-id',
+  'received',
+  'mime-version',
+  'return-path',
+  'delivered-to',
+];
+
+/** Headers worth keeping in the prompt — a booking reference often lives in the subject. */
+const SUMMARY_HEADERS: [string, string][] = [
+  ['from', 'From'],
+  ['to', 'To'],
+  ['subject', 'Subject'],
+  ['date', 'Date'],
+];
+
+/**
+ * Parse a `.eml` buffer into its decoded text bodies.
+ * Returns `null` when the buffer is not a mail, so callers can fall back to
+ * their previous handling instead of losing content.
+ */
+export function parseEmail(buffer: Buffer): ParsedMail | null {
+  // latin1 keeps one char per byte, so the structure can be walked as a string
+  // while every part is still re-encodable byte-exact for its own charset.
+  const raw = buffer.toString('latin1').replace(/^\u00ef\u00bb\u00bf/, '');
+  const message = splitMessage(raw);
+  if (!looksLikeMail(raw, message.headers)) return null;
+
+  const bodies: { html: string | null; text: string | null } = { html: null, text: null };
+  walkPart(message, bodies, 0);
+
+  return {
+    html: bodies.html,
+    text: bodies.text,
+    headerLines: summarizeHeaders(message.headers),
+  };
+}
+
+/**
+ * A mail opens with a header field and names at least one of the usual
+ * envelope headers; a bare HTML or text file saved as `.eml` does neither and
+ * is better served by the caller's raw handling.
+ */
+function looksLikeMail(raw: string, headers: Headers): boolean {
+  return /^[!-9;-~]+:/.test(raw) && MAIL_HEADERS.some(name => headers.has(name));
+}
+
+function summarizeHeaders(headers: Headers): string[] {
+  const lines: string[] = [];
+  for (const [name, label] of SUMMARY_HEADERS) {
+    const value = headers.get(name);
+    if (value) lines.push(`${label}: ${decodeEncodedWords(value)}`);
+  }
+  return lines;
+}
+
+/** Collect the first `text/html` and `text/plain` body across the part tree. */
+function walkPart(part: MessagePart, bodies: { html: string | null; text: string | null }, depth: number): void {
+  if (depth > MAX_DEPTH) return;
+  const contentType = parseContentType(part.headers.get('content-type'));
+  const type = contentType.type || 'text/plain';
+
+  if (type.startsWith('multipart/')) {
+    const boundary = contentType.params.boundary;
+    if (!boundary) return;
+    for (const raw of splitParts(part.body, boundary)) {
+      walkPart(splitMessage(raw), bodies, depth + 1);
+    }
+    return;
+  }
+
+  // A forwarded confirmation arrives as an embedded message — walk into it.
+  if (type === 'message/rfc822') {
+    walkPart(splitMessage(decodeToBuffer(part).toString('latin1')), bodies, depth + 1);
+    return;
+  }
+
+  if (isAttachment(part.headers)) return;
+  if (type === 'text/html') {
+    if (bodies.html === null) bodies.html = decodePartText(part, contentType);
+  } else if (type === 'text/plain') {
+    if (bodies.text === null) bodies.text = decodePartText(part, contentType);
+  }
+}
+
+function isAttachment(headers: Headers): boolean {
+  return parseContentType(headers.get('content-disposition')).type === 'attachment';
+}
+
+/** Split a message (or a part) into its unfolded headers and its still-encoded body. */
+function splitMessage(raw: string): MessagePart {
+  const separator = /\r?\n\r?\n/.exec(raw);
+  const headerBlock = separator ? raw.slice(0, separator.index) : raw;
+  const body = separator ? raw.slice(separator.index + separator[0].length) : '';
+  return { headers: parseHeaders(headerBlock), body };
+}
+
+function parseHeaders(block: string): Headers {
+  const headers: Headers = new Map();
+  let current = '';
+  const commit = () => {
+    const colon = current.indexOf(':');
+    // Keep the first occurrence: a re-sent mail can carry a header twice.
+    if (colon > 0) {
+      const name = current.slice(0, colon).trim().toLowerCase();
+      if (!headers.has(name)) headers.set(name, current.slice(colon + 1).trim());
+    }
+    current = '';
+  };
+  for (const line of block.split(/\r?\n/)) {
+    // A leading space or tab continues the previous header (RFC 5322 folding).
+    if (/^[ \t]/.test(line) && current) current += ' ' + line.trim();
+    else {
+      commit();
+      current = line;
+    }
+  }
+  commit();
+  return headers;
+}
+
+function parseContentType(value: string | undefined): ContentType {
+  const params: Record<string, string> = {};
+  if (!value) return { type: '', params };
+  const semicolon = value.indexOf(';');
+  const type = (semicolon < 0 ? value : value.slice(0, semicolon)).trim().toLowerCase();
+  const param = /;\s*([\w-]+)\s*=\s*(?:"([^"]*)"|([^;]*))/g;
+  let match: RegExpExecArray | null;
+  while ((match = param.exec(value)) !== null) {
+    params[match[1].toLowerCase()] = (match[2] ?? match[3] ?? '').trim();
+  }
+  return { type, params };
+}
+
+/**
+ * Cut a multipart body at its `--boundary` delimiters. The CRLF in front of a
+ * delimiter belongs to it, so it is matched (not captured) and stripped from
+ * the part that follows.
+ */
+function splitParts(body: string, boundary: string): string[] {
+  const delimiter = new RegExp(`(?:\\r?\\n|^)--${escapeRegExp(boundary)}(--)?[ \\t]*(?=\\r?\\n|$)`, 'g');
+  const parts: string[] = [];
+  let start = -1;
+  let match: RegExpExecArray | null;
+  while ((match = delimiter.exec(body)) !== null) {
+    if (start >= 0) parts.push(body.slice(start, match.index).replace(/^\r?\n/, ''));
+    if (match[1]) break; // closing delimiter
+    start = delimiter.lastIndex;
+  }
+  return parts;
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** Undo the transfer encoding of a part, yielding its raw bytes. */
+function decodeToBuffer(part: MessagePart): Buffer {
+  const encoding = (part.headers.get('content-transfer-encoding') ?? '').trim().toLowerCase();
+  if (encoding === 'base64') return decodeBase64(part.body);
+  if (encoding === 'quoted-printable') return decodeQuotedPrintable(part.body);
+  return Buffer.from(part.body, 'latin1');
+}
+
+function decodePartText(part: MessagePart, contentType: ContentType): string {
+  return decodeCharset(decodeToBuffer(part), contentType.params.charset).replace(/\r\n?/g, '\n');
+}
+
+function decodeBase64(payload: string): Buffer {
+  return Buffer.from(payload.replace(/[^A-Za-z0-9+/=]/g, ''), 'base64');
+}
+
+function decodeQuotedPrintable(payload: string): Buffer {
+  const joined = payload.replace(/=\r?\n/g, ''); // soft line breaks
+  const out = Buffer.allocUnsafe(joined.length);
+  let len = 0;
+  for (let i = 0; i < joined.length; i++) {
+    const hex = joined[i] === '=' ? joined.slice(i + 1, i + 3) : '';
+    if (/^[0-9a-fA-F]{2}$/.test(hex)) {
+      out[len++] = parseInt(hex, 16);
+      i += 2;
+    } else {
+      out[len++] = joined.charCodeAt(i) & 0xff;
+    }
+  }
+  return out.subarray(0, len);
+}
+
+function decodeCharset(bytes: Buffer, charset: string | undefined): string {
+  const label = (charset ?? '').trim().toLowerCase().replace(/^["']|["']$/g, '');
+  if (!label || label === 'utf-8' || label === 'utf8' || label === 'us-ascii' || label === 'ascii') {
+    return bytes.toString('utf8');
+  }
+  try {
+    return new TextDecoder(label).decode(bytes);
+  } catch {
+    // Unknown label — utf8 is still the best guess for modern mail.
+    return bytes.toString('utf8');
+  }
+}
+
+/** Resolve RFC 2047 encoded-words (`=?utf-8?B?…?=`), as used in subject lines. */
+function decodeEncodedWords(value: string): string {
+  return value
+    .replace(/=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g, (whole, charset: string, encoding: string, payload: string) => {
+      const bytes =
+        encoding.toUpperCase() === 'B' ? decodeBase64(payload) : decodeQuotedPrintable(payload.replace(/_/g, ' '));
+      const decoded = decodeCharset(bytes, charset);
+      return decoded || whole;
+    })
+    .replace(/\s+/g, ' ')
+    .trim();
+}
@@ -1,3 +1,4 @@
+import { parseEmail } from './mime-email';
 import { extname } from 'node:path';
 import { PDFParse } from 'pdf-parse';

@@ -12,18 +13,118 @@ export function isPdf(fileName: string): boolean {
  return extname(fileName).toLowerCase() === '.pdf';
 }

-/** Strip HTML/XML tags and collapse whitespace for a cleaner LLM prompt. */
+/**
+ * Entity names for U+00A0–U+00FF in code-point order. Booking mails are full of
+ * this block: umlauts, currency signs, the degree sign.
+ */
+const LATIN1_ENTITIES = (
+  'nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr ' +
+  'deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest ' +
+  'Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ' +
+  'ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig ' +
+  'agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml ' +
+  'eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml'
+).split(' ');
+
+/** Named entities a mail body realistically carries, resolved to their character. */
+const HTML_ENTITIES = new Map<string, string>([
+  ['amp', '&'],
+  ['lt', '<'],
+  ['gt', '>'],
+  ['quot', '"'],
+  ['apos', "'"],
+  ['ndash', '–'],
+  ['mdash', '—'],
+  ['hellip', '…'],
+  ['lsquo', '‘'],
+  ['rsquo', '’'],
+  ['sbquo', '‚'],
+  ['ldquo', '“'],
+  ['rdquo', '”'],
+  ['bdquo', '„'],
+  ['bull', '•'],
+  ['dagger', '†'],
+  ['Dagger', '‡'],
+  ['permil', '‰'],
+  ['lsaquo', '‹'],
+  ['rsaquo', '›'],
+  ['trade', '™'],
+  ['euro', '€'],
+  ['minus', '−'],
+  ['ne', '≠'],
+  ['le', '≤'],
+  ['ge', '≥'],
+  ['larr', '←'],
+  ['rarr', '→'],
+  ['harr', '↔'],
+  ...LATIN1_ENTITIES.map((name, i): [string, string] => [name, String.fromCharCode(0xa0 + i)]),
+]);
+
+function entityCodePoint(code: number): string | null {
+  // Surrogate halves and out-of-range values would throw or produce garbage.
+  if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff) return null;
+  if (code >= 0xd800 && code <= 0xdfff) return null;
+  return String.fromCodePoint(code);
+}
+
+/**
+ * Resolve HTML entities so the model sees "Zimmer für zwei – 120,00 €" instead
+ * of the escaped source. Numeric references come in decimal (`&#88;`) and hex
+ * (`&#x58;`) form; an unknown name is left as written rather than dropped.
+ */
+function decodeEntities(s: string): string {
+  if (!s.includes('&')) return s;
+  return s.replace(/&(#\d{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,30});/g, (whole, ref: string) => {
+    if (ref[0] === '#') {
+      const hex = ref[1] === 'x' || ref[1] === 'X';
+      return entityCodePoint(parseInt(hex ? ref.slice(2) : ref.slice(1), hex ? 16 : 10)) ?? whole;
+    }
+    // Case matters (&Uuml; vs &uuml;); only fall back to lower case for names
+    // that are shouted in old templates.
+    return HTML_ENTITIES.get(ref) ?? HTML_ENTITIES.get(ref.toLowerCase()) ?? whole;
+  });
+}
+
+/** Strip HTML/XML tags, resolve entities and collapse whitespace for a cleaner LLM prompt. */
 function stripMarkup(s: string): string {
-  return s
+  const withoutTags = s
    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
-    .replace(/<[^>]+>/g, ' ')
-    .replace(/&nbsp;/gi, ' ')
-    .replace(/[ \t]+/g, ' ')
+    .replace(/<[^>]+>/g, ' ');
+  // Entities come last so that a decoded `&lt;` cannot turn into a tag the
+  // stripper would then eat.
+  return decodeEntities(withoutTags)
+    .replace(/[ \t\u00a0]+/g, ' ')
    .replace(/\n{3,}/g, '\n\n')
    .trim();
 }

+/** Whitespace cleanup for text that is already plain — no tags to strip out. */
+function collapseWhitespace(s: string): string {
+  return s
+    .replace(/[ \t\u00a0]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+/**
+ * Decode a `.eml` upload: walk the MIME tree, prefer the `text/html` body over
+ * `text/plain`, and keep the interesting headers in front of it. Returns an
+ * empty string when the buffer is not a MIME message or when the walk finds no
+ * readable body, which leaves the caller on its raw-bytes path — the headers
+ * alone would be a worse prompt than the undecoded file.
+ */
+function extractEmailText(buffer: Buffer): string {
+  const mail = parseEmail(buffer);
+  if (!mail) return '';
+  // An empty html part must not shadow a filled text/plain alternative, so both
+  // are cleaned up before one is picked.
+  const html = mail.html !== null ? stripMarkup(mail.html) : '';
+  const body = html || (mail.text !== null ? collapseWhitespace(mail.text) : '');
+  if (!body) return '';
+  return [mail.headerLines.join('\n'), body].filter(section => section.length > 0).join('\n\n');
+}
+
 /** Extract the embedded text layer from a PDF (empty for scanned/image-only PDFs). */
 async function extractPdfText(buffer: Buffer): Promise<string> {
  const parser = new PDFParse({ data: new Uint8Array(buffer) });
@@ -56,7 +157,8 @@ function cleanPdfText(text: string): string {
 /**
 * Extract text from a booking file for the OpenAI-compatible/local LLM path
 * (Ollama can't ingest PDFs or `file` parts, so everything becomes text).
- *  - txt/html/htm/eml → decoded (markup stripped)
+ *  - eml              → MIME-decoded body part (markup stripped)
+ *  - txt/html/htm     → decoded (markup stripped)
 *  - pdf              → embedded text layer via pdf-parse
 *  - anything else    → best-effort UTF-8 decode
 * A scanned/image-only PDF yields empty text — that case needs a vision provider
@@ -65,6 +167,13 @@ function cleanPdfText(text: string): string {
 export async function extractText(buffer: Buffer, fileName: string): Promise<string> {
  const ext = extname(fileName).toLowerCase();
  if (isPdf(fileName)) return extractPdfText(buffer);
+  if (ext === '.eml') {
+    // Real mail hides its body behind base64/quoted-printable, so decode the MIME
+    // structure first; a file that isn't a message, or a message whose only content
+    // sits in parts we don't read, falls through to the raw bytes.
+    const mailText = extractEmailText(buffer);
+    if (mailText) return mailText;
+  }
  const raw = buffer.toString('utf8');
  if (ext === '.html' || ext === '.htm' || ext === '.eml') return stripMarkup(raw);
  return raw.trim();
@@ -0,0 +1,245 @@
+import { describe, it, expect } from 'vitest';
+
+import { parseEmail } from '../../../../src/nest/llm-parse/mime-email';
+
+/** Mail is CRLF-delimited on the wire — build the fixtures the same way. */
+const mail = (...lines: string[]) => Buffer.from(lines.join('\r\n'), 'latin1');
+
+const b64 = (s: string) => Buffer.from(s, 'utf8').toString('base64');
+
+describe('mime-email', () => {
+  it('rejects a buffer that is not a message', () => {
+    expect(parseEmail(Buffer.from('<html><body>Flight AB123</body></html>'))).toBeNull();
+    expect(parseEmail(Buffer.from(''))).toBeNull();
+    // Looks like a header field, but names none of the envelope headers.
+    expect(parseEmail(Buffer.from('note:\r\n\r\nnothing here'))).toBeNull();
+  });
+
+  it('tolerates a leading utf-8 BOM', () => {
+    const parsed = parseEmail(
+      Buffer.concat([
+        Buffer.from([0xef, 0xbb, 0xbf]),
+        mail('From: a@example.com', 'Subject: Hi', 'Content-Type: text/plain', '', 'Booking 4711', ''),
+      ]),
+    );
+    expect(parsed?.text).toBe('Booking 4711\n');
+  });
+
+  it('decodes a base64 body and keeps the summary headers', () => {
+    const parsed = parseEmail(
+      mail(
+        'Return-Path: <noreply@airline.example>',
+        'From: Airline <noreply@airline.example>',
+        'To: traveller@example.com',
+        'Subject: Your booking',
+        'Date: Mon, 12 May 2025 09:00:00 +0200',
+        'MIME-Version: 1.0',
+        'Content-Type: text/html; charset=utf-8',
+        'Content-Transfer-Encoding: base64',
+        '',
+        b64('<html><body><p>Flight AB123</p></body></html>'),
+        '',
+      ),
+    );
+    expect(parsed?.html).toBe('<html><body><p>Flight AB123</p></body></html>');
+    expect(parsed?.text).toBeNull();
+    expect(parsed?.headerLines).toEqual([
+      'From: Airline <noreply@airline.example>',
+      'To: traveller@example.com',
+      'Subject: Your booking',
+      'Date: Mon, 12 May 2025 09:00:00 +0200',
+    ]);
+  });
+
+  it('decodes quoted-printable including soft line breaks', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: hotel@example.com',
+        'Subject: Reservation',
+        'Content-Type: text/plain; charset=utf-8',
+        'Content-Transfer-Encoding: quoted-printable',
+        '',
+        'Zimmer f=C3=BCr zwei Personen, Preis 120,00 =E2=82=AC. Buchungsnummer=',
+        ' XYZ-42',
+        '',
+      ),
+    );
+    expect(parsed?.text).toBe('Zimmer für zwei Personen, Preis 120,00 €. Buchungsnummer XYZ-42\n');
+  });
+
+  it('reads both parts of a multipart/alternative', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: rail@example.com',
+        'Subject: Ticket',
+        'MIME-Version: 1.0',
+        'Content-Type: multipart/alternative; boundary="=_alt_1"',
+        '',
+        'This is a multi-part message in MIME format.',
+        '--=_alt_1',
+        'Content-Type: text/plain; charset=utf-8',
+        'Content-Transfer-Encoding: quoted-printable',
+        '',
+        'Train IC 2043, coach 7',
+        '--=_alt_1',
+        'Content-Type: text/html; charset=utf-8',
+        'Content-Transfer-Encoding: base64',
+        '',
+        b64('<p>Train IC 2043, coach 7</p>'),
+        '--=_alt_1--',
+        '',
+      ),
+    );
+    expect(parsed?.text).toBe('Train IC 2043, coach 7');
+    expect(parsed?.html).toBe('<p>Train IC 2043, coach 7</p>');
+  });
+
+  it('walks nested multiparts and skips attachments', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: rental@example.com',
+        'Subject: Voucher',
+        'Content-Type: multipart/mixed; boundary=outer',
+        '',
+        '--outer',
+        'Content-Type: multipart/related; boundary="inner"',
+        '',
+        '--inner',
+        'Content-Type: text/html; charset=utf-8',
+        'Content-Transfer-Encoding: base64',
+        '',
+        b64('<p>Pickup 14:00</p>'),
+        '--inner--',
+        '--outer',
+        'Content-Type: application/pdf; name="voucher.pdf"',
+        'Content-Disposition: attachment; filename="voucher.pdf"',
+        'Content-Transfer-Encoding: base64',
+        '',
+        b64('%PDF-1.4 binary junk'),
+        '--outer--',
+        '',
+      ),
+    );
+    expect(parsed?.html).toBe('<p>Pickup 14:00</p>');
+    expect(parsed?.text).toBeNull();
+  });
+
+  it('skips a text part that is flagged as an attachment', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: a@example.com',
+        'Content-Type: multipart/mixed; boundary=b1',
+        '',
+        '--b1',
+        'Content-Type: text/plain; charset=utf-8',
+        'Content-Disposition: attachment; filename="notes.txt"',
+        '',
+        'attached notes',
+        '--b1--',
+        '',
+      ),
+    );
+    expect(parsed?.text).toBeNull();
+    expect(parsed?.html).toBeNull();
+  });
+
+  it('honours the part charset', () => {
+    const latin1 = Buffer.from('Hôtel Rivière, Zürich', 'latin1').toString('base64');
+    const parsed = parseEmail(
+      mail(
+        'From: hotel@example.com',
+        'Content-Type: text/plain; charset="iso-8859-1"',
+        'Content-Transfer-Encoding: base64',
+        '',
+        latin1,
+        '',
+      ),
+    );
+    expect(parsed?.text).toBe('Hôtel Rivière, Zürich');
+  });
+
+  it('falls back to utf8 for an unknown charset', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: hotel@example.com',
+        'Content-Type: text/plain; charset=x-made-up',
+        'Content-Transfer-Encoding: base64',
+        '',
+        b64('Zürich Hbf'),
+        '',
+      ),
+    );
+    expect(parsed?.text).toBe('Zürich Hbf');
+  });
+
+  it('decodes RFC 2047 encoded-words in the subject', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: =?utf-8?q?Fl=C3=BCge?= <a@example.com>',
+        `Subject: =?utf-8?B?${b64('Buchungsbestätigung')}?= =?utf-8?Q?_f=C3=BCr_M=C3=BCnchen?=`,
+        'Content-Type: text/plain',
+        '',
+        'body',
+        '',
+      ),
+    );
+    expect(parsed?.headerLines).toContain('Subject: Buchungsbestätigung für München');
+    expect(parsed?.headerLines).toContain('From: Flüge <a@example.com>');
+  });
+
+  it('unfolds headers that span multiple lines', () => {
+    const parsed = parseEmail(
+      mail(
+        'From: a@example.com',
+        'Subject: Booking',
+        '\tconfirmation 12345',
+        'Content-Type: text/plain;',
+        ' charset=utf-8',
+        '',
+        'body',
+        '',
+      ),
+    );
+    expect(parsed?.headerLines).toContain('Subject: Booking confirmation 12345');
+    expect(parsed?.text).toBe('body\n');
+  });
+
+  it('walks into a forwarded message/rfc822 part', () => {
+    const inner = [
+      'From: airline@example.com',
+      'Subject: Original',
+      'Content-Type: text/html; charset=utf-8',
+      'Content-Transfer-Encoding: base64',
+      '',
+      b64('<p>Seat 12A</p>'),
+    ].join('\r\n');
+    const parsed = parseEmail(
+      mail(
+        'From: colleague@example.com',
+        'Subject: Fwd: Original',
+        'Content-Type: multipart/mixed; boundary=fwd',
+        '',
+        '--fwd',
+        'Content-Type: message/rfc822',
+        '',
+        inner,
+        '--fwd--',
+        '',
+      ),
+    );
+    expect(parsed?.html).toBe('<p>Seat 12A</p>');
+  });
+
+  it('returns empty bodies for a multipart without a usable boundary', () => {
+    const parsed = parseEmail(
+      mail('From: a@example.com', 'Content-Type: multipart/mixed', '', 'nothing to split here', ''),
+    );
+    expect(parsed?.html).toBeNull();
+    expect(parsed?.text).toBeNull();
+  });
+
+  it('treats a message without a content-type as plain text', () => {
+    const parsed = parseEmail(mail('From: a@example.com', 'Subject: Hi', '', 'Booking 4711', ''));
+    expect(parsed?.text).toBe('Booking 4711\n');
+  });
+});
@@ -32,9 +32,198 @@ describe('text-extract', () => {
    expect(out).not.toContain('x{}');
  });

+  it('decodes html entities so the model reads the actual characters', async () => {
+    const html =
+      '<p>Zimmer f&uuml;r zwei &ndash; 120,00 &euro;</p><p>Gate &#88; / Terminal &#x58;</p>' +
+      '<p>&Uuml;berfahrt: Meier &amp; S&ouml;hne</p>';
+    const out = await extractText(Buffer.from(html, 'utf8'), 'a.html');
+    expect(out).toContain('Zimmer für zwei – 120,00 €');
+    expect(out).toContain('Gate X / Terminal X');
+    expect(out).toContain('Überfahrt: Meier & Söhne');
+    expect(out).not.toContain('&uuml;');
+    expect(out).not.toContain('&#88;');
+  });
+
+  it('turns non-breaking spaces into normal ones and leaves unknown entities alone', async () => {
+    const out = await extractText(
+      Buffer.from('<p>Sitz&nbsp;12A</p><p>Gleis&#160;7</p><p>&notanentity;</p>', 'utf8'),
+      'a.html',
+    );
+    expect(out).toContain('Sitz 12A');
+    expect(out).toContain('Gleis 7');
+    expect(out).toContain('&notanentity;');
+  });
+
+  it('resolves shouted entity names and keeps unusable numeric references', async () => {
+    const out = await extractText(Buffer.from('<p>Preis 12 &EURO; &#0; &#xD800; &#1114112;</p>', 'utf8'), 'a.html');
+    expect(out).toContain('Preis 12 €');
+    expect(out).toContain('&#0;');
+    expect(out).toContain('&#xD800;');
+    expect(out).toContain('&#1114112;');
+  });
+
+  it('decodes entities after the tags are gone, so escaped markup survives', async () => {
+    const out = await extractText(Buffer.from('<p>Regel: &lt;b&gt;fett&lt;/b&gt;</p>', 'utf8'), 'a.html');
+    expect(out).toContain('Regel: <b>fett</b>');
+  });
+
  it('extracts the embedded text layer from a pdf', async () => {
    const out = await extractText(Buffer.from('%PDF-1.4'), 'a.pdf');
    expect(out).toBe('Hotel X — confirmation ABC');
    expect(getText).toHaveBeenCalled();
  });
+
+  describe('eml', () => {
+    const mail = (...lines: string[]) => Buffer.from(lines.join('\r\n'), 'latin1');
+    const b64 = (s: string) => Buffer.from(s, 'utf8').toString('base64');
+
+    it('decodes a base64 html body instead of feeding the encoded blob to the model', async () => {
+      const body = b64('<html><body><h1>Flight AB123</h1><p>Seat 12A</p></body></html>');
+      const out = await extractText(
+        mail(
+          'From: Airline <noreply@airline.example>',
+          'Subject: Your booking',
+          'MIME-Version: 1.0',
+          'Content-Type: text/html; charset=utf-8',
+          'Content-Transfer-Encoding: base64',
+          '',
+          body,
+          '',
+        ),
+        'booking.eml',
+      );
+      expect(out).toContain('Flight AB123');
+      expect(out).toContain('Seat 12A');
+      expect(out).toContain('Subject: Your booking');
+      expect(out).not.toContain(body.slice(0, 24));
+      expect(out).not.toContain('<h1>');
+    });
+
+    it('decodes a quoted-printable html body', async () => {
+      const out = await extractText(
+        mail(
+          'From: hotel@example.com',
+          'Subject: Reservation',
+          'Content-Type: text/html; charset=utf-8',
+          'Content-Transfer-Encoding: quoted-printable',
+          '',
+          '<p>Zimmer f=C3=BCr zwei, 120,00 =E2=82=AC, Buchungsnummer=',
+          ' XYZ-42</p>',
+          '',
+        ),
+        'hotel.eml',
+      );
+      expect(out).toContain('Zimmer für zwei, 120,00 €, Buchungsnummer XYZ-42');
+      expect(out).not.toContain('=C3=BC');
+    });
+
+    it('prefers the html part of a multipart/alternative', async () => {
+      const out = await extractText(
+        mail(
+          'From: rail@example.com',
+          'Subject: Ticket',
+          'Content-Type: multipart/alternative; boundary="=_alt_1"',
+          '',
+          '--=_alt_1',
+          'Content-Type: text/plain; charset=utf-8',
+          '',
+          'plain fallback',
+          '--=_alt_1',
+          'Content-Type: text/html; charset=utf-8',
+          'Content-Transfer-Encoding: base64',
+          '',
+          b64('<p>Train IC 2043, coach 7</p>'),
+          '--=_alt_1--',
+          '',
+        ),
+        'ticket.eml',
+      );
+      expect(out).toContain('Train IC 2043, coach 7');
+      expect(out).not.toContain('plain fallback');
+    });
+
+    it('reads a plain-text mail', async () => {
+      const out = await extractText(
+        mail(
+          'From: ferry@example.com',
+          'Subject: Crossing',
+          'Content-Type: text/plain; charset=utf-8',
+          '',
+          'Booking 4711',
+          'Departure 08:30',
+          '',
+        ),
+        'ferry.eml',
+      );
+      expect(out).toContain('Booking 4711');
+      expect(out).toContain('Departure 08:30');
+    });
+
+    it('falls back to markup stripping when the .eml is not a MIME message', async () => {
+      const out = await extractText(Buffer.from('<html><body><p>Flight AB123</p></body></html>'), 'saved.eml');
+      expect(out).toBe('Flight AB123');
+    });
+
+    it('keeps the raw content when the mail carries no readable body part', async () => {
+      const attachment = b64('%PDF-1.4 ticket');
+      const out = await extractText(
+        mail(
+          'From: airline@example.com',
+          'Subject: Booking 4711',
+          'MIME-Version: 1.0',
+          'Content-Type: multipart/mixed; boundary="=_mix_1"',
+          '',
+          '--=_mix_1',
+          'Content-Type: application/pdf; name="ticket.pdf"',
+          'Content-Transfer-Encoding: base64',
+          'Content-Disposition: attachment; filename="ticket.pdf"',
+          '',
+          attachment,
+          '--=_mix_1--',
+          '',
+        ),
+        'attached.eml',
+      );
+      expect(out).toContain('Subject: Booking 4711');
+      expect(out).toContain(attachment);
+    });
+
+    it('uses the plain-text alternative when the html part is empty', async () => {
+      const out = await extractText(
+        mail(
+          'From: hotel@example.com',
+          'Subject: Reservation',
+          'Content-Type: multipart/alternative; boundary="=_alt_2"',
+          '',
+          '--=_alt_2',
+          'Content-Type: text/plain; charset=utf-8',
+          '',
+          'Zimmer 12, Anreise 04.08.',
+          '--=_alt_2',
+          'Content-Type: text/html; charset=utf-8',
+          '',
+          '<html><body></body></html>',
+          '--=_alt_2--',
+          '',
+        ),
+        'empty-html.eml',
+      );
+      expect(out).toContain('Zimmer 12, Anreise 04.08.');
+    });
+
+    it('decodes entities in an html mail body', async () => {
+      const out = await extractText(
+        mail(
+          'From: hotel@example.com',
+          'Subject: Reservierung',
+          'Content-Type: text/html; charset=utf-8',
+          '',
+          '<p>Zimmer f&uuml;r zwei &ndash; 120,00 &euro;</p>',
+          '',
+        ),
+        'entities.eml',
+      );
+      expect(out).toContain('Zimmer für zwei – 120,00 €');
+    });
+  });
 });