diff --git a/server/src/nest/llm-parse/mime-walker.ts b/server/src/nest/llm-parse/mime-walker.ts index e2f8fe530..2a68b85ee 100644 --- a/server/src/nest/llm-parse/mime-walker.ts +++ b/server/src/nest/llm-parse/mime-walker.ts @@ -388,6 +388,7 @@ function unwrapForwards(text: string): { text: string; layers: number } { function unwrapLayer(lines: string[]): { lines: string[]; changed: boolean } { const out: string[] = []; + const bareBlockStarts = bareHeaderBlockStarts(lines); let changed = false; let i = 0; while (i < lines.length) { @@ -399,7 +400,7 @@ function unwrapLayer(lines: string[]): { lines: string[]; changed: boolean } { if (opensBlock) { i = skipHeaderBlock(lines, i + 1); changed = true; - } else if (isBareHeaderBlock(lines, i)) { + } else if (bareBlockStarts[i]) { i = skipHeaderBlock(lines, i); changed = true; } else if (ATTRIBUTION.test(line)) { @@ -418,16 +419,31 @@ function unwrapLayer(lines: string[]): { lines: string[]; changed: boolean } { } /** - * A header block with no marker above it, as Outlook forwards: a sender line - * and at least two more header lines, one of them the subject. The subject is - * what tells it from a booking's own lines: a French ticket reads - * "De : Paris / À : Lyon / Date : 12/05" and must stay. + * The lines a header block with no marker above it starts on, as Outlook + * forwards: a sender line and at least two more header lines, one of them the + * subject. The subject is what tells it from a booking's own lines: a French + * ticket reads "De : Paris / À : Lyon / Date : 12/05" and must stay. + * + * Worked out for every line in one pass from the bottom up, carrying the end of + * the header run below and whether it holds a subject. Scanning the run again + * from each line would make a mail of nothing but sender lines quadratic, and + * the walk runs on whatever arrives at the ingest address. */ -function isBareHeaderBlock(lines: string[], start: number): boolean { - if (!FROM_LINE.test(lines[start])) return false; - let end = start; - while (end < lines.length && HEADER_LINE.test(lines[end])) end++; - return end - start >= MIN_BARE_HEADER_BLOCK && lines.slice(start, end).some((line) => SUBJECT_LINE.test(line)); +function bareHeaderBlockStarts(lines: string[]): boolean[] { + const starts = new Array(lines.length).fill(false); + let runEnd = lines.length; + let subjectInRun = false; + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i]; + if (!HEADER_LINE.test(line)) { + runEnd = i; + subjectInRun = false; + continue; + } + if (SUBJECT_LINE.test(line)) subjectInRun = true; + starts[i] = subjectInRun && runEnd - i >= MIN_BARE_HEADER_BLOCK && FROM_LINE.test(line); + } + return starts; } /** The index after a header block starting at `start`: its header lines, their folded continuations, one blank line. */ diff --git a/server/tests/unit/nest/llm-parse/mime-walker.test.ts b/server/tests/unit/nest/llm-parse/mime-walker.test.ts index 8736db373..0c1cadfe7 100644 --- a/server/tests/unit/nest/llm-parse/mime-walker.test.ts +++ b/server/tests/unit/nest/llm-parse/mime-walker.test.ts @@ -240,6 +240,41 @@ describe('walkMail: forwarded and quoted messages', () => { expect(walked.body).toBe(''); expect(walked.forwardLayers).toBe(10); }); + + it('MIME-WALK-016: a run of sender lines ending in a subject goes, the run without one under it stays', () => { + const walked = walk( + mail( + 'From: ana@example.org', + 'Content-Type: text/plain; charset=utf-8', + '', + 'From: a', + 'From: b', + 'From: Ferry ', + 'Subject: Ticket', + '', + 'From: Naxos', + 'To: Piraeus', + '', + 'Ferry ticket F-12', + '', + ), + ); + expect(walked.body).toBe('From: Naxos\nTo: Piraeus\n\nFerry ticket F-12'); + expect(walked.forwardLayers).toBe(1); + }); + + it( + 'MIME-WALK-017: a mail of nothing but sender lines is read in one pass instead of rescanning them per line', + { timeout: 5_000 }, + () => { + const lines = 30_000; + const walked = walk( + mail('From: ana@example.org', 'Content-Type: text/plain', '', ...Array(lines).fill('From: a'), ''), + ); + expect(walked.body.split('\n')).toHaveLength(lines); + expect(walked.forwardLayers).toBe(0); + }, + ); }); describe('walkMail: attachments', () => {