mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-21 03:26:07 +00:00
fix(mail-ingest): Outlook HTML forwards, the default AI limit, secret env vars, and the pipeline tests
- mime-walker: trim each line of a stripped HTML body, so the space a tag leaves is not read as a folded header line that swallows the booking under an Outlook forward's header block. - settings: an unset daily AI limit is the default of ten, not zero (Number(null) is 0), which kept the AI from ever reading a mail. - env: MAIL_INGEST_IMAP_PASS and MAIL_INGEST_WEBHOOK_SECRET are secrets and never printed in the boot report. - pipeline test: the subject is kept without its forward prefixes, the demo move adds its booking before the second trip exists, the typed notification mock, and AI reading stays offered while the person switched it off. - e2e: the pipeline tick runs in a request context of its own, as the cron registrar runs it, and the 401 checks run without awaiting in a loop.
This commit is contained in:
@@ -33,6 +33,8 @@ export const SECRET_ENV_KEYS: ReadonlySet<string> = new Set([
|
||||
'MAPBOX_ACCESS_TOKEN',
|
||||
'CARTO_API_KEY',
|
||||
'DEMO_ADMIN_PASS',
|
||||
'MAIL_INGEST_IMAP_PASS',
|
||||
'MAIL_INGEST_WEBHOOK_SECRET',
|
||||
]);
|
||||
|
||||
/** How a variable's value appears in the boot report. */
|
||||
|
||||
@@ -284,9 +284,20 @@ function safeFileName(name: string, kind: MailAttachmentKind, position: number):
|
||||
/** Line breaks and block ends, so the header lines of a forwarded HTML mail stay lines without their tags. */
|
||||
const HTML_LINE_BREAK = /<(?:br|hr)\b[^>]*>|<\/(?:p|div|tr|li|h[1-6]|blockquote|table)\s*>/gi;
|
||||
|
||||
/** A section as text: its HTML stripped when it has a readable one, else its plain text. */
|
||||
/**
|
||||
* A section as text: its HTML stripped when it has a readable one, else its plain text.
|
||||
* Each line of the HTML is trimmed: a stripped tag leaves a space in front of a
|
||||
* line, which the forward unwrapping would read as a folded header continuation,
|
||||
* dropping the booking under a forwarded header block with it. HTML folds no headers.
|
||||
*/
|
||||
function sectionText(section: Section): string {
|
||||
const html = section.html === null ? '' : stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n'));
|
||||
const html =
|
||||
section.html === null
|
||||
? ''
|
||||
: stripMarkup(section.html.replace(HTML_LINE_BREAK, '\n'))
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.join('\n');
|
||||
return html || (section.text === null ? '' : collapseWhitespace(section.text));
|
||||
}
|
||||
|
||||
|
||||
@@ -100,8 +100,10 @@ export class MailIngestSettingsService {
|
||||
return this.config.webhookSecret ?? (decrypt_api_key(stored) || null);
|
||||
}
|
||||
|
||||
/** The AI reads a person gets a day; an unset or blank setting is the default, not zero (`Number('')` is 0). */
|
||||
async aiDailyCap(): Promise<number> {
|
||||
const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily'));
|
||||
const stored = (await this.settings.getAppSetting('mail_ingest_ai_per_user_daily'))?.trim();
|
||||
const value = stored ? Number(stored) : Number.NaN;
|
||||
return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import { importExternalId } from '../../src/nest/booking-import/booking-origin';
|
||||
import { encrypt_api_key } from '../../src/nest/common/crypto/apiKeyCrypto';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
|
||||
import { withRequestContext } from '../../src/nest/database/request-context';
|
||||
import { FilesService } from '../../src/nest/files/files.service';
|
||||
import { MailReadService } from '../../src/nest/llm-parse/mail-read.service';
|
||||
import { walkMail } from '../../src/nest/llm-parse/mime-walker';
|
||||
@@ -29,6 +30,7 @@ import { makeAdmin, makeUser } from '../helpers/factories/users';
|
||||
import { createTestMikroOrmModule, createTestOrm, type TestOrm } from '../helpers/test-orm';
|
||||
import { TestUnitOfWorkModule } from '../helpers/test-uow';
|
||||
import { sessionCookie } from './harness';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import type { BookingImportPreviewItem } from '@trek/shared';
|
||||
|
||||
@@ -128,8 +130,11 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => {
|
||||
|
||||
const as = (userId: number) => ({ Cookie: sessionCookie(userId) });
|
||||
|
||||
/** Runs the pipeline over what came in, the way its two-minute job does. */
|
||||
const readQueued = () => app.get(MailIngestProcessJob).tick();
|
||||
/**
|
||||
* Runs the pipeline over what came in, the way its two-minute job does: the
|
||||
* cron registrar runs every tick in a request context of its own.
|
||||
*/
|
||||
const readQueued = () => withRequestContext(app.get(MikroORM), () => app.get(MailIngestProcessJob).tick());
|
||||
|
||||
beforeAll(async () => {
|
||||
orm = await createTestOrm(db);
|
||||
@@ -150,9 +155,9 @@ describe('/api/mail-ingest e2e (real guards + snapshot SQLite)', () => {
|
||||
});
|
||||
|
||||
it('MAIL-E2E-001: 401 without a session on every route a person or an admin uses', async () => {
|
||||
for (const path of ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status']) {
|
||||
expect((await request(server).get(path)).status, path).toBe(401);
|
||||
}
|
||||
const paths = ['/api/mail-ingest/settings', '/api/mail-ingest/inbox', '/api/admin/mail-ingest/status'];
|
||||
const answers = await Promise.all(paths.map((path) => request(server).get(path)));
|
||||
answers.forEach((answer, i) => expect(answer.status, paths[i]).toBe(401));
|
||||
expect((await request(server).post(`/api/trips/${tripId}/mail-ingest/address`)).status).toBe(401);
|
||||
});
|
||||
|
||||
|
||||
@@ -84,7 +84,10 @@ const files = {
|
||||
createFileFromBytes: vi.fn(async () => ({ id: 77 })),
|
||||
softDeleteFile: vi.fn(async () => undefined),
|
||||
};
|
||||
const notifications = { send: vi.fn(async () => ({ attempted: 1, delivered: 1 })), registerResponseAction: vi.fn() };
|
||||
const notifications = {
|
||||
send: vi.fn(async (_payload: { event: string; scope: string }) => ({ attempted: 1, delivered: 1 })),
|
||||
registerResponseAction: vi.fn(),
|
||||
};
|
||||
|
||||
const db = createSnapshotTestDb();
|
||||
let t: TestModule;
|
||||
@@ -322,8 +325,10 @@ describe('a booking that passes every check', () => {
|
||||
it('is not moved by a demo account', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' });
|
||||
// The booking is added to the one trip in its dates first; a second one then would leave it waiting.
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
expect(entry).toMatchObject({ resolution: 'auto_applied' });
|
||||
const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' });
|
||||
const demo = vi.spyOn(t.get(DemoService), 'isDemoUserId').mockResolvedValue(true);
|
||||
try {
|
||||
await expect(actions().move(entry.id, user.id, second.id, true, NOW)).rejects.toThrow(
|
||||
@@ -509,7 +514,8 @@ describe('what the person sees', () => {
|
||||
reason: 'ask_first',
|
||||
suggested_trip: { id: trip.id, title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' },
|
||||
undo_until: null,
|
||||
message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Fwd: Your flight LH 716' },
|
||||
// The subject is kept without the prefixes forwarding stacked onto it.
|
||||
message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Your flight LH 716' },
|
||||
});
|
||||
expect(page.entries[0].message).not.toHaveProperty('raw_eml_enc');
|
||||
expect((await inbox().inbox(other.id, { status: 'open', limit: 50 }, NOW)).entries).toEqual([]);
|
||||
@@ -538,7 +544,7 @@ describe('what the person sees', () => {
|
||||
const messageId = await forward(user.id, eml('nothing'));
|
||||
|
||||
const preview = await inbox().preview(user.id, messageId);
|
||||
expect(preview).toMatchObject({ id: messageId, subject: 'Fwd: Your flight LH 716', attachments: [] });
|
||||
expect(preview).toMatchObject({ id: messageId, subject: 'Your flight LH 716', attachments: [] });
|
||||
expect(preview.text).toContain('Booking reference ABC123');
|
||||
expect(preview.text).not.toContain('mia@example.com');
|
||||
await expect(inbox().preview(other.id, messageId)).rejects.toThrow('Mail not found');
|
||||
@@ -561,6 +567,18 @@ describe('what the person sees', () => {
|
||||
expect(await inbox().count(user.id)).toEqual({ pending: 0 });
|
||||
});
|
||||
|
||||
it('keeps AI reading offered while the person switched it off, so the switch can come back on', async () => {
|
||||
const user = await person();
|
||||
reader.isAvailable.mockResolvedValue(true);
|
||||
const updated = await inbox().updatePrefs(user.id, { ai_enabled: false });
|
||||
expect(updated).toMatchObject({ prefs: { ai_enabled: false }, ai_available: true });
|
||||
expect(await inbox().settingsOf(user.id)).toMatchObject({
|
||||
available: true,
|
||||
prefs: { enabled: true, ai_enabled: false, auto_add: true },
|
||||
ai_available: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('reads an uploaded mail at once, knows one it has, and refuses what it cannot take', async () => {
|
||||
const user = await person();
|
||||
const soon = vi.spyOn(job(), 'runSoon');
|
||||
|
||||
Reference in New Issue
Block a user