diff --git a/server/src/db/json-columns.ts b/server/src/db/json-columns.ts index 9fc84d252..2040d88b6 100644 --- a/server/src/db/json-columns.ts +++ b/server/src/db/json-columns.ts @@ -6,8 +6,15 @@ import { assistantChangePreviewWriteSchema, assistantMessagePartWriteSchema, assistantStoredPartsSchema, + bookingImportPreviewItemSchema, ferrataSchema, hikingPreferencesSchema, + mailIngestAttachmentSchema, + mailIngestCheckResultSchema, + mailIngestCheckSchema, + mailIngestFieldSchema, + mailIngestHintSchema, + mailIngestSuggestedTripSchema, obstacleSnapshotSchema, paddleGeometrySchema, paddleLockSchema, @@ -26,9 +33,15 @@ import { type DawarichTrackSegment, type AssistantChangePreview, type AssistantMessagePart, + type BookingImportPreviewItem, type Ferrata, type GeometryPoint, type HikingPreferences, + type MailIngestAttachment, + type MailIngestCheck, + type MailIngestCheckResult, + type MailIngestField, + type MailIngestHint, type ObstacleSnapshot, type PaddleGeometry, type PaddleLock, @@ -531,3 +544,73 @@ export const PLACE_DETAILS_CACHE_PAYLOAD: JsonColumn | n schema: z.record(z.string(), z.unknown()).nullable(), fallback: () => null, }; + +// ── The booking mail import (mail-ingest) ────────────────────────────────── + +/** The booking read from a mail, as the inbox and the form show it; null for an entry that stands for the mail. */ +export const MAIL_INGEST_ITEM: JsonColumn = { + column: 'mail_ingest_candidates.item_json', + schema: bookingImportPreviewItemSchema.nullable(), + fallback: () => null, +}; + +/** The server's checks on the booking, by name. A broken row reads as none run. */ +export const MAIL_INGEST_CHECKS: JsonColumn>> = { + column: 'mail_ingest_candidates.checks_json', + schema: z.partialRecord(mailIngestCheckSchema, mailIngestCheckResultSchema), + fallback: () => ({}), +}; + +/** The fields of the booking marked for a second look. */ +export const MAIL_INGEST_UNCERTAIN_FIELDS: JsonColumn = { + column: 'mail_ingest_candidates.uncertain_fields', + schema: z.array(mailIngestFieldSchema), + fallback: () => [], +}; + +const mailIngestMatchSchema = z.object({ + /** A change or cancellation of an earlier booking rather than a new one. */ + kind: z.enum(['new_booking', 'change', 'cancellation']), + /** The trip the AI proposed to create when none fits; never created on its own. */ + suggested_new_trip: mailIngestSuggestedTripSchema.nullable(), +}); +export type MailIngestMatch = z.infer; + +/** How the booking was matched to a trip, past the suggested trip's own column. */ +export const MAIL_INGEST_MATCH: JsonColumn = { + column: 'mail_ingest_candidates.match_json', + schema: mailIngestMatchSchema, + fallback: () => ({ kind: 'new_booking', suggested_new_trip: null }), +}; + +const idList = z.array(z.number().int().positive()); +const mailIngestCreatedSchema = z.object({ + reservation_ids: idList, + place_ids: idList, + budget_item_ids: idList, + file_ids: idList, + /** The booking as it was written, to tell an edit since then. */ + fingerprint: z.string().nullable(), +}); +export type MailIngestCreated = z.infer; + +/** What adding the booking created, for its undo. A broken row undoes nothing. */ +export const MAIL_INGEST_CREATED: JsonColumn = { + column: 'mail_ingest_candidates.created_ids_json', + schema: mailIngestCreatedSchema, + fallback: () => ({ reservation_ids: [], place_ids: [], budget_item_ids: [], file_ids: [], fingerprint: null }), +}; + +/** What the inbox notes about a mail without holding it back. */ +export const MAIL_INGEST_HINTS: JsonColumn = { + column: 'mail_ingest_messages.hints', + schema: z.array(mailIngestHintSchema), + fallback: () => [], +}; + +/** The attachments a mail carried that a booking travels in, without their bytes. */ +export const MAIL_INGEST_ATTACHMENTS: JsonColumn = { + column: 'mail_ingest_messages.attachments', + schema: z.array(mailIngestAttachmentSchema), + fallback: () => [], +}; diff --git a/server/src/db/repositories/MailIngestCandidates.repository.ts b/server/src/db/repositories/MailIngestCandidates.repository.ts index 74f392f4b..9e7fdce50 100644 --- a/server/src/db/repositories/MailIngestCandidates.repository.ts +++ b/server/src/db/repositories/MailIngestCandidates.repository.ts @@ -49,6 +49,11 @@ export interface MailIngestCandidateInsert { resolution: string; reason: string | null; createdAt: DbTimestamp; + /** For an entry decided as it is written: the trip it went to, what it created, until when it can be undone. */ + resolvedTripId?: number | null; + createdIdsJson?: string | null; + resolvedAt?: DbTimestamp | null; + undoUntil?: DbTimestamp | null; } /** What deciding an entry writes. */ @@ -107,6 +112,8 @@ type CandidatesKyselyDB = Pick< | 'places' | 'users' | 'reservations' + | 'day_assignments' + | 'day_accommodations' | 'budget_item_payers' >; @@ -144,6 +151,10 @@ export class MailIngestCandidatesRepository extends TrekRepository> { + /** One trip the person owns or is a member of and that is not archived, whatever its dates; null otherwise. */ + async findOpenTrip(userId: number, tripId: number): Promise { + const row = await this.db() + .selectFrom('trips as t') + .select(['t.id', 't.title', 't.start_date', 't.end_date', 't.user_id', 't.currency']) + .where('t.id', '=', tripId) + .where((eb) => + eb.or([ + eb('t.user_id', '=', userId), + eb.exists( + eb + .selectFrom('trip_members as m') + .select('m.id') + .whereRef('m.trip_id', '=', 't.id') + .where('m.user_id', '=', userId), + ), + ]), + ) + .where((eb) => eb.or([eb('t.is_archived', 'is', null), eb('t.is_archived', '=', 0)])) + .executeTakeFirst(); + return row ?? null; + } + + /** The dated days of these trips, by date. */ + async listDatedDays(tripIds: readonly number[]): Promise> { if (tripIds.length === 0) return []; const rows = await this.db() .selectFrom('days') - .select(['trip_id', 'date']) + .select(['id', 'trip_id', 'date']) .where('trip_id', 'in', [...tripIds]) .where('date', 'is not', null) .orderBy('trip_id', 'asc') .orderBy('date', 'asc') .execute(); - return rows.flatMap((row) => (row.date === null ? [] : [{ trip_id: row.trip_id, date: row.date }])); + return rows.flatMap((row) => (row.date === null ? [] : [{ id: row.id, trip_id: row.trip_id, date: row.date }])); } /** The places of these trips that have a position, oldest first. */ @@ -346,6 +380,16 @@ export class MailIngestCandidatesRepository extends TrekRepository { + const [booking, planned, stay] = await Promise.all([ + this.db().selectFrom('reservations').select('id').where('place_id', '=', placeId).executeTakeFirst(), + this.db().selectFrom('day_assignments').select('id').where('place_id', '=', placeId).executeTakeFirst(), + this.db().selectFrom('day_accommodations').select('id').where('place_id', '=', placeId).executeTakeFirst(), + ]); + return booking !== undefined || planned !== undefined || stay !== undefined; + } + /** How many payers the costs name: a cost someone settled with moves balances. */ async countPayers(budgetItemIds: readonly number[]): Promise { if (budgetItemIds.length === 0) return 0; diff --git a/server/src/db/repositories/MailIngestMessages.repository.ts b/server/src/db/repositories/MailIngestMessages.repository.ts index 6f6a415c3..08ea85d58 100644 --- a/server/src/db/repositories/MailIngestMessages.repository.ts +++ b/server/src/db/repositories/MailIngestMessages.repository.ts @@ -158,22 +158,21 @@ export class MailIngestMessagesRepository extends TrekRepository ({ status: 'processing', lease_until: leaseUntil, attempts: eb('attempts', '+', 1) })) - .where('id', '=', id) - .where((eb) => - eb.or([ - eb('status', '=', 'queued'), - eb.and([eb('status', '=', 'processing'), eb('lease_until', '<', now)]), - ]), - ) - .executeTakeFirst(); - if (Number(result.numUpdatedRows) > 0) claimed.push(id); - } - return claimed; + const won = await Promise.all(due.map(({ id }) => this.claim(id, now, leaseUntil))); + return due.flatMap(({ id }, i) => (won[i] ? [id] : [])); + } + + /** One mail's claim: it is still queued, or its last claim ran out at `now`. */ + private async claim(id: number, now: DbTimestamp, leaseUntil: DbTimestamp): Promise { + const result = await this.db() + .updateTable('mail_ingest_messages') + .set((eb) => ({ status: 'processing', lease_until: leaseUntil, attempts: eb('attempts', '+', 1) })) + .where('id', '=', id) + .where((eb) => + eb.or([eb('status', '=', 'queued'), eb.and([eb('status', '=', 'processing'), eb('lease_until', '<', now)])]), + ) + .executeTakeFirst(); + return Number(result.numUpdatedRows) > 0; } /** Writes what the pipeline made of the mail and lets go of its claim. */ diff --git a/server/src/nest/app.module.ts b/server/src/nest/app.module.ts index bb79f4d46..d39dbe77e 100644 --- a/server/src/nest/app.module.ts +++ b/server/src/nest/app.module.ts @@ -58,6 +58,7 @@ import { JourneyAccessModule } from './journey/journey-access.module'; import { JourneyModule } from './journey/journey.module'; import { TripJournalLinkModule } from './journey/trip-journal-link.module'; import { LlmParseModule } from './llm-parse/llm-parse.module'; +import { MailIngestModule } from './mail-ingest/mail-ingest.module'; import { ManagedExtModule } from './managed/managed-ext.module'; import { MapsModule } from './maps/maps.module'; import { McpTransportModule } from './mcp-transport/mcp-transport.module'; @@ -209,6 +210,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core'; ReservationImportModule, ReceiptScanModule, LlmParseModule, + MailIngestModule, AiConnectionsModule, AssistantModule, ManagedExtModule, diff --git a/server/src/nest/common/app-settings.registry.ts b/server/src/nest/common/app-settings.registry.ts index 248c23283..a4dca77b0 100644 --- a/server/src/nest/common/app-settings.registry.ts +++ b/server/src/nest/common/app-settings.registry.ts @@ -1,6 +1,6 @@ import { readEnv } from '../../app-config'; import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; -import { aiPolicySchema } from '@trek/shared'; +import { aiPolicySchema, mailIngestTransportSettingSchema } from '@trek/shared'; import { z } from 'zod'; @@ -161,6 +161,15 @@ export const APP_SETTINGS = { synology_capture_repair_3019: { schema: text, default: null }, /** How many starts have tried that so far, while some photo could not be confirmed. */ synology_capture_repair_3019_tries: { schema: intText, default: null }, + // The booking mail import (mail-ingest). Off until switched on: on a self-hosted install by its + // admin, on a managed one by the operator, for the plans that include it and never for Start. + mail_ingest_enabled: { schema: flag, default: 'false' }, + /** How mail reaches the install: not at all (uploads only), IMAP, a signed webhook, or the TREK Cloud relay. */ + mail_ingest_transport: { schema: mailIngestTransportSettingSchema, default: 'off' }, + /** The forwarding address, `{token}` standing where a person's token goes: `trek+{token}@example.com`. */ + mail_ingest_address_template: { schema: text, default: null }, + /** How many forwarded mails a day an AI reads for one person. */ + mail_ingest_ai_per_user_daily: { schema: intText, default: '10' }, } as const satisfies Record; export type AppSettingKey = keyof typeof APP_SETTINGS; diff --git a/server/src/nest/llm-parse/mail-read.service.ts b/server/src/nest/llm-parse/mail-read.service.ts index da9770d71..b8e79cedf 100644 --- a/server/src/nest/llm-parse/mail-read.service.ts +++ b/server/src/nest/llm-parse/mail-read.service.ts @@ -1,3 +1,4 @@ +import { logError } from '../audit/audit-log.logger'; import type { KiReservation } from '../booking-import/kitinerary.types'; import { capImage, renderPdfPages } from './image-input'; import { createLlmClient } from './llm-client.factory'; @@ -96,7 +97,7 @@ export class MailReadService { } catch (err) { // The provider and the error only: the mail's content stays out of the log. const reason = err instanceof Error ? err.message : String(err); - console.error(`[llm-parse] Mail read failed (provider=${config.provider}): ${reason}`); + logError(`[llm-parse] Mail read failed (provider=${config.provider}): ${reason}`); return nothing(failureOutcome(err)); } } diff --git a/server/src/nest/llm-parse/reservation-normalize.ts b/server/src/nest/llm-parse/reservation-normalize.ts index 2fc73a66c..ee91fdcfb 100644 --- a/server/src/nest/llm-parse/reservation-normalize.ts +++ b/server/src/nest/llm-parse/reservation-normalize.ts @@ -53,7 +53,10 @@ export function normalizeNode(node: Record): Record, documentText: string | undefined): Record { +export function normalizeCurrency( + node: Record, + documentText: string | undefined, +): Record { const hasPrice = node.price != null && node.price !== ''; if (node.priceCurrency == null && !hasPrice) return node; const code = diff --git a/server/src/nest/mail-ingest/mail-classify.ts b/server/src/nest/mail-ingest/mail-classify.ts new file mode 100644 index 000000000..8a55941de --- /dev/null +++ b/server/src/nest/mail-ingest/mail-classify.ts @@ -0,0 +1,75 @@ +import type { WalkedMail } from '../llm-parse/mail-read.types'; + +/** + * What a mail is before anything reads it for bookings: a forwarding + * confirmation from Gmail or Outlook (its code is shown, nothing is clicked), + * mail that is no booking (a newsletter, an automatic reply, an empty mail), + * or a mail to read. + */ +export type MailClass = + | { kind: 'read' } + | { kind: 'forwarding_confirmation'; code: string | null } + | { kind: 'drop'; reason: 'newsletter' | 'auto_reply' | 'empty_mail' }; + +const FORWARDING_SENDERS = /forwarding-noreply@google\.com|@accountprotection\.microsoft\.com|@microsoft\.com/i; +const FORWARDING_SUBJECT = + /(forwarding|weiterleitung|transfert|reenv[ií]o|inoltro|doorsturen).*(confirm|bestätig|verif|conferma|bevestig)/i; +const CODE_RE = /(?:code|código|codice|kód)\D{0,40}?(\d{6,10})\b/i; +const AUTO_REPLY_SUBJECT = + /^(automatic reply|auto(matische)? ?(reply|antwort)|out of (the )?office|abwesenheit|r[ée]ponse automatique|risposta automatica|respuesta autom[áa]tica)/i; +/** + * What makes a list mail worth reading anyway. An automatic forward keeps the + * original headers, and a booking confirmation often carries List-Unsubscribe + * just like a newsletter; an attachment or a booking word keeps it in. + */ +const BOOKING_WORDS = + /\b(booking|reservation|confirmation|itinerary|e-?ticket|boarding|check-in|pnr|buchung|reservierung|bestätigung|réservation|reserva|prenotazione|boeking|rezerwacja)\b/i; +/** Less body than this, and no attachment, is an empty mail. */ +const EMPTY_BODY_CHARS = 20; + +export function classifyMail(mail: WalkedMail): MailClass { + const { envelope } = mail; + const subject = mail.subject ?? ''; + const fromProvider = FORWARDING_SENDERS.test(envelope.from ?? '') && CODE_RE.test(mail.body); + if (FORWARDING_SUBJECT.test(subject) || fromProvider) { + return { kind: 'forwarding_confirmation', code: CODE_RE.exec(mail.body)?.[1] ?? null }; + } + const autoSubmitted = (envelope.autoSubmitted ?? '').trim().toLowerCase(); + const precedence = (envelope.precedence ?? '').trim().toLowerCase(); + // `auto-replied` only (RFC 3834): a booking system marks its confirmations `auto-generated`. + if (autoSubmitted.startsWith('auto-replied') || precedence === 'auto_reply' || AUTO_REPLY_SUBJECT.test(subject)) { + return { kind: 'drop', reason: 'auto_reply' }; + } + if (mail.body.trim().length < EMPTY_BODY_CHARS && mail.attachments.length === 0) { + return { kind: 'drop', reason: 'empty_mail' }; + } + const listMail = !!envelope.listUnsubscribe || ['bulk', 'list', 'junk'].includes(precedence); + const bookingSignal = mail.attachments.length > 0 || BOOKING_WORDS.test(`${subject}\n${mail.body}`); + if (listMail && !bookingSignal) return { kind: 'drop', reason: 'newsletter' }; + return { kind: 'read' }; +} + +/** + * Whether the mail came from the person themselves, signed: the From address + * is their account's and the receiving server saw a DKIM or DMARC pass for + * its domain. Only ever a hint shown with the mail (owner decision + * 13.11.2026), never a reason to stop. + */ +export function senderVerified(mail: WalkedMail, accountEmail: string): boolean { + const from = addressOf(mail.envelope.from); + if (!from || from !== accountEmail.trim().toLowerCase()) return false; + const domain = from.split('@')[1] ?? ''; + const results = (mail.envelope.authenticationResults ?? '').toLowerCase(); + const dkim = new RegExp(`dkim=pass[^;]*header\\.(d|i)=@?${escapeRegExp(domain)}\\b`); + return dkim.test(results) || /dmarc=pass/.test(results); +} + +/** The bare address of a From header: `Mia ` is `mia@example.com`. */ +export function addressOf(header: string | null): string | null { + const match = /<([^>]+)>/.exec(header ?? '') ?? /([^\s<>"]+@[^\s<>"]+)/.exec(header ?? ''); + return match ? match[1].trim().toLowerCase() : null; +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} diff --git a/server/src/nest/mail-ingest/mail-ingest-actions.service.ts b/server/src/nest/mail-ingest/mail-ingest-actions.service.ts new file mode 100644 index 000000000..c344745dc --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-actions.service.ts @@ -0,0 +1,289 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import { + MAIL_INGEST_CREATED, + MAIL_INGEST_ITEM, + MAIL_INGEST_UNCERTAIN_FIELDS, + type MailIngestCreated, +} from '../../db/json-columns'; +import type { + MailIngestCandidateRow, + MailIngestCandidatesRepository, +} from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow, parseDbTimestamp } from '../../db/types'; +import { decodeJson, encodeJson } from '../../utils/json-column'; +import { AirportsService } from '../airports/airports.service'; +import { DemoService } from '../common/demo.service'; +import { DomainError } from '../common/domain-error'; +import { UnitOfWork } from '../database/unit-of-work'; +import { MailReadService } from '../llm-parse/mail-read.service'; +import type { WalkedMail } from '../llm-parse/mail-read.types'; +import { MailIngestApplyService } from './mail-ingest-apply.service'; +import { implausible, type TripOption } from './mail-ingest-checks'; +import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { IMPORT_SOURCES, UNDO_WINDOW_DAYS } from './mail-ingest.constants'; +import { openMail } from './mail-seal'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import type { BookingImportPreviewItem } from '@trek/shared'; + +const DAY_MS = 24 * 60 * 60 * 1000; +/** The states an added booking is undone from. */ +const ADDED = ['auto_applied', 'applied'] as const; +const ISO_DATE_RE = /\b(20\d\d-[01]\d-[0-3]\d)\b/; +/** A booking reference as a mail writes it: after its label, letters and digits. */ +const REFERENCE_RE = + /(?:booking (?:reference|code|number)|confirmation (?:code|number)|reservation (?:code|number)|pnr|buchungsnummer|buchungscode|référence|localizador)\W{0,5}([A-Z0-9-]{5,20})/i; +const TYPE_WORDS: ReadonlyArray<[RegExp, string]> = [ + [/\b(flight|flug|vol|vuelo|volo|boarding)\b/i, 'flight'], + [/\b(train|zug|bahn|treno|tren|rail)\b/i, 'train'], + [/\b(hotel|apartment|check-in|unterkunft|hébergement|alojamiento)\b/i, 'hotel'], + [/\b(rental car|car hire|mietwagen|location de voiture)\b/i, 'car'], + [/\b(restaurant|table)\b/i, 'restaurant'], + [/\b(ticket|event|concert|konzert|museum)\b/i, 'event'], +]; + +/** + * What a person does with an entry of their mail inbox: add it to a trip, + * file the mail as documents, fill the booking form in themselves, discard + * it, or undo a booking a mail added. Every action checks again that the + * person may add bookings to the trip it touches, and that every id belongs + * to that trip. + */ +@Injectable() +export class MailIngestActionsService { + constructor( + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + private readonly apply: MailIngestApplyService, + private readonly trips: MailIngestTripsService, + private readonly reader: MailReadService, + private readonly airports: AirportsService, + private readonly demo: DemoService, + private readonly uow: UnitOfWork, + ) {} + + /** + * Removes exactly what adding the booking created, within the undo window, + * for the person who forwarded it or any member who may edit the trip's + * bookings. A booking whose cost moved balances, or that was edited since, + * is only undone with `confirm`. + * + * @txIndependent the removal is one transaction; the documents go to the + * trash after it, as storage I/O. + */ + async undo( + candidateId: number, + userId: number, + confirm: boolean, + now: Date = new Date(), + ): Promise { + const entry = await this.candidates.findById(candidateId); + const tripId = entry?.resolved_trip_id ?? null; + if (!entry || tripId === null || !(await this.editableBy(userId, tripId))) { + throw new DomainError(404, 'Booking not found'); + } + if (!(ADDED as readonly string[]).includes(entry.resolution)) { + throw new DomainError(409, 'This booking is not added from a mail any more'); + } + const until = parseDbTimestamp(entry.undo_until); + if (!until || until < now) throw new DomainError(409, 'The time to undo this booking has passed'); + + const created = decodeJson(MAIL_INGEST_CREATED, entry.created_ids_json); + const affects = await this.undoAffects(tripId, created); + if (affects.length > 0 && !confirm) { + throw new DomainError(409, 'Undoing this booking changes more than the booking', { + details: { code: 'needs_confirmation', affects }, + }); + } + const removed = await this.uow.transactional(async () => { + const claimed = await this.candidates.resolve(entry.id, ADDED, { + resolution: 'undone', + reason: null, + resolvedTripId: tripId, + createdIdsJson: entry.created_ids_json, + resolvedBy: userId, + resolvedAt: dbNow(now), + undoUntil: null, + }); + if (!claimed) throw new DomainError(409, 'This booking was undone already'); + return this.apply.removeInTx(tripId, created); + }); + await this.apply.announceRemoved(removed, created.file_ids); + return this.reload(entry.id); + } + + /** + * Adds a waiting booking to the trip the inbox suggested or the person + * picked. The person decides the trip; the server still checks that they + * may add bookings there, that the booking is plausible and that the trip + * does not hold it already. + * + * @txIndependent the booking and the entry are one transaction; the mail's + * documents are attached after it, as storage I/O, and noted on the entry. + */ + async applyToTrip( + candidateId: number, + userId: number, + tripId: number | null, + now: Date = new Date(), + ): Promise { + const entry = await this.pendingEntry(candidateId, userId); + const item = decodeJson(MAIL_INGEST_ITEM, entry.item_json); + if (!item) throw new DomainError(409, 'This mail holds no booking to add; file it as a document instead'); + if (await this.demo.isDemoUserId(userId)) throw new DomainError(403, 'Demo accounts cannot add bookings'); + const trip = await this.tripFor(userId, tripId ?? entry.suggested_trip_id); + if (implausible(item, now, (code) => this.airports.findByIata(code) !== null)) { + throw new DomainError(422, 'This booking has values that cannot be right; fill it in yourself instead'); + } + if (await this.candidates.tripHasExternal(trip.id, IMPORT_SOURCES, this.apply.externalKey(item))) { + throw new DomainError(409, 'This booking is already in the trip.'); + } + const uncertain = decodeJson(MAIL_INGEST_UNCERTAIN_FIELDS, entry.uncertain_fields); + const prepared = await this.apply.prepare(item, trip, entry.user_id, uncertain); + const written = await this.uow.transactional(async () => { + const done = await this.apply.writeInTx(prepared); + const claimed = await this.candidates.resolve(entry.id, ['pending'], { + resolution: 'applied', + reason: null, + resolvedTripId: trip.id, + createdIdsJson: encodeJson(MAIL_INGEST_CREATED, done.created), + resolvedBy: userId, + resolvedAt: dbNow(now), + undoUntil: dbNow(new Date(now.getTime() + UNDO_WINDOW_DAYS * DAY_MS)), + }); + if (!claimed) throw new DomainError(409, 'This entry was decided already'); + return done; + }); + await this.apply.announce(written); + const mail = await this.mailOf(entry); + if (mail) { + const { fileIds, error } = await this.apply.attachDocuments(trip.id, written.reservation.id, mail, userId); + const created = { ...written.created, file_ids: fileIds }; + await this.candidates.setAttachments(entry.id, encodeJson(MAIL_INGEST_CREATED, created), error); + } + return this.reload(entry.id); + } + + /** + * "Add as document": the mail's attachments, or its text, as files of the + * trip, and the entry settled as filed. Without the documents addon there + * is nothing to file into. + * + * @txIndependent each document is storage I/O; the entry is settled once they are in. + */ + async fileToTrip( + candidateId: number, + userId: number, + tripId: number, + now: Date = new Date(), + ): Promise { + const entry = await this.pendingEntry(candidateId, userId); + const trip = await this.tripFor(userId, tripId); + const mail = await this.mailOf(entry); + if (!mail) throw new DomainError(410, 'The mail is no longer stored'); + const { fileIds, error } = await this.apply.attachDocuments(trip.id, null, mail, userId); + if (fileIds.length === 0) throw new DomainError(409, error ?? 'This trip takes no documents'); + const created = { reservation_ids: [], place_ids: [], budget_item_ids: [], file_ids: fileIds, fingerprint: null }; + await this.candidates.resolve(entry.id, ['pending'], { + resolution: 'filed', + reason: null, + resolvedTripId: trip.id, + createdIdsJson: encodeJson(MAIL_INGEST_CREATED, created), + resolvedBy: userId, + resolvedAt: dbNow(now), + undoUntil: null, + }); + return this.reload(entry.id); + } + + /** + * "Fill in yourself": the booking form's starting values. The booking read + * from the mail where there is one; otherwise what the mail says plainly: a + * type by its words, the subject as title, the first date, a reference. + */ + async prefill( + candidateId: number, + userId: number, + ): Promise<{ trip_id: number | null; item: BookingImportPreviewItem }> { + const entry = await this.pendingEntry(candidateId, userId); + const item = decodeJson(MAIL_INGEST_ITEM, entry.item_json); + if (item) return { trip_id: entry.suggested_trip_id, item }; + const mail = await this.mailOf(entry); + const message = await this.messages.findById(entry.message_id); + return { trip_id: entry.suggested_trip_id, item: guessedItem(mail, message?.subject ?? null) }; + } + + /** Puts a waiting entry aside; the mail goes with the retention like any other. */ + async discard(candidateId: number, userId: number, now: Date = new Date()): Promise { + const entry = await this.pendingEntry(candidateId, userId); + await this.candidates.resolve(entry.id, ['pending'], { + resolution: 'discarded', + reason: entry.reason, + resolvedTripId: null, + createdIdsJson: null, + resolvedBy: userId, + resolvedAt: dbNow(now), + undoUntil: null, + }); + return this.reload(entry.id); + } + + /** What an undo would touch beyond the booking: costs someone settled with, and an edit since. */ + private async undoAffects(tripId: number, created: MailIngestCreated): Promise> { + const affects: Array<'balances' | 'edited'> = []; + if ((await this.candidates.countPayers(created.budget_item_ids)) > 0) affects.push('balances'); + if (await this.apply.editedSince(tripId, created)) affects.push('edited'); + return affects; + } + + /** The person's own waiting entry; anyone else's answers as missing. */ + private async pendingEntry(candidateId: number, userId: number): Promise { + const entry = await this.candidates.findById(candidateId); + if (!entry || entry.user_id !== userId) throw new DomainError(404, 'Entry not found'); + if (entry.resolution !== 'pending') throw new DomainError(409, 'This entry was decided already'); + return entry; + } + + private async tripFor(userId: number, tripId: number | null): Promise { + const person = await this.trips.person(userId); + const trip = person && tripId !== null ? await this.trips.editableTrip(person, tripId) : null; + if (!trip) throw new DomainError(404, 'Trip not found'); + return trip; + } + + private async editableBy(userId: number, tripId: number): Promise { + const person = await this.trips.person(userId); + return person !== null && (await this.trips.editableTrip(person, tripId)) !== null; + } + + /** The mail an entry came from, as long as it is stored. */ + private async mailOf(entry: MailIngestCandidateRow): Promise { + const message = await this.messages.findById(entry.message_id); + const raw = openMail(message?.raw_eml_enc ?? null); + return raw ? this.reader.walk(raw) : null; + } + + private async reload(id: number): Promise { + const row = await this.candidates.findById(id); + if (!row) throw new DomainError(404, 'Entry not found'); + return row; + } +} + +/** A booking form started from what a mail says in plain words, for a mail no reading made sense of. */ +function guessedItem(mail: WalkedMail | null, storedSubject: string | null): BookingImportPreviewItem { + const subject = mail?.subject ?? storedSubject ?? ''; + const text = `${subject}\n${mail?.body ?? ''}`; + const type = TYPE_WORDS.find(([words]) => words.test(text))?.[1] ?? 'other'; + const date = ISO_DATE_RE.exec(text)?.[1] ?? null; + const reference = REFERENCE_RE.exec(text)?.[1] ?? null; + return { + type, + title: subject.trim().slice(0, 200) || 'Booking', + reservation_time: date, + confirmation_number: reference ? reference.toUpperCase() : null, + source: { fileName: 'mail', index: 0 }, + }; +} diff --git a/server/src/nest/mail-ingest/mail-ingest-address.service.ts b/server/src/nest/mail-ingest/mail-ingest-address.service.ts new file mode 100644 index 000000000..5926cce5f --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-address.service.ts @@ -0,0 +1,200 @@ +import { MailIngestAddresses } from '../../db/entities/MailIngestAddresses.entity'; +import type { + MailIngestAddressesRepository, + MailIngestAddressRow, +} from '../../db/repositories/MailIngestAddresses.repository'; +import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository'; +import { dbNow, utcSuffix } from '../../db/types'; +import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto'; +import { DomainError } from '../common/domain-error'; +import { UnitOfWork } from '../database/unit-of-work'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { TOKEN_LENGTH } from './mail-ingest.constants'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import type { MailIngestAddress } from '@trek/shared'; + +import { createHash, randomBytes } from 'node:crypto'; + +/** RFC 4648 base32, lower case: a mail's local part keeps it as it is. */ +const BASE32 = 'abcdefghijklmnopqrstuvwxyz234567'; + +/** A live address and the person it belongs to, as a mail that reached it is filed under. */ +export interface ResolvedAddress { + address: MailIngestAddressRow; + person: MailIngestPersonRow; +} + +/** + * The forwarding addresses. A person has one of their own and may make one + * per trip they can add bookings to; each address is a random token behind + * the admin's catch-all, stored as its sha256 for the lookup and encrypted + * so its owner can see it again. Renewing replaces the token at once. + * + * A trip address belongs to the member who made it, and stops working the + * moment they can no longer add bookings to the trip: every mail and every + * write checks that again. + */ +@Injectable() +export class MailIngestAddressService { + constructor( + @InjectRepository(MailIngestAddresses) private readonly addresses: MailIngestAddressesRepository, + private readonly settings: MailIngestSettingsService, + private readonly trips: MailIngestTripsService, + private readonly uow: UnitOfWork, + ) {} + + /** The person's own address; null until they asked for one or while no address template is set. */ + async ownAddress(userId: number): Promise { + return this.view(await this.addresses.findLive(userId, null)); + } + + /** A new own address for the person, the old one stops working. */ + async renewOwn(userId: number): Promise { + await this.requireTemplate(); + return this.view(await this.rotate(userId, null)); + } + + /** The person's address for the trip, if they made one and may still add bookings there. */ + async tripAddress(userId: number, tripId: number): Promise { + await this.requireTripEdit(userId, tripId); + return this.view(await this.addresses.findLive(userId, tripId)); + } + + /** A new address for the trip, for a person who may add bookings to it; an old one stops working. */ + async renewTripAddress(userId: number, tripId: number): Promise { + await this.requireTripEdit(userId, tripId); + await this.requireTemplate(); + return this.view(await this.rotate(userId, tripId)); + } + + /** The person's address for the trip stops working. */ + async revokeTripAddress(userId: number, tripId: number): Promise { + await this.addresses.revokeLive(userId, tripId, dbNow()); + } + + /** + * The live address a mail was sent to, from the recipients in the order the + * receiving server names them (Delivered-To, X-Original-To, To). Null for a + * token nobody holds, a revoked one, a person who switched the import off, + * and a trip address whose owner may no longer add bookings to the trip. + */ + async resolve(recipients: readonly string[]): Promise { + const template = await this.settings.addressTemplate(); + const tokens = recipients + .map((recipient) => tokenOf(recipient, template)) + .filter((token): token is string => token !== null); + const found = await Promise.all(tokens.map((token) => this.addresses.findByTokenHash(hashToken(token)))); + const address = found.find((row): row is MailIngestAddressRow => row !== null); + if (!address || !(await this.isLive(address))) return null; + return this.withPerson(address); + } + + /** The person's own live address, made now when they have none: an uploaded mail goes through it. */ + async ensureOwn(userId: number): Promise { + return (await this.addresses.findLive(userId, null)) ?? (await this.rotate(userId, null)); + } + + /** Whether an address takes mail: not revoked, its person's import on, a trip address's owner still an editor. */ + async isLive(address: MailIngestAddressRow): Promise { + if (address.revoked_at) return false; + if (!(await this.settings.prefsOf(address.user_id)).enabled) return false; + if (address.trip_id === null) return true; + const person = await this.trips.person(address.user_id); + return person !== null && (await this.trips.editableTrip(person, address.trip_id)) !== null; + } + + async findById(id: number): Promise { + return this.addresses.findById(id); + } + + /** Notes that a mail came in on the address. */ + async touch(addressId: number): Promise { + await this.addresses.touchUsed(addressId, dbNow()); + } + + private async withPerson(address: MailIngestAddressRow): Promise { + const person = await this.trips.person(address.user_id); + return person ? { address, person } : null; + } + + private async rotate(userId: number, tripId: number | null): Promise { + const token = newToken(); + const id = await this.uow.transactional(async () => { + const now = dbNow(); + await this.addresses.revokeLive(userId, tripId, now); + return this.addresses.insertAddress({ + userId, + tripId, + tokenHash: hashToken(token), + tokenEnc: encrypt_api_key(token), + createdAt: now, + }); + }); + const row = await this.addresses.findById(id); + if (!row) throw new Error(`mail address ${id} vanished after its insert`); + return row; + } + + private async view(row: MailIngestAddressRow | null): Promise { + if (!row) return null; + const template = await this.settings.addressTemplate(); + const token = decrypt_api_key(row.token_enc); + if (!template || !token) return null; + return { + id: row.id, + address: template.replace('{token}', token), + trip_id: row.trip_id, + created_at: utcSuffix(row.created_at) ?? row.created_at, + last_used_at: utcSuffix(row.last_used_at), + }; + } + + private async requireTemplate(): Promise { + if (!(await this.settings.addressTemplate())) { + throw new DomainError(409, 'No forwarding address is set up on this server'); + } + } + + private async requireTripEdit(userId: number, tripId: number): Promise { + const person = await this.trips.person(userId); + if (!person || !(await this.trips.editableTrip(person, tripId))) { + throw new DomainError(404, 'Trip not found'); + } + } +} + +/** A random token of {@link TOKEN_LENGTH} base32 characters, from the system's cryptographic source. */ +export function newToken(): string { + const bytes = randomBytes(TOKEN_LENGTH); + return [...bytes].map((byte) => BASE32[byte % 32]).join(''); +} + +/** The sha256 a token is looked up by; tokens are case-insensitive, as a mail's local part is in practice. */ +export function hashToken(token: string): string { + return createHash('sha256').update(token.toLowerCase()).digest('hex'); +} + +/** + * The token a recipient address carries: what stands where `{token}` stands + * in the template, or, for a box with sub-addressing, what follows the `+`. + * Null for an address that holds none. + */ +export function tokenOf(recipient: string, template: string | null): string | null { + const address = recipient.replace(/^.*.*$/, '').trim().toLowerCase(); + const local = address.split('@')[0] ?? ''; + if (template) { + const [prefix, suffix] = template.toLowerCase().split('@')[0].split('{token}'); + if (suffix !== undefined && local.startsWith(prefix) && local.endsWith(suffix)) { + const token = local.slice(prefix.length, local.length - suffix.length); + if (isToken(token)) return token; + } + } + const plus = local.split('+')[1]; + return plus && isToken(plus) ? plus : null; +} + +function isToken(value: string): boolean { + return value.length >= 20 && /^[a-z2-7]+$/.test(value); +} diff --git a/server/src/nest/mail-ingest/mail-ingest-apply.service.ts b/server/src/nest/mail-ingest/mail-ingest-apply.service.ts new file mode 100644 index 000000000..159376ebe --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-apply.service.ts @@ -0,0 +1,386 @@ +import { ADDON_IDS } from '../../addons'; +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import type { MailIngestCreated } from '../../db/json-columns'; +import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository'; +import { AddonsService } from '../addons/addons.service'; +import { logWarn } from '../audit/audit-log.logger'; +import { BookingImportService } from '../booking-import/booking-import.service'; +import { BudgetService } from '../budget/budget.service'; +import { isUploadTypeAllowed } from '../files/files.constants'; +import { FilesService } from '../files/files.service'; +import type { WalkedMail } from '../llm-parse/mail-read.types'; +import { MapsService } from '../maps/maps.service'; +import { PlacesService, type PlaceCreateInput } from '../places/places.service'; +import { RealtimeService } from '../realtime/realtime.service'; +import { ReservationsService, type CostEvent, type CreateReservationData } from '../reservations/reservations.service'; +import { contentHash, dateOf, type TripOption } from './mail-ingest-checks'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MAIL_EXTERNAL_SOURCE } from './mail-ingest.constants'; +import { parsePriceMinor } from './mail-money'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import { + normalizePlaceWebsite, + typeToCostCategory, + type BookingImportPreviewItem, + type MailIngestField, +} from '@trek/shared'; + +import { createHash } from 'node:crypto'; + +type CostEntry = Parameters[2]; + +/** What a booking's fingerprint covers: the fields a person edits. */ +const FINGERPRINT_KEYS = [ + 'title', + 'type', + 'status', + 'reservation_time', + 'reservation_end_time', + 'location', + 'confirmation_number', + 'notes', + 'metadata', +] as const; +type Written = Awaited>; + +/** A booking ready to write: everything that needs the network (geocoding, the frozen rate) is done. */ +export interface PreparedBooking { + tripId: number; + data: CreateReservationData; + venue: PlaceCreateInput | null; + cost: CostEntry; + travelerIds: number[]; + /** The fields marked for a second look, the price among them when no cost could be written. */ + uncertain: MailIngestField[]; +} + +/** What writing a booking created, for the announcement after the commit and for its undo. */ +export interface WrittenBooking { + tripId: number; + reservation: Written['reservation']; + place: { id: number } | null; + accommodationCreated: boolean; + costEvents: CostEvent[]; + stayMirror: Written['stayMirror']; + created: MailIngestCreated; +} + +/** + * Writes a booking read from a mail into its trip, with every field the mail + * gave: the booking or transport with its endpoints and legs, a stay for a + * hotel, the venue as a place (an existing one when the trip has it), the + * forwarder as traveller, and the cost when the price reads to the cent. It + * goes through the same services a booking made by hand goes through, and + * records exactly what it created, so an undo removes that and nothing else. + */ +@Injectable() +export class MailIngestApplyService { + constructor( + private readonly reservations: ReservationsService, + private readonly places: PlacesService, + private readonly budget: BudgetService, + private readonly addons: AddonsService, + private readonly files: FilesService, + private readonly maps: MapsService, + private readonly realtime: RealtimeService, + private readonly bookingImport: BookingImportService, + private readonly settings: MailIngestSettingsService, + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + ) {} + + /** The key a mail booking is stored under: its reference, or, without one, what it says. */ + externalKey(item: BookingImportPreviewItem): string { + return this.bookingImport.externalKey(item) ?? `content:${contentHash(item)}`; + } + + /** The network half of a write: locate the venue and freeze the cost's rate. Writes nothing. */ + async prepare( + item: BookingImportPreviewItem, + trip: TripOption, + forwarderId: number, + uncertain: readonly MailIngestField[], + ): Promise { + const marked = new Set(uncertain); + const cost = await this.costOf(item, trip); + if (cost === null) marked.add('price'); + const venue = await this.venueOf(item); + return { + tripId: trip.id, + data: this.reservationData(item, trip, marked), + venue, + cost: cost ?? undefined, + travelerIds: [forwarderId], + uncertain: [...marked], + }; + } + + /** + * The write itself, inside the caller's transaction: the place (matched or + * new), the booking with its stay and cost, its traveller. Nothing is + * announced here; the caller announces once the transaction committed. + */ + async writeInTx(prepared: PreparedBooking): Promise { + const tripId = String(prepared.tripId); + const matched = prepared.venue ? await this.places.findMatchingPlaceId(tripId, prepared.venue) : null; + const place = prepared.venue && matched === null ? await this.places.create(tripId, prepared.venue) : null; + const placeId = matched ?? place?.id ?? undefined; + const stay = prepared.data.create_accommodation; + const written = await this.reservations.createWithCostInTx( + tripId, + { ...prepared.data, place_id: placeId, ...(stay && { create_accommodation: { ...stay, place_id: placeId } }) }, + prepared.cost, + ); + await this.reservations.setReservationTravelers(written.reservation.id, tripId, prepared.travelerIds); + // Read back once everything is written: the fingerprint an undo compares against is the booking as it stands. + const stored = await this.reservations.getReservationWithJoins(written.reservation.id, tripId); + return { + tripId: prepared.tripId, + reservation: written.reservation, + place: place ? { id: place.id } : null, + accommodationCreated: written.accommodationCreated, + costEvents: written.costEvents, + stayMirror: written.stayMirror, + created: { + reservation_ids: [written.reservation.id], + place_ids: place ? [place.id] : [], + budget_item_ids: createdItemIds(written.costEvents), + file_ids: [], + fingerprint: fingerprintOf(stored ?? written.reservation), + }, + }; + } + + /** What a committed write tells the trip, to everyone, the forwarder's devices included. */ + async announce(written: WrittenBooking): Promise { + const { tripId } = written; + if (written.place) this.realtime.broadcast(tripId, 'place:created', { place: written.place }); + this.realtime.broadcast(tripId, 'reservation:created', { reservation: written.reservation }); + if (written.accommodationCreated) this.realtime.broadcast(tripId, 'accommodation:created', {}); + await this.reservations.announceStayMirror(tripId, written.stayMirror); + this.reservations.announceCost(tripId, written.costEvents, undefined); + } + + /** + * The mail's documents as trip files on the booking (or on the trip, for a + * mail filed without one), when the documents addon is on: each attachment + * of a type the trip takes, or the mail's text when none is. A failure + * leaves the booking standing and is reported, never thrown. + * + * @txIndependent one file per attachment on purpose: each upload is storage + * I/O, and a file that failed leaves the ones before it in place. + */ + async attachDocuments( + tripId: number, + reservationId: number | null, + mail: WalkedMail, + userId: number, + ): Promise<{ fileIds: number[]; error: string | null }> { + if (!(await this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS))) return { fileIds: [], error: null }; + const allowed = await this.settings.allowedFileTypes(); + const kept = mail.attachments.filter((file) => isUploadTypeAllowed(file.fileName, file.mimeType, allowed)); + const uploads = kept.length + ? kept.map((file) => ({ originalname: file.fileName, mimetype: file.mimeType, bytes: file.data })) + : [{ originalname: 'mail.txt', mimetype: 'text/plain', bytes: Buffer.from(mailText(mail), 'utf8') }]; + const fileIds: number[] = []; + try { + await uploads.reduce(async (previous, upload) => { + await previous; + const file = await this.files.createFileFromBytes(tripId, upload, userId, { reservation_id: reservationId }); + fileIds.push(file.id); + this.realtime.broadcast(tripId, 'file:created', { file }); + }, Promise.resolve()); + return { fileIds, error: null }; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logWarn(`[mail-ingest] Attaching a mail's documents to trip ${tripId} failed: ${reason}`); + return { fileIds, error: reason.slice(0, 300) }; + } + } + + /** + * Removes what adding a booking created, inside the caller's transaction: + * the booking with its stay and costs, and a place it made that nothing + * else names any more. Answers what to announce once that committed. + */ + async removeInTx(tripId: number, created: MailIngestCreated): Promise { + const removed: RemovedBooking = { + tripId, + reservationIds: [], + accommodationIds: [], + budgetItemIds: [], + placeIds: [], + }; + await created.reservation_ids.reduce(async (previous, id) => { + await previous; + const gone = await this.reservations.remove(id, tripId); + if (!gone.deleted) return; + removed.reservationIds.push(id); + removed.budgetItemIds.push(...gone.deletedBudgetItemIds); + if (gone.accommodationDeleted && gone.deleted.accommodation_id) { + removed.accommodationIds.push(gone.deleted.accommodation_id); + } + }, Promise.resolve()); + await created.place_ids.reduce(async (previous, placeId) => { + await previous; + if (await this.candidates.placeInUse(placeId)) return; + if ((await this.places.remove(String(tripId), String(placeId))).deleted) removed.placeIds.push(placeId); + }, Promise.resolve()); + return removed; + } + + /** What an undo tells the trip once it committed, and the documents it moves to the trash. */ + async announceRemoved(removed: RemovedBooking, fileIds: readonly number[]): Promise { + const { tripId } = removed; + for (const accommodationId of removed.accommodationIds) { + this.realtime.broadcast(tripId, 'accommodation:deleted', { accommodationId }); + } + for (const itemId of removed.budgetItemIds) this.realtime.broadcast(tripId, 'budget:deleted', { itemId }); + for (const reservationId of removed.reservationIds) { + this.realtime.broadcast(tripId, 'reservation:deleted', { reservationId }); + } + for (const placeId of removed.placeIds) this.realtime.broadcast(tripId, 'place:deleted', { placeId }); + await Promise.all(fileIds.map((fileId) => this.files.softDeleteFile(fileId))); + for (const fileId of fileIds) this.realtime.broadcast(tripId, 'file:deleted', { fileId }); + } + + /** Whether the booking was changed since it was written, by its fingerprint then and now. */ + async editedSince(tripId: number, created: MailIngestCreated): Promise { + if (created.fingerprint === null) return false; + const [reservationId] = created.reservation_ids; + if (reservationId === undefined) return true; + const now = await this.reservations.getReservationWithJoins(reservationId, tripId); + return !now || fingerprintOf(now) !== created.fingerprint; + } + + private reservationData( + item: BookingImportPreviewItem, + trip: TripOption, + marked: Set, + ): CreateReservationData { + const { _venue, _accommodation, source: _source, type_guessed: _guessed, needs_review: _review, ...fields } = item; + const endpoints = (fields.endpoints ?? []).flatMap((endpoint) => + endpoint.lat != null && endpoint.lng != null ? [{ ...endpoint, lat: endpoint.lat, lng: endpoint.lng }] : [], + ); + if (endpoints.length < (fields.endpoints ?? []).length) marked.add('endpoints'); + return { + title: fields.title, + type: fields.type, + status: statusOf(item), + reservation_time: fields.reservation_time ?? undefined, + reservation_end_time: fields.reservation_end_time ?? undefined, + location: fields.location ?? undefined, + confirmation_number: fields.confirmation_number ?? undefined, + metadata: fields.metadata, + endpoints, + needs_review: marked.size > 0, + external_source: MAIL_EXTERNAL_SOURCE, + external_id: this.externalKey(item), + sync_enabled: false, + ...(item.type === 'hotel' && _accommodation && { create_accommodation: stayOf(_accommodation, trip) }), + }; + } + + /** + * The cost a price becomes: only with the costs addon on, a price read to + * the cent and, in a foreign currency, a rate frozen now. Undefined where + * there is no cost to write, null where the mail names a price that cannot + * be written without guessing (the field is marked instead). + */ + private async costOf(item: BookingImportPreviewItem, trip: TripOption): Promise { + const meta = item.metadata ?? {}; + if (meta.price == null || meta.price === '') return undefined; + if (!(await this.addons.isAddonEnabled(ADDON_IDS.BUDGET))) return undefined; + const currency = typeof meta.priceCurrency === 'string' ? meta.priceCurrency.toUpperCase() : null; + const minor = parsePriceMinor(meta.price, currency ?? trip.currency); + if (minor === null || minor <= 0) return null; + const entry: NonNullable = { + total_price: minor / 100, + category: typeToCostCategory(item.type), + currency, + }; + try { + await this.budget.freezeForeignRate(trip.id, entry); + } catch { + return null; + } + const abroad = currency !== null && currency !== trip.currency; + return abroad && entry.exchange_rate === undefined ? null : entry; + } + + /** The venue as a place: its position from the mail, or looked up by its name and address. */ + private async venueOf(item: BookingImportPreviewItem): Promise { + const venue = item._venue; + if (!venue?.name) return null; + let position = venue.lat != null && venue.lng != null ? { lat: venue.lat, lng: venue.lng } : null; + if (!position) { + const query = venue.address ? `${venue.name} ${venue.address}` : venue.name; + position = await this.maps.geocodeQuery(query).catch(() => null); + } + return { + name: venue.name, + lat: position?.lat, + lng: position?.lng, + address: venue.address, + website: normalizePlaceWebsite(venue.website) ?? undefined, + phone: venue.phone, + }; + } +} + +/** What an undo removed, for the announcement after it committed. */ +export interface RemovedBooking { + tripId: number; + reservationIds: number[]; + accommodationIds: string[]; + budgetItemIds: number[]; + placeIds: number[]; +} + +/** The status a booking is written with: confirmed when the mail said so, pending otherwise. */ +function statusOf(item: BookingImportPreviewItem): string { + const status = item.metadata?.reservationStatus; + return typeof status === 'string' && /confirm/i.test(status) ? 'confirmed' : 'pending'; +} + +/** A hotel's stay: its days by check-in and check-out date, its times of day alone. */ +function stayOf( + stay: NonNullable, + trip: TripOption, +): NonNullable { + const checkIn = dateOf(stay.check_in); + const checkOut = dateOf(stay.check_out); + return { + start_day_id: checkIn ? trip.days.get(checkIn) : undefined, + end_day_id: checkOut ? trip.days.get(checkOut) : undefined, + check_in: timeOf(stay.check_in), + check_out: timeOf(stay.check_out), + confirmation: stay.confirmation, + }; +} + +/** The time of day of an ISO date-time, `HH:MM`; undefined for a bare date. */ +function timeOf(value: string | undefined): string | undefined { + const match = /T(\d{2}:\d{2})/.exec(value ?? ''); + return match ? match[1] : undefined; +} + +/** The ids of the costs a write created. */ +function createdItemIds(events: readonly CostEvent[]): number[] { + return events.flatMap((event) => { + if (event.event !== 'budget:created') return []; + const item = event.payload.item; + return item && typeof item === 'object' && 'id' in item && typeof item.id === 'number' ? [item.id] : []; + }); +} + +/** A booking's content as one hash: an edit after the write changes it. */ +function fingerprintOf(reservation: Record): string { + const content = FINGERPRINT_KEYS.map((key) => reservation[key] ?? null); + return createHash('sha256').update(JSON.stringify(content)).digest('hex'); +} + +/** A mail as a text file: its subject and its readable body, without any header. */ +function mailText(mail: WalkedMail): string { + return `${mail.subject ?? ''}\n\n${mail.body}`.trim(); +} diff --git a/server/src/nest/mail-ingest/mail-ingest-checks.ts b/server/src/nest/mail-ingest/mail-ingest-checks.ts new file mode 100644 index 000000000..67c929f79 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-checks.ts @@ -0,0 +1,318 @@ +import { haversineKm } from '../common/geo'; +import { NEAR_TRIP_KM } from './mail-ingest.constants'; +import { + bookingImportPreviewItemSchema, + type BookingImportPreviewItem, + type MailIngestCheck, + type MailIngestCheckResult, + type MailIngestField, + type MailIngestReason, + type MailIngestSource, +} from '@trek/shared'; + +import { createHash } from 'node:crypto'; + +/** + * The server's checks on one booking read from a mail (plan 5.3, G1 to G8), + * and the choice of its trip. Pure: everything they weigh is handed in, so a + * mail cannot steer them, and a test can name every case. + */ + +/** A trip a booking may go to, as the checks see it. */ +export interface TripOption { + id: number; + title: string; + start_date: string | null; + end_date: string | null; + ownerId: number; + /** The trip's currency, upper case. */ + currency: string; + /** The trip's day ids by date. */ + days: ReadonlyMap; + places: ReadonlyArray<{ name: string; address: string | null; lat: number; lng: number }>; +} + +/** A booking read from a mail, before the checks. */ +export interface MailDraft { + item: BookingImportPreviewItem; + source: MailIngestSource; + kind: 'new_booking' | 'change' | 'cancellation'; + /** The trip the AI picked, already mapped from its reference to an id of the list; null when it picked none. */ + aiTripId: number | null; + uncertainFields: MailIngestField[]; + suggestedNewTrip: { destination: string; start_date: string | null; end_date: string | null } | null; + /** Read from page images alone: nothing can be checked against text. */ + imageOnly: boolean; + /** The AI read a critical field differently from the machine-readable data (G4). */ + disagrees: boolean; +} + +/** What the mail around the booking adds to the checks. */ +export interface CheckContext { + trips: readonly TripOption[]; + /** The trip whose address the mail came in on; null for the person's own address. */ + addressTripId: number | null; + subject: string | null; + /** Everything the mail and its attachments say, for the in-text check. */ + documentText: string; + now: Date; + knownAirport: (code: string) => boolean; +} + +export interface TripChoice { + /** The trip the booking goes to; null when none is certain. */ + tripId: number | null; + /** The trip to offer in the inbox. */ + suggestedTripId: number | null; + reason: MailIngestReason | null; +} + +export interface BookingVerdict { + checks: Partial>; + /** The first reason the booking cannot be added on its own; null when every check passed. */ + reason: MailIngestReason | null; + uncertain: MailIngestField[]; + choice: TripChoice; +} + +const DAY_MS = 24 * 60 * 60 * 1000; +/** Types that run from one place to another and need both to be added. */ +const ROUTED_TYPES = new Set(['flight', 'train', 'bus', 'ferry', 'cruise']); +/** Types that are only added on their own with a booking reference: a mail without one is too vague. */ +const REFERENCE_TYPES = new Set(['flight', 'train', 'bus', 'ferry', 'cruise', 'car', 'hotel']); +const MAX_HOURS: Readonly> = { flight: 20, train: 48, bus: 48 }; +const MAX_STAY_NIGHTS = 60; +const MAX_PRICE = 100_000; +const REFERENCE_RE = /^[A-Z0-9-]{3,40}$/; +const CURRENCY_RE = /^[A-Z]{3}$/; +/** + * Words that mark a cancellation in the subject. Only the subject: every + * hotel confirmation speaks of its free cancellation in the body. + */ +const CANCEL_RE = /\b(cancel+ed|cancel+ation|storniert|stornierung|annul[ée]e?|annulation|cancelad[ao]|annullat[ao]|geannuleerd|anulowan[ay]|отмен)/i; + +/** The date part of an ISO date or date-time, null for anything else. */ +export function dateOf(value: string | null | undefined): string | null { + const date = value?.slice(0, 10) ?? ''; + return /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : null; +} + +/** When the booking starts: its time, or a stay's check-in. */ +export function bookingStart(item: BookingImportPreviewItem): string | null { + return item.reservation_time || item._accommodation?.check_in || null; +} + +/** When the booking ends: its end time, or a stay's check-out. */ +export function bookingEnd(item: BookingImportPreviewItem): string | null { + return item.reservation_end_time || item._accommodation?.check_out || null; +} + +/** A booking reference as an operator prints it, without the spacing and case a mail adds. */ +export function normalizeReference(value: string | null | undefined): string { + return (value ?? '').replace(/\s+/g, '').toUpperCase(); +} + +/** No type, no start, or a transport without a named start and end. */ +export function missingCritical(item: BookingImportPreviewItem): boolean { + if (!item.type || !dateOf(bookingStart(item))) return true; + if (!ROUTED_TYPES.has(item.type)) return false; + return (item.endpoints ?? []).filter((endpoint) => endpoint.name.trim() !== '').length < 2; +} + +/** A transport, stay or rental without a booking reference. */ +export function missingReference(item: BookingImportPreviewItem): boolean { + return REFERENCE_TYPES.has(item.type) && normalizeReference(item.confirmation_number) === ''; +} + +/** G5: a value that cannot be right. */ +export function implausible( + item: BookingImportPreviewItem, + now: Date, + knownAirport: (code: string) => boolean, +): boolean { + const start = Date.parse(bookingStart(item) ?? ''); + if (!Number.isFinite(start)) return true; + if (start < now.getTime() - 366 * DAY_MS || start > now.getTime() + 3 * 366 * DAY_MS) return true; + const end = Date.parse(bookingEnd(item) ?? ''); + if (Number.isFinite(end) && (end < start || tooLong(item.type, end - start))) return true; + if (item.type === 'flight' && (item.endpoints ?? []).some((e) => e.code && !knownAirport(e.code))) return true; + const reference = normalizeReference(item.confirmation_number); + if (reference && !REFERENCE_RE.test(reference)) return true; + return !plausibleMoney(item.metadata); +} + +function tooLong(type: string, ms: number): boolean { + if (type === 'hotel') return ms > MAX_STAY_NIGHTS * DAY_MS; + const hours = MAX_HOURS[type]; + return hours !== undefined && ms > hours * 60 * 60 * 1000; +} + +function plausibleMoney(metadata: Record | undefined): boolean { + const currency = metadata?.priceCurrency; + if (currency != null && !(typeof currency === 'string' && CURRENCY_RE.test(currency.toUpperCase()))) return false; + const price = metadata?.price; + if (price == null || price === '') return true; + const amount = Number(price); + // A price written with separators is left to the money parser, which marks rather than refuses it. + return !Number.isFinite(amount) || (amount > 0 && amount < MAX_PRICE); +} + +/** G8: the booking, or the mail it came in, is a cancellation. */ +export function isCancellation( + item: BookingImportPreviewItem, + kind: MailDraft['kind'], + subject: string | null, +): boolean { + if (kind === 'cancellation') return true; + const status = item.metadata?.reservationStatus; + if (typeof status === 'string' && /cancel/i.test(status)) return true; + return CANCEL_RE.test(subject ?? ''); +} + +/** Letters and digits only, upper case: how a reference or a flight number is looked for in the text. */ +function compact(text: string): string { + return text.toUpperCase().replace(/[^\p{L}\p{N}]/gu, ''); +} + +/** + * G6: what an AI reading names has to be in the mail. A reference or a + * service number that is not refuses the booking; a free text that is not + * (its title, its place) only marks the field for a second look. + */ +export function checkInText( + item: BookingImportPreviewItem, + documentText: string, +): { ok: boolean; uncertain: MailIngestField[] } { + const text = compact(documentText); + const reference = compact(item.confirmation_number ?? ''); + const number = compact(String(item.metadata?.flight_number ?? item.metadata?.train_number ?? '')); + const ok = (!reference || text.includes(reference)) && (!number || text.includes(number)); + const uncertain: MailIngestField[] = []; + if (!freeTextIn(item.title, documentText)) uncertain.push('title'); + if (item.location && !freeTextIn(item.location, documentText)) uncertain.push('location'); + return { ok, uncertain }; +} + +/** Whether most words of a free text appear in the document, case and accents aside. */ +function freeTextIn(value: string, documentText: string): boolean { + const fold = (s: string) => s.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase(); + const haystack = fold(documentText); + const words = fold(value) + .split(/[^\p{L}\p{N}]+/u) + .filter((word) => word.length >= 3); + if (words.length === 0) return true; + return words.filter((word) => haystack.includes(word)).length * 2 >= words.length; +} + +/** + * G3: every date of the booking lies within the trip (a day of slack on each + * side) and the days it needs exist: a stay its check-in and check-out day, + * anything else the day it starts on. + */ +export function fitsTrip(item: BookingImportPreviewItem, trip: TripOption): boolean { + if (!trip.start_date || !trip.end_date) return false; + const from = shiftDate(trip.start_date, -1); + const to = shiftDate(trip.end_date, 1); + const start = dateOf(bookingStart(item)); + const end = dateOf(bookingEnd(item)); + const dates = [start, end].filter((date): date is string => date !== null); + if (dates.length === 0 || dates.some((date) => date < from || date > to)) return false; + const needed = item.type === 'hotel' ? dates : [start]; + return needed.every((date) => date !== null && trip.days.has(date)); +} + +function shiftDate(date: string, days: number): string { + return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10); +} + +/** Whether the booking's destination lies near one of the trip's places, by coordinates the server holds. */ +export function nearTrip(item: BookingImportPreviewItem, trip: TripOption): boolean { + const points = [...(item.endpoints ?? []).slice(-1), ...(item._venue ? [item._venue] : [])].filter( + (point): point is typeof point & { lat: number; lng: number } => point.lat != null && point.lng != null, + ); + return points.some((point) => + trip.places.some((place) => haversineKm(point.lat, point.lng, place.lat, place.lng) <= NEAR_TRIP_KM), + ); +} + +/** + * G2 and G7: the trip the booking goes to. The server decides; the AI's pick + * is one input. A trip address names its trip. Otherwise the booking goes + * where exactly one trip fits its dates, or, of several that do, to the AI's + * pick when the server confirms the booking lies near it. + */ +export function chooseTrip(item: BookingImportPreviewItem, aiTripId: number | null, context: CheckContext): TripChoice { + const { trips, addressTripId } = context; + if (addressTripId !== null) { + const trip = trips.find((candidate) => candidate.id === addressTripId); + if (!trip) return { tripId: null, suggestedTripId: null, reason: 'no_edit_permission' }; + if (trip.start_date && trip.end_date && !fitsTrip(item, trip)) { + return { tripId: null, suggestedTripId: trip.id, reason: 'outside_trip_dates' }; + } + return { tripId: trip.id, suggestedTripId: trip.id, reason: null }; + } + const passing = trips.filter((trip) => fitsTrip(item, trip)); + const picked = passing.find((trip) => trip.id === aiTripId); + if (passing.length === 1) return { tripId: passing[0].id, suggestedTripId: passing[0].id, reason: null }; + if (passing.length > 1) { + if (picked && nearTrip(item, picked)) return { tripId: picked.id, suggestedTripId: picked.id, reason: null }; + return { tripId: null, suggestedTripId: (picked ?? passing[0]).id, reason: 'trip_ambiguous' }; + } + const suggested = trips.some((trip) => trip.id === aiTripId) ? aiTripId : null; + const undatedOnly = trips.length > 0 && trips.every((trip) => !trip.start_date || !trip.end_date); + return { tripId: null, suggestedTripId: suggested, reason: undatedOnly ? 'trip_without_dates' : 'no_trip_in_range' }; +} + +/** + * Every check on one booking, and the first reason it cannot be added on its + * own. The trip is chosen even for a booking that fails, so the inbox can + * suggest it. + */ +export function judgeBooking(draft: MailDraft, context: CheckContext): BookingVerdict { + const { item } = draft; + const choice = chooseTrip(item, draft.aiTripId, context); + const fromAi = draft.source === 'ai'; + const inText = fromAi ? checkInText(item, context.documentText) : { ok: true, uncertain: [] }; + const checks: BookingVerdict['checks'] = { + schema: bookingImportPreviewItemSchema.safeParse(item).success ? 'pass' : 'fail', + trip: choice.reason === 'no_edit_permission' ? 'fail' : 'pass', + dates: choice.tripId !== null || choice.reason === 'trip_ambiguous' ? 'pass' : 'fail', + agreement: draft.disagrees ? 'fail' : fromAi ? 'pass' : 'skipped', + plausible: implausible(item, context.now, context.knownAirport) ? 'fail' : 'pass', + in_text: !fromAi ? 'skipped' : inText.ok && !draft.imageOnly ? 'pass' : 'fail', + assignment: choice.tripId !== null ? 'pass' : 'fail', + cancellation: isCancellation(item, draft.kind, context.subject) ? 'fail' : 'pass', + }; + const uncertain = [...new Set([...draft.uncertainFields, ...inText.uncertain])]; + return { checks, reason: firstReason(draft, checks, choice), uncertain, choice }; +} + +function firstReason(draft: MailDraft, checks: BookingVerdict['checks'], choice: TripChoice): MailIngestReason | null { + if (draft.kind === 'change') return 'change_or_cancellation'; + if (checks.cancellation === 'fail') return draft.kind === 'cancellation' ? 'change_or_cancellation' : 'cancelled'; + if (checks.schema === 'fail' || missingCritical(draft.item)) return 'missing_critical_field'; + if (checks.plausible === 'fail') return 'implausible'; + if (checks.agreement === 'fail') return 'source_mismatch'; + if (checks.in_text === 'fail') return draft.imageOnly ? 'image_only' : 'not_in_text'; + if (missingReference(draft.item)) return 'missing_confirmation'; + return choice.reason; +} + +/** + * What the booking says, as one hash: two mails about the same booking with + * the same content give the same hash, and a different time or place a + * different one. + */ +export function contentHash(item: BookingImportPreviewItem): string { + const fingerprint = { + type: item.type, + title: item.title, + start: item.reservation_time ?? null, + end: item.reservation_end_time ?? null, + reference: normalizeReference(item.confirmation_number), + location: item.location ?? null, + stay: item._accommodation ? [item._accommodation.check_in ?? null, item._accommodation.check_out ?? null] : null, + endpoints: (item.endpoints ?? []).map((endpoint) => [endpoint.code ?? null, endpoint.name]), + }; + return createHash('sha256').update(JSON.stringify(fingerprint)).digest('hex'); +} diff --git a/server/src/nest/mail-ingest/mail-ingest-intake.service.ts b/server/src/nest/mail-ingest/mail-ingest-intake.service.ts new file mode 100644 index 000000000..2916cd223 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-intake.service.ts @@ -0,0 +1,128 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import type { MailIngestAddressRow } from '../../db/repositories/MailIngestAddresses.repository'; +import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow } from '../../db/types'; +import { MailReadService } from '../llm-parse/mail-read.service'; +import { MailIngestAddressService } from './mail-ingest-address.service'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { sealMail } from './mail-seal'; +import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import type { MailIngestTransport } from '@trek/shared'; + +import { createHash } from 'node:crypto'; + +/** Why a mail was not taken in. A transport answers each the way its protocol does (a 550, a 404, a 413). */ +export type MailIntakeRefusal = + | 'unavailable' + | 'too_large' + | 'not_a_mail' + | 'unknown_address' + | 'address_daily_limit' + | 'inbox_full'; + +export type MailIntakeResult = + | { status: 'accepted' | 'duplicate'; messageId: number } + | { status: 'refused'; reason: MailIntakeRefusal }; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * Where every mail comes in, whatever carried it: an IMAP box, the signed + * webhook, the TREK Cloud relay, or an .eml the person uploaded. It only + * stores the mail, encrypted and queued, once per address and Message-ID, + * and answers at once; the processing job reads it later. So a transport can + * acknowledge as soon as this returns, and a mail delivered twice is read once. + */ +@Injectable() +export class MailIngestIntakeService { + constructor( + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + private readonly settings: MailIngestSettingsService, + private readonly addresses: MailIngestAddressService, + private readonly reader: MailReadService, + ) {} + + /** + * A mail a transport received. `envelopeRecipients` are the RCPT TO + * addresses where the transport knows them; otherwise the headers name the + * address, in the order the receiving server adds them. + */ + async receive( + raw: Buffer, + transport: Exclude, + envelopeRecipients: readonly string[] = [], + ): Promise { + const refused = await this.precheck(raw); + if (refused) return { status: 'refused', reason: refused }; + const mail = this.reader.walk(raw); + if (!mail) return { status: 'refused', reason: 'not_a_mail' }; + const { deliveredTo, xOriginalTo, to } = mail.envelope; + const headerRecipients = [deliveredTo, xOriginalTo, ...(to ?? '').split(',')].filter( + (value): value is string => !!value && value.trim() !== '', + ); + const resolved = await this.addresses.resolve([...envelopeRecipients, ...headerRecipients]); + if (!resolved) return { status: 'refused', reason: 'unknown_address' }; + return this.store(raw, transport, resolved.address, mail.envelope.messageId); + } + + /** An .eml the person uploaded, read as if they had forwarded it to their own address. */ + async upload(raw: Buffer, userId: number): Promise { + const refused = await this.precheck(raw); + if (refused) return { status: 'refused', reason: refused }; + const mail = this.reader.walk(raw); + if (!mail) return { status: 'refused', reason: 'not_a_mail' }; + const address = await this.addresses.ensureOwn(userId); + return this.store(raw, 'upload', address, mail.envelope.messageId); + } + + private async precheck(raw: Buffer): Promise { + if (!(await this.settings.isAvailable())) return 'unavailable'; + if (raw.length > this.settings.maxMailBytes()) return 'too_large'; + return null; + } + + /** @txIndependent the mail is stored by itself; noting the address's last use stands on its own. */ + private async store( + raw: Buffer, + transport: MailIngestTransport, + address: MailIngestAddressRow, + messageIdHdr: string | null, + ): Promise { + const dedupeHash = createHash('sha256') + .update(`${address.id}\n${messageIdHdr ?? createHash('sha256').update(raw).digest('hex')}`) + .digest('hex'); + const known = await this.messages.findIdByDedupeHash(dedupeHash); + if (known !== null) return { status: 'duplicate', messageId: known }; + + const now = new Date(); + const today = await this.messages.countForAddressSince(address.id, dbNow(new Date(now.getTime() - DAY_MS))); + if (today >= MAIL_INGEST_LIMITS.mailsPerAddressPerDay) return { status: 'refused', reason: 'address_daily_limit' }; + if ((await this.candidates.countPending(address.user_id)) >= MAIL_INGEST_LIMITS.pendingPerUser) { + return { status: 'refused', reason: 'inbox_full' }; + } + + const id = await this.messages.insertIfNew({ + transport, + messageIdHdr: messageIdHdr?.slice(0, 500) ?? null, + dedupeHash, + addressId: address.id, + userId: address.user_id, + tripId: address.trip_id, + size: raw.length, + rawEnc: sealMail(raw), + receivedAt: dbNow(now), + }); + // Another delivery of the same mail won the insert in between: it is in, once. + if (id === null) { + const winner = await this.messages.findIdByDedupeHash(dedupeHash); + return winner === null ? { status: 'refused', reason: 'not_a_mail' } : { status: 'duplicate', messageId: winner }; + } + await this.addresses.touch(address.id); + return { status: 'accepted', messageId: id }; + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-notification-actions.ts b/server/src/nest/mail-ingest/mail-ingest-notification-actions.ts new file mode 100644 index 000000000..ee11514ac --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-notification-actions.ts @@ -0,0 +1,29 @@ +import { NotificationsService } from '../notifications/notifications.service'; +import { MailIngestActionsService } from './mail-ingest-actions.service'; +import { KEEP_ACTION, UNDO_ACTION } from './mail-ingest.constants'; +import { Injectable, type OnModuleInit } from '@nestjs/common'; + +/** + * What the forwarder's notification buttons do. Undo takes the booking back + * out unless its cost moved balances or it was edited since, which needs the + * inbox's own confirmation; keep only answers the question. The handlers run + * in the request that answered the notification, under its context. + */ +@Injectable() +export class MailIngestNotificationActions implements OnModuleInit { + constructor( + private readonly notifications: NotificationsService, + private readonly actions: MailIngestActionsService, + ) {} + + onModuleInit(): void { + this.notifications.registerResponseAction(UNDO_ACTION, (payload, userId) => this.undo(payload, userId)); + this.notifications.registerResponseAction(KEEP_ACTION, async () => undefined); + } + + async undo(payload: Record, userId: number): Promise { + const candidateId = Number(payload.candidateId); + if (!Number.isInteger(candidateId) || candidateId <= 0) throw new Error('This notification names no booking'); + await this.actions.undo(candidateId, userId, false); + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-notify.service.ts b/server/src/nest/mail-ingest/mail-ingest-notify.service.ts new file mode 100644 index 000000000..aa1b7d8c9 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-notify.service.ts @@ -0,0 +1,84 @@ +import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository'; +import { logError } from '../audit/audit-log.logger'; +import { NotificationsService } from '../notifications/notifications.service'; +import { KEEP_ACTION, UNDO_ACTION } from './mail-ingest.constants'; +import { Injectable } from '@nestjs/common'; + +/** A booking a mail added, as its notifications name it. */ +export interface AddedBooking { + candidateId: number; + tripId: number; + tripTitle: string; + title: string; + type: string; +} + +/** + * What the mail import tells people. The forwarder hears of every booking it + * added, with keep and undo; the other members of a shared trip hear that a + * booking came in (owner decision 13.11.2026); and a mail whose bookings wait + * in the inbox sends one note for the lot. A notification never fails the + * write it reports. + */ +@Injectable() +export class MailIngestNotifyService { + constructor(private readonly notifications: NotificationsService) {} + + /** @txStandalone a notice reports a write that committed; a failed one undoes nothing. */ + async bookingAdded(forwarder: MailIngestPersonRow, booking: AddedBooking): Promise { + const params = { + booking: booking.title, + type: booking.type, + trip: booking.tripTitle, + tripId: String(booking.tripId), + actor: forwarder.username, + }; + await this.quietly( + this.notifications.send({ + event: 'mail_booking_added', + actorId: null, + scope: 'user', + targetId: forwarder.id, + params: { ...params, role: 'forwarder' }, + inApp: { + type: 'boolean', + positiveTextKey: 'notif.action.keep', + negativeTextKey: 'notif.action.undo', + positiveCallback: { action: KEEP_ACTION, payload: { candidateId: booking.candidateId } }, + negativeCallback: { action: UNDO_ACTION, payload: { candidateId: booking.candidateId } }, + }, + }), + ); + // Trip scope reaches every member but the actor, who heard above. + await this.quietly( + this.notifications.send({ + event: 'mail_booking_added', + actorId: forwarder.id, + scope: 'trip', + targetId: booking.tripId, + params: { ...params, role: 'traveller' }, + }), + ); + } + + /** @txStandalone one note for the bookings of a mail that wait in the inbox. */ + async bookingsPending(userId: number, pending: number): Promise { + await this.quietly( + this.notifications.send({ + event: 'mail_booking_pending', + actorId: null, + scope: 'user', + targetId: userId, + params: { count: String(pending) }, + }), + ); + } + + private async quietly(sent: Promise): Promise { + try { + await sent; + } catch (err) { + logError(`[mail-ingest] A notification failed: ${err instanceof Error ? err.message : String(err)}`); + } + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-process.job.ts b/server/src/nest/mail-ingest/mail-ingest-process.job.ts new file mode 100644 index 000000000..b86fe9212 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-process.job.ts @@ -0,0 +1,71 @@ +import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow } from '../../db/types'; +import { logError } from '../audit/audit-log.logger'; +import { withRequestContext } from '../database/request-context'; +import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service'; +import { MailIngestProcessService } from './mail-ingest-process.service'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; +import { MikroORM } from '@mikro-orm/core'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; + +/** How long a run holds the pipeline and its mails before another run may take them over. */ +const LEASE_MS = 10 * 60 * 1000; + +/** + * Reads the queued mails every two minutes, at most three per run, one after + * the other. The pipeline's own lease on its cursor row keeps two processes + * on one database (or a run and an upload that asked for one now) from + * reading at once; each mail has its claim as well, so a run that died hands + * its mail on when the claim runs out. Every mail is read in a request + * context of its own, so one mail's identity map never leaks into the next. + */ +@Injectable() +export class MailIngestProcessJob implements OnApplicationBootstrap { + constructor( + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + @InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository, + private readonly settings: MailIngestSettingsService, + private readonly process: MailIngestProcessService, + private readonly registrar: CronRegistrarService, + private readonly orm: MikroORM, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + this.registrar.register('mail-ingest-process', '*/2 * * * *', () => this.tick()); + } + + /** + * One run; answers how many mails it read. Nothing happens while the import + * is off, which is how a plan without it costs nothing. + * + * @txIndependent every mail is read and written on its own: one that fails + * leaves the others, and its retry is its own claim's business. + */ + async tick(now: Date = new Date()): Promise { + if (!(await this.settings.isAvailable())) return 0; + const until = dbNow(new Date(now.getTime() + LEASE_MS)); + if (!(await this.cursor.acquireLease('pipeline', '', LEASE_OWNER, until, dbNow(now)))) return 0; + try { + const ids = await this.messages.claimDue(dbNow(now), until, MAIL_INGEST_LIMITS.mailsPerRun); + await ids.reduce(async (previous, id) => { + await previous; + await withRequestContext(this.orm, () => this.process.process(id, now)); + }, Promise.resolve()); + await this.cursor.recordOk('pipeline', '', dbNow(new Date())); + return ids.length; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logError(`[mail-ingest] A processing run failed: ${reason}`); + await this.cursor.recordError('pipeline', '', reason); + return 0; + } finally { + await this.cursor.releaseLease('pipeline', '', LEASE_OWNER); + } + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-process.service.ts b/server/src/nest/mail-ingest/mail-ingest-process.service.ts new file mode 100644 index 000000000..05a2c47ed --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-process.service.ts @@ -0,0 +1,465 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import { + MAIL_INGEST_ATTACHMENTS, + MAIL_INGEST_CHECKS, + MAIL_INGEST_CREATED, + MAIL_INGEST_HINTS, + MAIL_INGEST_ITEM, + MAIL_INGEST_MATCH, + MAIL_INGEST_UNCERTAIN_FIELDS, +} from '../../db/json-columns'; +import type { + MailIngestCandidatesRepository, + MailIngestPersonRow, +} from '../../db/repositories/MailIngestCandidates.repository'; +import type { + MailIngestMessageOutcome, + MailIngestMessageRow, + MailIngestMessagesRepository, +} from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow } from '../../db/types'; +import { encodeJson } from '../../utils/json-column'; +import { AirportsService } from '../airports/airports.service'; +import { logError } from '../audit/audit-log.logger'; +import { DemoService } from '../common/demo.service'; +import { UnitOfWork } from '../database/unit-of-work'; +import { MailReadService } from '../llm-parse/mail-read.service'; +import type { WalkedMail } from '../llm-parse/mail-read.types'; +import { classifyMail, senderVerified } from './mail-classify'; +import { MailIngestApplyService, type PreparedBooking, type WrittenBooking } from './mail-ingest-apply.service'; +import { + bookingStart, + contentHash, + judgeBooking, + normalizeReference, + type BookingVerdict, + type CheckContext, + type MailDraft, + type TripOption, +} from './mail-ingest-checks'; +import { MailIngestAddressService } from './mail-ingest-address.service'; +import { MailIngestNotifyService } from './mail-ingest-notify.service'; +import { MailIngestReadingService } from './mail-ingest-reading.service'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { IMPORT_SOURCES, MAIL_INGEST_LIMITS, UNDO_WINDOW_DAYS } from './mail-ingest.constants'; +import { openMail } from './mail-seal'; +import { UniqueConstraintViolationException } from '@mikro-orm/core'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import type { MailIngestHint, MailIngestPrefs, MailIngestReason, MailIngestResolution } from '@trek/shared'; + +/** One booking of a mail with the decision on it. */ +interface PlannedBooking { + position: number; + draft: MailDraft; + verdict: BookingVerdict; + key: string; + hash: string; + resolution: MailIngestResolution; + reason: MailIngestReason | null; + /** Set for a booking that is added on its own, with the trip it goes to. */ + prepared: PreparedBooking | null; + trip: TripOption | null; +} + +/** A booking the write added, with its entry, for what follows the commit. */ +interface AddedEntry { + candidateId: number; + trip: TripOption; + written: WrittenBooking; +} + +const MINUTE_MS = 60 * 1000; +const DAY_MS = 24 * 60 * MINUTE_MS; +/** How long a failed mail waits per try before the next. */ +const RETRY_STEP_MS = 5 * MINUTE_MS; +/** A booking an hour either side of one in the trip is the same booking, as far as adding it on its own goes. */ +const SIMILAR_WINDOW_MS = 60 * MINUTE_MS; + +/** + * Reads one forwarded mail to its end (plan section 5). It filters what is no + * booking without AI, reads the bookings (machine-readable data always, the + * AI where allowed), checks each one, adds those that pass to their trip in + * one transaction with the mail's own status, and keeps the rest in the + * inbox with a reason. A mail is hostile: it can pick neither a person, nor a + * trip outside the person's own list, nor anything but adding a booking. + */ +@Injectable() +export class MailIngestProcessService { + constructor( + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + private readonly settings: MailIngestSettingsService, + private readonly addresses: MailIngestAddressService, + private readonly trips: MailIngestTripsService, + private readonly reading: MailIngestReadingService, + private readonly apply: MailIngestApplyService, + private readonly notify: MailIngestNotifyService, + private readonly reader: MailReadService, + private readonly airports: AirportsService, + private readonly demo: DemoService, + private readonly uow: UnitOfWork, + ) {} + + /** + * Reads a mail the caller claimed. A failure is not the caller's: the mail + * is tried again later, and given up after its last try. + * + * @txIndependent the mail's own write is one transaction; the retry note on + * a failure, and what follows the commit, stand on their own. + */ + async process(messageId: number, now: Date = new Date()): Promise { + const message = await this.messages.findById(messageId); + if (!message || message.status !== 'processing') return; + try { + await this.read(message, now); + } catch (err) { + await this.failed(message, err, now); + } + } + + /** @txIndependent each branch settles the mail in one write; the notices after it stand on their own. */ + private async read(message: MailIngestMessageRow, now: Date): Promise { + const raw = openMail(message.raw_eml_enc); + const mail = raw ? this.reader.walk(raw) : null; + const person = await this.trips.person(message.user_id); + if (!raw || !mail || !person) { + await this.keepForInbox(message, null, 'mail_unreadable', now); + return; + } + const outcome = this.outcomeBase(mail, person, now); + const address = message.address_id === null ? null : await this.addresses.findById(message.address_id); + if (!address || !(await this.addresses.isLive(address))) { + await this.messages.finish(message.id, { ...outcome, status: 'discarded', reason: 'address_invalid' }); + return; + } + const kind = classifyMail(mail); + if (kind.kind === 'drop') { + await this.messages.finish(message.id, { ...outcome, status: 'discarded', reason: kind.reason }); + return; + } + if (kind.kind === 'forwarding_confirmation') { + await this.keepForInbox(message, { ...outcome, forwardingCode: kind.code }, 'forwarding_confirmation', now); + await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id)); + return; + } + await this.readBookings(message, raw, mail, person, outcome, now); + } + + /** + * @txIndependent the bookings and the mail's status are one transaction; + * announcements, documents and notices follow it. + */ + private async readBookings( + message: MailIngestMessageRow, + raw: Buffer, + mail: WalkedMail, + person: MailIngestPersonRow, + outcome: MailIngestMessageOutcome, + now: Date, + ): Promise { + const prefs = await this.settings.prefsOf(person.id); + const machine = await this.reading.readMachine(raw, mail, person.id); + const trips = await this.tripsFor(message, person, now, this.reading.hintDates(mail, machine)); + const read = await this.reading.read({ mail, machine, person, prefs, trips, now }); + const done = { ...outcome, aiUsed: read.aiUsed }; + if (read.drafts.length === 0) { + if (read.emptyReason === 'not_a_booking') { + await this.messages.finish(message.id, { ...done, status: 'discarded', reason: 'not_a_booking' }); + return; + } + await this.keepForInbox(message, done, read.emptyReason, now); + await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id)); + return; + } + const context: CheckContext = { + trips, + addressTripId: message.trip_id, + subject: mail.subject, + documentText: read.documentText, + now, + knownAirport: (code) => this.airports.findByIata(code) !== null, + }; + const planned = await this.plan(read.drafts, context, person, prefs); + const added = await this.write(message, planned, done, now); + await this.afterCommit(added, planned, mail, person); + } + + /** The trips a booking of this mail may go to: the address's trip alone, or the person's editable ones. */ + private async tripsFor( + message: MailIngestMessageRow, + person: MailIngestPersonRow, + now: Date, + hintDates: string[], + ): Promise { + if (message.trip_id === null) return this.trips.editableTrips(person, now, hintDates); + const trip = await this.trips.editableTrip(person, message.trip_id); + return trip ? [trip] : []; + } + + /** Every booking with its checks, the checks that read its trip and, for one added on its own, its network half. */ + private async plan( + drafts: readonly MailDraft[], + context: CheckContext, + person: MailIngestPersonRow, + prefs: MailIngestPrefs, + ): Promise { + const tooMany = drafts.length > MAIL_INGEST_LIMITS.autoBookings; + const demo = await this.demo.isDemoUserId(person.id); + const judged = await Promise.all(drafts.map((draft, position) => this.judge(draft, position, context))); + const seen = new Set(); + for (const booking of judged) { + const tripKey = `${booking.verdict.choice.tripId}|${booking.key}`; + if (booking.resolution === 'pending' && booking.reason === null && seen.has(tripKey)) { + booking.resolution = 'duplicate'; + } + seen.add(tripKey); + if (booking.resolution !== 'pending' || booking.reason !== null) continue; + if (tooMany) booking.reason = 'too_many_bookings'; + else if (demo) booking.reason = 'demo_mode'; + else if (!prefs.auto_add) booking.reason = 'ask_first'; + } + return judged.reduce>(async (previous, booking) => { + const done = await previous; + const trip = context.trips.find((candidate) => candidate.id === booking.verdict.choice.tripId); + if (booking.resolution === 'pending' && booking.reason === null && trip) { + booking.prepared = await this.apply.prepare(booking.draft.item, trip, person.id, booking.verdict.uncertain); + booking.trip = trip; + } + return [...done, booking]; + }, Promise.resolve([])); + } + + /** One booking's checks, and the checks that read the trip it would go to (G9, an undone twin, a look-alike). */ + private async judge(draft: MailDraft, position: number, context: CheckContext): Promise { + const verdict = judgeBooking(draft, context); + const planned: PlannedBooking = { + position, + draft, + verdict, + key: this.apply.externalKey(draft.item), + hash: contentHash(draft.item), + resolution: 'pending', + reason: verdict.reason, + prepared: null, + trip: null, + }; + const tripId = verdict.choice.tripId; + if (tripId === null || verdict.reason !== null) return planned; + if (await this.candidates.tripHasExternal(tripId, IMPORT_SOURCES, planned.key)) { + verdict.checks.duplicate = 'fail'; + return { ...planned, resolution: 'duplicate' }; + } + verdict.checks.duplicate = 'pass'; + if ((await this.candidates.wasUndoneInTrip(tripId, planned.hash)) || (await this.looksAlike(tripId, draft))) { + return { ...planned, reason: 'possible_duplicate' }; + } + return planned; + } + + private async looksAlike(tripId: number, draft: MailDraft): Promise { + const start = Date.parse(bookingStart(draft.item) ?? ''); + const window = (offset: number) => + Number.isFinite(start) ? new Date(start + offset).toISOString().slice(0, 16) : null; + return this.candidates.tripHasSimilar(tripId, { + type: draft.item.type, + startFrom: window(-SIMILAR_WINDOW_MS), + startTo: window(SIMILAR_WINDOW_MS), + confirmation: normalizeReference(draft.item.confirmation_number) || null, + }); + } + + /** + * One transaction per mail: every entry, the bookings added on their own + * (each in a savepoint, so one the unique index refuses is a duplicate and + * the others still land), and the mail's own status. A crash before the + * commit leaves the mail to be read again, never half added. + */ + private async write( + message: MailIngestMessageRow, + planned: readonly PlannedBooking[], + outcome: MailIngestMessageOutcome, + now: Date, + ): Promise { + const createdAt = dbNow(now); + const undoUntil = dbNow(new Date(now.getTime() + UNDO_WINDOW_DAYS * DAY_MS)); + return this.uow.transactional(async () => { + const added = await planned.reduce>(async (previous, booking) => { + const done = await previous; + const written = booking.prepared ? await this.tryWrite(booking) : null; + const candidateId = await this.candidates.insertCandidate({ + messageId: message.id, + userId: message.user_id, + position: booking.position, + itemJson: encodeJson(MAIL_INGEST_ITEM, booking.draft.item), + source: booking.draft.source, + checksJson: encodeJson(MAIL_INGEST_CHECKS, booking.verdict.checks), + uncertainFields: encodeJson( + MAIL_INGEST_UNCERTAIN_FIELDS, + booking.prepared?.uncertain ?? booking.verdict.uncertain, + ), + matchJson: encodeJson(MAIL_INGEST_MATCH, { + kind: booking.draft.kind, + suggested_new_trip: booking.draft.suggestedNewTrip, + }), + suggestedTripId: booking.verdict.choice.suggestedTripId, + dedupeKey: booking.key, + contentHash: booking.hash, + resolution: written ? 'auto_applied' : booking.prepared ? 'duplicate' : booking.resolution, + reason: written || booking.prepared ? null : booking.reason, + createdAt, + ...(written && { + resolvedTripId: written.tripId, + createdIdsJson: encodeJson(MAIL_INGEST_CREATED, written.created), + resolvedAt: createdAt, + undoUntil, + }), + }); + return written && booking.trip ? [...done, { candidateId, trip: booking.trip, written }] : done; + }, Promise.resolve([])); + await this.messages.finish(message.id, { ...outcome, status: 'processed', reason: null }); + return added; + }); + } + + /** A savepoint around one booking: the unique index refusing it makes it a duplicate, not a failed mail. */ + private async tryWrite(booking: PlannedBooking): Promise { + const prepared = booking.prepared; + if (!prepared) return null; + try { + return await this.uow.transactional(() => this.apply.writeInTx(prepared)); + } catch (err) { + if (err instanceof UniqueConstraintViolationException) return null; + throw err; + } + } + + /** + * After the commit: what the trips are told, the mail's documents on the + * first booking of each trip, and the notifications. + * + * @txIndependent each step stands on its own after the commit: a document + * that fails to attach leaves the booking, and is noted on its entry. + */ + private async afterCommit( + added: readonly AddedEntry[], + planned: readonly PlannedBooking[], + mail: WalkedMail, + person: MailIngestPersonRow, + ): Promise { + const attachedTrips = new Set(); + await added.reduce(async (previous, entry) => { + await previous; + await this.apply.announce(entry.written); + if (!attachedTrips.has(entry.written.tripId)) { + attachedTrips.add(entry.written.tripId); + const { fileIds, error } = await this.apply.attachDocuments( + entry.written.tripId, + entry.written.reservation.id, + mail, + person.id, + ); + const created = { ...entry.written.created, file_ids: fileIds }; + await this.candidates.setAttachments(entry.candidateId, encodeJson(MAIL_INGEST_CREATED, created), error); + } + await this.notify.bookingAdded(person, { + candidateId: entry.candidateId, + tripId: entry.written.tripId, + tripTitle: entry.trip.title, + title: entry.written.reservation.title, + type: entry.written.reservation.type, + }); + }, Promise.resolve()); + if (planned.some((booking) => booking.resolution === 'pending' && booking.prepared === null)) { + await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id)); + } + } + + /** The entry standing for a mail that gave no booking, with the reason, so the ways out still apply. */ + private async keepForInbox( + message: MailIngestMessageRow, + outcome: MailIngestMessageOutcome | null, + reason: MailIngestReason | null, + now: Date, + ): Promise { + const createdAt = dbNow(now); + await this.uow.transactional(async () => { + await this.candidates.insertCandidate({ + messageId: message.id, + userId: message.user_id, + position: 0, + itemJson: null, + source: null, + checksJson: null, + uncertainFields: null, + matchJson: null, + suggestedTripId: message.trip_id, + dedupeKey: null, + contentHash: null, + resolution: 'pending', + reason, + createdAt, + }); + await this.messages.finish(message.id, { + ...(outcome ?? emptyOutcome(createdAt)), + status: 'processed', + reason, + }); + }); + } + + /** What the mail says about itself, before any booking in it is read. */ + private outcomeBase(mail: WalkedMail, person: MailIngestPersonRow, now: Date): MailIngestMessageOutcome { + const hints: MailIngestHint[] = []; + if (!senderVerified(mail, person.email)) hints.push('sender_unverified'); + if (mail.skippedAttachments > 0) hints.push('attachments_skipped'); + return { + status: 'processed', + reason: null, + subject: mail.subject?.slice(0, 300) ?? null, + fromAddr: mail.envelope.from?.slice(0, 320) ?? null, + fromVerified: !hints.includes('sender_unverified'), + authResults: mail.envelope.authenticationResults?.slice(0, 2000) ?? null, + hints: encodeJson(MAIL_INGEST_HINTS, hints), + attachments: encodeJson( + MAIL_INGEST_ATTACHMENTS, + mail.attachments.map((file) => ({ file_name: file.fileName, kind: file.kind, size: file.data.length })), + ), + forwardingCode: null, + aiUsed: false, + processedAt: dbNow(now), + }; + } + + /** A try that failed: again later, or given up after the last one. The mail's content stays out of the log. */ + private async failed(message: MailIngestMessageRow, err: unknown, now: Date): Promise { + logError(`[mail-ingest] Reading mail ${message.id} failed: ${err instanceof Error ? err.message : String(err)}`); + if (message.attempts < MAIL_INGEST_LIMITS.maxAttempts) { + await this.messages.retryLater(message.id, dbNow(new Date(now.getTime() + RETRY_STEP_MS * message.attempts))); + return; + } + await this.messages.finish(message.id, { + ...emptyOutcome(dbNow(now)), + status: 'failed', + reason: 'processing_failed', + }); + } +} + +function emptyOutcome(at: MailIngestMessageOutcome['processedAt']): MailIngestMessageOutcome { + return { + status: 'processed', + reason: null, + subject: null, + fromAddr: null, + fromVerified: false, + authResults: null, + hints: null, + attachments: null, + forwardingCode: null, + aiUsed: false, + processedAt: at, + }; +} diff --git a/server/src/nest/mail-ingest/mail-ingest-reading.service.ts b/server/src/nest/mail-ingest/mail-ingest-reading.service.ts new file mode 100644 index 000000000..1e9f555ac --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-reading.service.ts @@ -0,0 +1,203 @@ +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow } from '../../db/types'; +import { BookingImportService } from '../booking-import/booking-import.service'; +import { MailReadService } from '../llm-parse/mail-read.service'; +import type { MailReadBooking, MailReadTrip, WalkedMail } from '../llm-parse/mail-read.types'; +import { bookingStart, dateOf, normalizeReference, type MailDraft, type TripOption } from './mail-ingest-checks'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import { + MAIL_INGEST_REASON_BY_LLM_OUTCOME, + type BookingImportPreviewItem, + type LlmParseOutcome, + type MailIngestPrefs, + type MailIngestReason, +} from '@trek/shared'; + +/** What reading a mail gave. */ +export interface MailReading { + drafts: MailDraft[]; + /** Whether a request went to the AI, which counts against the person's daily limit. */ + aiUsed: boolean; + /** Why the mail gave no booking; null when it gave some. */ + emptyReason: MailIngestReason | null; + /** Everything the mail and its attachments say, for the in-text check. */ + documentText: string; +} + +/** The outcomes of a reading that never reached a model. */ +const NOT_SENT: ReadonlySet = new Set(['ai_disabled', 'no_text', 'user_cap']); +const ISO_DATE_RE = /\b(20\d\d-[01]\d-[0-3]\d)\b/g; +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * Reads the bookings out of a forwarded mail. The machine-readable parts + * (JSON-LD, calendar files, passes, through kitinerary) are read always and + * cost nothing. The AI reads the mail when the person allows it and is under + * their daily limit, and picks a trip from the list it is given; where both + * name a booking, the machine-readable values win, and a critical field they + * disagree on sends the booking to the inbox (G4). + */ +@Injectable() +export class MailIngestReadingService { + constructor( + private readonly bookingImport: BookingImportService, + private readonly reader: MailReadService, + private readonly settings: MailIngestSettingsService, + private readonly trips: MailIngestTripsService, + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + ) {} + + /** The bookings the machine-readable parts of the mail name. */ + async readMachine(raw: Buffer, mail: WalkedMail, userId: number): Promise { + const attachments = mail.attachments.map(({ fileName, data }) => ({ fileName, data })); + return this.bookingImport.readMailBookings(raw, attachments, userId); + } + + /** The dates the mail names: they rank the trips the AI is offered. */ + hintDates(mail: WalkedMail, machine: readonly BookingImportPreviewItem[]): string[] { + const named = [...`${mail.subject ?? ''}\n${mail.body}`.matchAll(ISO_DATE_RE)].map((match) => match[1]); + const read = machine.map((item) => dateOf(bookingStart(item))).filter((date): date is string => date !== null); + return [...new Set([...read, ...named])]; + } + + async read(args: { + mail: WalkedMail; + machine: BookingImportPreviewItem[]; + person: MailIngestPersonRow; + prefs: MailIngestPrefs; + trips: readonly TripOption[]; + now: Date; + }): Promise { + const { mail, machine, person, prefs, trips, now } = args; + const skipped = await this.aiSkipped(person.id, prefs, now); + const refs = trips.map((trip, i) => ({ trip, ref: `t${i + 1}` })); + const reading = skipped + ? null + : await this.reader.read( + { + subject: mail.subject, + date: mail.envelope.date, + body: mail.body, + attachments: mail.attachments, + hints: machine.map(hintOf), + trips: refs.map(({ trip, ref }) => this.tripForAi(trip, ref)), + }, + person.id, + now.toISOString().slice(0, 10), + ); + const documentText = reading?.documentText ?? `${mail.subject ?? ''}\n${mail.body}`; + const aiUsed = reading !== null && !NOT_SENT.has(reading.outcome); + + if (reading && reading.bookings.length > 0) { + const tripIdOf = new Map(refs.map(({ trip, ref }) => [ref, trip.id])); + const drafts = await this.fromAi(reading.bookings, machine, tripIdOf, reading.imageOnly); + return { drafts, aiUsed, emptyReason: null, documentText }; + } + if (machine.length > 0) return { drafts: machine.map(fromMachine), aiUsed, emptyReason: null, documentText }; + const emptyReason = reading ? MAIL_INGEST_REASON_BY_LLM_OUTCOME[reading.outcome] : skipped; + return { drafts: [], aiUsed, emptyReason: emptyReason ?? 'not_a_booking', documentText }; + } + + /** Why the AI does not read this mail: switched off, not set up, or the person's daily limit is reached. */ + private async aiSkipped(userId: number, prefs: MailIngestPrefs, now: Date): Promise { + if (!prefs.ai_enabled || !(await this.reader.isAvailable(userId))) return 'ai_disabled'; + const since = dbNow(new Date(now.getTime() - DAY_MS)); + if ((await this.messages.countAiReadSince(userId, since)) >= (await this.settings.aiDailyCap())) { + return 'ai_user_cap'; + } + return null; + } + + private tripForAi(trip: TripOption, ref: string): MailReadTrip { + return { + ref, + title: trip.title.slice(0, 80), + start_date: trip.start_date, + end_date: trip.end_date, + destinations: this.trips.destinationsOf(trip), + }; + } + + private async fromAi( + bookings: readonly MailReadBooking[], + machine: readonly BookingImportPreviewItem[], + tripIdOf: ReadonlyMap, + imageOnly: boolean, + ): Promise { + const mapped = await Promise.all( + bookings.map((booking) => this.bookingImport.mapAiReservations(booking.reservations, 'mail')), + ); + return bookings + .flatMap((booking, i) => mapped[i].map((item) => ({ booking, item }))) + .slice(0, MAIL_INGEST_LIMITS.storedBookings) + .map(({ booking, item }) => { + const match = machine.find((candidate) => sameBooking(candidate, item)); + const disagrees = match !== undefined && criticalDiffers(match, item); + return { + item: match && !disagrees ? match : item, + source: 'ai' as const, + kind: booking.kind, + aiTripId: booking.tripRef === null ? null : (tripIdOf.get(booking.tripRef) ?? null), + uncertainFields: booking.uncertainFields, + suggestedNewTrip: booking.suggestedNewTrip, + imageOnly: imageOnly && match === undefined, + disagrees, + }; + }); + } +} + +function fromMachine(item: BookingImportPreviewItem): MailDraft { + return { + item, + source: 'kitinerary', + kind: 'new_booking', + aiTripId: null, + uncertainFields: [], + suggestedNewTrip: null, + imageOnly: false, + disagrees: false, + }; +} + +/** A booking as a hint for the AI: what it is, when, its reference and route, no more. */ +function hintOf(item: BookingImportPreviewItem): Record { + return { + type: item.type, + title: item.title, + start: bookingStart(item), + reference: item.confirmation_number ?? null, + route: (item.endpoints ?? []).map((endpoint) => endpoint.code ?? endpoint.name), + }; +} + +/** The same booking read twice: one type and one reference, or one type starting on one day. */ +function sameBooking(a: BookingImportPreviewItem, b: BookingImportPreviewItem): boolean { + if (a.type !== b.type) return false; + const reference = normalizeReference(a.confirmation_number); + if (reference && reference === normalizeReference(b.confirmation_number)) return true; + const day = dateOf(bookingStart(a)); + return day !== null && day === dateOf(bookingStart(b)); +} + +/** G4: a critical field both readings give, read differently: the start to the minute, the reference, the route. */ +function criticalDiffers(machine: BookingImportPreviewItem, ai: BookingImportPreviewItem): boolean { + const differs = (x: string | null, y: string | null) => x !== null && y !== null && x !== y; + const minute = (item: BookingImportPreviewItem) => bookingStart(item)?.slice(0, 16) ?? null; + const reference = (item: BookingImportPreviewItem) => normalizeReference(item.confirmation_number) || null; + const route = (item: BookingImportPreviewItem) => { + const codes = (item.endpoints ?? []).map((endpoint) => endpoint.code).filter((code): code is string => !!code); + return codes.length >= 2 ? `${codes[0]}-${codes[codes.length - 1]}`.toUpperCase() : null; + }; + return ( + differs(minute(machine), minute(ai)) || + differs(reference(machine), reference(ai)) || + differs(route(machine), route(ai)) + ); +} diff --git a/server/src/nest/mail-ingest/mail-ingest-retention.job.ts b/server/src/nest/mail-ingest/mail-ingest-retention.job.ts new file mode 100644 index 000000000..e953063ce --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-retention.job.ts @@ -0,0 +1,53 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { dbNow } from '../../db/types'; +import { logError, logInfo } from '../audit/audit-log.logger'; +import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; + +const DAY_MS = 24 * 60 * 60 * 1000; +/** A mail is kept this long after it was settled, for the ways out of the inbox and an undo. */ +const RAW_AFTER_DONE_DAYS = 7; +/** And never longer than this after it came in. */ +const RAW_MAX_DAYS = 30; +/** Decided entries, and mails without any, are kept this long, for the person's history and the admin's counts. */ +const DECIDED_DAYS = 90; + +/** + * The nightly sweep of the mail import (plan section 7): the stored mail goes + * 7 days after it was settled and at the latest 30 days after it came in; + * decided entries and the mails left without one go after 90 days. Waiting + * entries stay until the person decides. Bookings the import added are + * ordinary bookings and are not touched. + */ +@Injectable() +export class MailIngestRetentionJob implements OnApplicationBootstrap { + constructor( + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + private readonly registrar: CronRegistrarService, + ) {} + + onApplicationBootstrap(): void { + if (!this.registrar.isEnabled()) return; + this.registrar.register('mail-ingest-retention', '20 4 * * *', () => this.tick()); + } + + /** @txIndependent a retention sweep in passes, each complete on its own. */ + async tick(now: Date = new Date()): Promise { + const before = (days: number) => dbNow(new Date(now.getTime() - days * DAY_MS)); + try { + const purged = await this.messages.purgeRaw(before(RAW_AFTER_DONE_DAYS), before(RAW_MAX_DAYS), dbNow(now)); + const entries = await this.candidates.deleteResolvedBefore(before(DECIDED_DAYS)); + const mails = await this.messages.deleteSettledBefore(before(DECIDED_DAYS)); + if (purged + entries + mails > 0) { + logInfo(`Mail import retention: ${purged} mail(s) purged, ${entries} entr(ies) and ${mails} mail(s) removed`); + } + } catch (err) { + logError(`Mail import retention: ${err instanceof Error ? err.message : String(err)}`); + } + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts new file mode 100644 index 000000000..509aa20b8 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts @@ -0,0 +1,87 @@ +import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity'; +import type { MailIngestPrefsPatch, MailIngestPrefsRepository } from '../../db/repositories/MailIngestPrefs.repository'; +import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { DEFAULT_ALLOWED_EXTENSIONS } from '../files/files.constants'; +import { SettingsService } from '../settings/settings.service'; +import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import { + mailIngestTransportSettingSchema, + type MailIngestPrefs as MailIngestPrefsView, + type MailIngestPrefsUpdateRequest, + type MailIngestTransportSetting, +} from '@trek/shared'; + +/** The daily AI cap when the setting is missing or unreadable. */ +const DEFAULT_AI_PER_USER_DAILY = 10; + +/** + * Whether the mail import runs on this install, how mail reaches it, and the + * person's own switches. + * + * The import is off until `mail_ingest_enabled` says otherwise. On a managed + * install that setting is the operator's: it is written for the plans that + * include the import and never for Start, so on Start every surface reports + * the import as unavailable and nothing is read. + */ +@Injectable() +export class MailIngestSettingsService { + constructor( + private readonly settings: SettingsService, + private readonly env: RuntimeEnvService, + @InjectRepository(MailIngestPrefs) private readonly prefs: MailIngestPrefsRepository, + ) {} + + async isAvailable(): Promise { + return (await this.settings.getAppSetting('mail_ingest_enabled')) === 'true'; + } + + /** How mail reaches the install; a managed one takes it from the TREK Cloud relay alone. */ + async transport(): Promise { + if (this.env.isManaged()) return 'cloud'; + const stored = await this.settings.getAppSetting('mail_ingest_transport'); + const parsed = mailIngestTransportSettingSchema.safeParse(stored); + return parsed.success ? parsed.data : 'off'; + } + + /** The forwarding address with `{token}` where a person's token goes; null until an admin set one. */ + async addressTemplate(): Promise { + const template = (await this.settings.getAppSetting('mail_ingest_address_template'))?.trim(); + return template && template.includes('{token}') ? template : null; + } + + async aiDailyCap(): Promise { + const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily')); + return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY; + } + + /** The largest mail taken in: more on the TREK Cloud, whose relay already capped it. */ + maxMailBytes(): number { + return this.env.isManaged() ? MAIL_INGEST_LIMITS.cloudMailBytes : MAIL_INGEST_LIMITS.selfHostedMailBytes; + } + + /** The file types a trip takes, as the admin set them, for the documents a mail leaves behind. */ + async allowedFileTypes(): Promise { + return (await this.settings.getAppSetting('allowed_file_types')) || DEFAULT_ALLOWED_EXTENSIONS; + } + + /** The person's switches; a person who never changed one has every one on. */ + async prefsOf(userId: number): Promise { + const row = await this.prefs.findForUser(userId); + return { + enabled: row ? row.enabled === 1 : true, + ai_enabled: row ? row.ai_enabled === 1 : true, + auto_add: row ? row.auto_add === 1 : true, + }; + } + + async updatePrefs(userId: number, update: MailIngestPrefsUpdateRequest): Promise { + const patch: MailIngestPrefsPatch = {}; + if (update.enabled !== undefined) patch.enabled = update.enabled ? 1 : 0; + if (update.ai_enabled !== undefined) patch.ai_enabled = update.ai_enabled ? 1 : 0; + if (update.auto_add !== undefined) patch.auto_add = update.auto_add ? 1 : 0; + await this.prefs.upsertForUser(userId, patch); + return this.prefsOf(userId); + } +} diff --git a/server/src/nest/mail-ingest/mail-ingest-trips.service.ts b/server/src/nest/mail-ingest/mail-ingest-trips.service.ts new file mode 100644 index 000000000..a667caa78 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-trips.service.ts @@ -0,0 +1,90 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import type { + MailIngestCandidatesRepository, + MailIngestPersonRow, + MailIngestTripRow, +} from '../../db/repositories/MailIngestCandidates.repository'; +import { PermissionsService } from '../permissions/permissions.service'; +import type { TripOption } from './mail-ingest-checks'; +import { MAX_TRIP_CANDIDATES, TRIP_LOOKBACK_DAYS } from './mail-ingest.constants'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; + +/** Open trips read before the permission filter, so trips a person may not edit cannot crowd out the rest. */ +const TRIPS_READ = 60; +/** The destinations a trip names to the AI. */ +const DESTINATIONS_PER_TRIP = 5; +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * The trips a booking from a mail may go to: those the person may add + * bookings to (`reservation_edit`), not archived, that ended at most 30 days + * ago or have no dates, at most 20 of them. The list is the mail import's + * whole world: the AI picks from it, the checks weigh it, and nothing outside + * it is ever written to. + */ +@Injectable() +export class MailIngestTripsService { + constructor( + @InjectRepository(MailIngestCandidates) private readonly reads: MailIngestCandidatesRepository, + private readonly permissions: PermissionsService, + ) {} + + async person(userId: number): Promise { + return this.reads.findPerson(userId); + } + + /** + * The trips the person may add a booking to, those whose dates hold one of + * `hintDates` first (the dates the mail names), then by start, newest first. + */ + async editableTrips(person: MailIngestPersonRow, now: Date, hintDates: readonly string[]): Promise { + const cutoff = new Date(now.getTime() - TRIP_LOOKBACK_DAYS * DAY_MS).toISOString().slice(0, 10); + const rows = await this.reads.listOpenTrips(person.id, cutoff, TRIPS_READ); + const allowed = await Promise.all(rows.map((row) => this.mayEdit(person, row))); + const ranked = rows + .filter((_, i) => allowed[i]) + .map((row, order) => ({ row, order, hit: hintDates.some((date) => covers(row, date)) })) + .sort((a, b) => Number(b.hit) - Number(a.hit) || a.order - b.order) + .slice(0, MAX_TRIP_CANDIDATES) + .map(({ row }) => row); + return this.withDaysAndPlaces(ranked); + } + + /** One trip the person may add a booking to, whatever its dates; null when they may not. */ + async editableTrip(person: MailIngestPersonRow, tripId: number): Promise { + const row = await this.reads.findOpenTrip(person.id, tripId); + if (!row || !(await this.mayEdit(person, row))) return null; + return (await this.withDaysAndPlaces([row]))[0]; + } + + /** The names a trip is known by, for the AI's list: its places' names, the first few. */ + destinationsOf(trip: TripOption): string[] { + return [...new Set(trip.places.map((place) => place.name))].slice(0, DESTINATIONS_PER_TRIP); + } + + private async mayEdit(person: MailIngestPersonRow, trip: MailIngestTripRow): Promise { + const isMember = trip.user_id !== person.id; + return this.permissions.checkPermission('reservation_edit', person.role, trip.user_id, person.id, isMember); + } + + private async withDaysAndPlaces(rows: readonly MailIngestTripRow[]): Promise { + const ids = rows.map((row) => row.id); + const days = await this.reads.listDatedDays(ids); + const places = await this.reads.listPlacesWithPosition(ids); + return rows.map((row) => ({ + id: row.id, + title: row.title, + start_date: row.start_date, + end_date: row.end_date, + ownerId: row.user_id, + currency: (row.currency || 'EUR').toUpperCase(), + days: new Map(days.filter((day) => day.trip_id === row.id).map((day) => [day.date, day.id])), + places: places.filter((place) => place.trip_id === row.id), + })); + } +} + +function covers(trip: MailIngestTripRow, date: string): boolean { + return !!trip.start_date && !!trip.end_date && trip.start_date <= date && date <= trip.end_date; +} diff --git a/server/src/nest/mail-ingest/mail-ingest.constants.ts b/server/src/nest/mail-ingest/mail-ingest.constants.ts new file mode 100644 index 000000000..0c808b87b --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest.constants.ts @@ -0,0 +1,42 @@ +/** The `external_source` of a booking the mail import wrote. */ +export const MAIL_EXTERNAL_SOURCE = 'mail'; + +/** The sources whose keys say a booking is in a trip already: the file import's and the mail import's. */ +export const IMPORT_SOURCES: readonly string[] = ['import', MAIL_EXTERNAL_SOURCE]; + +/** The limits of the import (plan section 11). */ +export const MAIL_INGEST_LIMITS = { + /** The largest mail a self-hosted install takes in, and the largest the TREK Cloud relay hands on. */ + selfHostedMailBytes: 10 * 1024 * 1024, + cloudMailBytes: 25 * 1024 * 1024, + /** Bookings stored per mail, and bookings added on their own per mail. */ + storedBookings: 10, + autoBookings: 5, + /** Mails one run of the pipeline reads. */ + mailsPerRun: 3, + /** Mails one address takes in a day. */ + mailsPerAddressPerDay: 50, + /** Entries waiting in one person's inbox before new mails stop being read. */ + pendingPerUser: 200, + /** Tries before a mail that keeps failing is given up. */ + maxAttempts: 3, +} as const; + +/** How long an added booking can be undone in one tap. */ +export const UNDO_WINDOW_DAYS = 7; + +/** Trips that ended longer ago than this are not offered for a new booking. */ +export const TRIP_LOOKBACK_DAYS = 30; + +/** The most trips a mail is weighed against. */ +export const MAX_TRIP_CANDIDATES = 20; + +/** Random base32 characters in a token: 24 of them are 120 bits. */ +export const TOKEN_LENGTH = 24; + +/** A booking whose destination lies within this of one of a trip's places belongs to that trip. */ +export const NEAR_TRIP_KM = 150; + +/** The in-app actions of the forwarder's notification. */ +export const UNDO_ACTION = 'mail_ingest_undo'; +export const KEEP_ACTION = 'mail_ingest_keep'; diff --git a/server/src/nest/mail-ingest/mail-ingest.module.ts b/server/src/nest/mail-ingest/mail-ingest.module.ts new file mode 100644 index 000000000..f8c162c73 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest.module.ts @@ -0,0 +1,85 @@ +import { MailIngestAddresses } from '../../db/entities/MailIngestAddresses.entity'; +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity'; +import { AddonsModule } from '../addons/addons.module'; +import { AirportsModule } from '../airports/airports.module'; +import { AppConfigModule } from '../app-config/app-config.module'; +import { BookingImportModule } from '../booking-import/booking-import.module'; +import { BudgetModule } from '../budget/budget.module'; +import { DemoModule } from '../common/demo.module'; +import { FilesModule } from '../files/files.module'; +import { LlmParseModule } from '../llm-parse/llm-parse.module'; +import { MapsModule } from '../maps/maps.module'; +import { NotificationsModule } from '../notifications/notifications.module'; +import { PermissionsModule } from '../permissions/permissions.module'; +import { PlacesModule } from '../places/places.module'; +import { ReservationsModule } from '../reservations/reservations.module'; +import { SchedulingModule } from '../scheduling/scheduling.module'; +import { SettingsModule } from '../settings/settings.module'; +import { MailIngestActionsService } from './mail-ingest-actions.service'; +import { MailIngestAddressService } from './mail-ingest-address.service'; +import { MailIngestApplyService } from './mail-ingest-apply.service'; +import { MailIngestIntakeService } from './mail-ingest-intake.service'; +import { MailIngestNotificationActions } from './mail-ingest-notification-actions'; +import { MailIngestNotifyService } from './mail-ingest-notify.service'; +import { MailIngestProcessJob } from './mail-ingest-process.job'; +import { MailIngestProcessService } from './mail-ingest-process.service'; +import { MailIngestReadingService } from './mail-ingest-reading.service'; +import { MailIngestRetentionJob } from './mail-ingest-retention.job'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { MikroOrmModule } from '@mikro-orm/nestjs'; +import { Module } from '@nestjs/common'; + +/** + * The booking mail import: a person forwards a booking mail to an address of + * their own, and the booking lands in the matching trip, or waits in their + * inbox with a reason. Transports (IMAP, the signed webhook, the TREK Cloud + * relay, an .eml upload) only hand mail to MailIngestIntakeService; the + * processing job reads it; everything a person does with an entry goes + * through MailIngestActionsService. + */ +@Module({ + imports: [ + AppConfigModule, + DemoModule, + SettingsModule, + AddonsModule, + PermissionsModule, + ReservationsModule, + PlacesModule, + BudgetModule, + FilesModule, + MapsModule, + AirportsModule, + NotificationsModule, + BookingImportModule, + LlmParseModule, + SchedulingModule, + MikroOrmModule.forFeature([ + MailIngestAddresses, + MailIngestPrefs, + MailIngestCursor, + MailIngestMessages, + MailIngestCandidates, + ]), + ], + providers: [ + MailIngestSettingsService, + MailIngestTripsService, + MailIngestAddressService, + MailIngestIntakeService, + MailIngestReadingService, + MailIngestApplyService, + MailIngestNotifyService, + MailIngestProcessService, + MailIngestActionsService, + MailIngestNotificationActions, + MailIngestProcessJob, + MailIngestRetentionJob, + ], + exports: [MailIngestSettingsService, MailIngestAddressService, MailIngestIntakeService, MailIngestActionsService], +}) +export class MailIngestModule {} diff --git a/server/src/nest/mail-ingest/mail-money.ts b/server/src/nest/mail-ingest/mail-money.ts new file mode 100644 index 000000000..30a2df6d3 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-money.ts @@ -0,0 +1,50 @@ +import { currencyDecimals, toMinor } from '@trek/shared'; + +/** + * A price from a mail as whole hundredths, or null when it cannot be read to + * the cent without guessing. A number is taken as it is. A text decides its + * decimal mark by itself: with both a dot and a comma the last one is the + * decimal mark; with one of them, two digits after it make it the decimal + * mark, three are a thousands group or a third decimal, so the reading is + * left alone. More decimals than the currency has are no price either. + */ +export function parsePriceMinor(value: unknown, currency: string | null): number | null { + const decimals = currency ? currencyDecimals(currency) : 2; + if (typeof value === 'number') return Number.isFinite(value) ? toMinor(value) : null; + if (typeof value !== 'string') return null; + const text = value.replace(/[^\d.,]/g, ''); + const parts = splitAmount(text); + if (!parts) return null; + const [whole, fraction] = parts; + if (fraction.length > decimals) return null; + return toMinor(Number(`${whole}.${fraction || '0'}`)); +} + +/** The whole and the fractional digits of an amount, or null when its separators are ambiguous. */ +function splitAmount(text: string): [string, string] | null { + if (/^\d+$/.test(text)) return [text, '']; + const lastDot = text.lastIndexOf('.'); + const lastComma = text.lastIndexOf(','); + if (lastDot >= 0 && lastComma >= 0) { + const decimalAt = Math.max(lastDot, lastComma); + const thousands = text[Math.min(lastDot, lastComma)]; + const whole = text.slice(0, decimalAt); + return grouped(whole, thousands) ? [digits(whole), text.slice(decimalAt + 1)] : null; + } + const mark = lastDot >= 0 ? '.' : ','; + const pieces = text.split(mark); + if (pieces.length > 2) return grouped(text, mark) ? [digits(text), ''] : null; + const [whole, fraction] = pieces; + if (!/^\d+$/.test(whole) || !/^\d+$/.test(fraction) || fraction.length === 3) return null; + return [whole, fraction]; +} + +/** Whether `text` is digits in groups of three behind its first group, split by `separator`. */ +function grouped(text: string, separator: string): boolean { + const [head, ...groups] = text.split(separator); + return /^\d{1,3}$/.test(head) && groups.length > 0 && groups.every((group) => /^\d{3}$/.test(group)); +} + +function digits(text: string): string { + return text.replace(/\D/g, ''); +} diff --git a/server/src/nest/mail-ingest/mail-seal.ts b/server/src/nest/mail-ingest/mail-seal.ts new file mode 100644 index 000000000..5a48ad8e9 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-seal.ts @@ -0,0 +1,17 @@ +import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto'; + +/** + * The stored form of a forwarded mail: encrypted the way API keys are. A mail + * is bytes; latin1 maps each byte to one character and back, so nothing is + * lost on the way through the text column. + */ +export function sealMail(raw: Buffer): string { + return encrypt_api_key(raw.toString('latin1')); +} + +/** The mail a sealed text holds; null for one that does not open (another key, a broken row). */ +export function openMail(sealed: string | null): Buffer | null { + if (!sealed) return null; + const text = decrypt_api_key(sealed); + return text === null ? null : Buffer.from(text, 'latin1'); +} diff --git a/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts b/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts index 4c4802f97..0b1c523df 100644 --- a/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts +++ b/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts @@ -15,9 +15,10 @@ import type { MailIngestMessagesRepository } from '../../../../src/db/repositori import { dbNow } from '../../../../src/db/types'; import { createSnapshotTestDb } from '../../../helpers/db-mock'; import { addBudgetItemPayer, makeBudgetItem } from '../../../helpers/factories/budget'; +import { makeDayAssignment } from '../../../helpers/factories/itinerary'; import { makePlace } from '../../../helpers/factories/places'; import { makeReservation } from '../../../helpers/factories/reservations'; -import { addTripMember, makeTrip } from '../../../helpers/factories/trips'; +import { addTripMember, makeTrip, readTripDays } from '../../../helpers/factories/trips'; import { makeUser } from '../../../helpers/factories/users'; import { resetTestDb } from '../../../helpers/test-db'; import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; @@ -67,7 +68,8 @@ async function mailOf(userId: number): Promise { rawEnc: 'enc', receivedAt: DAY1, }); - return id!; + if (id === null) throw new Error('the mail was stored already'); + return id; } function entry(messageId: number, userId: number, over: Partial = {}) { @@ -128,6 +130,26 @@ describe('MailIngestCandidatesRepository: the entries', () => { }); }); + it('writes an entry decided as it is written with its trip, ids and undo window', async () => { + const { user } = await makeUser(t); + const trip = await makeTrip(t, user.id); + const until = at('2026-11-08T10:00:00Z'); + const id = await entry(await mailOf(user.id), user.id, { + resolution: 'auto_applied', + reason: null, + resolvedTripId: trip.id, + createdIdsJson: '{"reservation_ids":[3]}', + resolvedAt: DAY1, + undoUntil: until, + }); + expect(await candidates.findById(id)).toMatchObject({ + resolved_trip_id: trip.id, + created_ids_json: '{"reservation_ids":[3]}', + resolved_at: DAY1, + undo_until: until, + }); + }); + it('remembers a booking undone in a trip by its content', async () => { const { user } = await makeUser(t); const trip = await makeTrip(t, user.id); @@ -184,17 +206,32 @@ describe('MailIngestCandidatesRepository: what it reads of the trips', () => { expect(await candidates.listOpenTrips(user.id, '2026-10-01', 1)).toHaveLength(1); }); + it('finds one open trip of the person whatever its dates, and none of a stranger', async () => { + const { user } = await makeUser(t); + const { user: other } = await makeUser(t); + const ended = await makeTrip(t, user.id, { start_date: '2025-01-01', end_date: '2025-01-02' }); + const archived = await makeTrip(t, user.id, { is_archived: 1 }); + const foreign = await makeTrip(t, other.id); + expect(await candidates.findOpenTrip(user.id, ended.id)).toMatchObject({ id: ended.id, user_id: user.id }); + expect(await candidates.findOpenTrip(user.id, archived.id)).toBeNull(); + expect(await candidates.findOpenTrip(user.id, foreign.id)).toBeNull(); + await addTripMember(t, foreign.id, user.id); + expect(await candidates.findOpenTrip(user.id, foreign.id)).toMatchObject({ id: foreign.id }); + }); + it('reads the dated days and the placed places of trips', async () => { const { user } = await makeUser(t); const trip = await makeTrip(t, user.id, { start_date: '2027-03-01', end_date: '2027-03-02' }); await makePlace(t, trip.id, { name: 'Hotel Sakura', address: 'Tokyo', lat: 35.68, lng: 139.76 }); await makePlace(t, trip.id, { name: 'Somewhere', lat: null, lng: null }); - expect(await candidates.listDayDates([trip.id])).toEqual([ - { trip_id: trip.id, date: '2027-03-01' }, - { trip_id: trip.id, date: '2027-03-02' }, + const days = await candidates.listDatedDays([trip.id]); + expect(days.map(({ trip_id, date }) => [trip_id, date])).toEqual([ + [trip.id, '2027-03-01'], + [trip.id, '2027-03-02'], ]); - expect(await candidates.listDayDates([])).toEqual([]); + expect(days.every((day) => day.id > 0)).toBe(true); + expect(await candidates.listDatedDays([])).toEqual([]); expect(await candidates.listPlacesWithPosition([trip.id])).toEqual([ { trip_id: trip.id, name: 'Hotel Sakura', address: 'Tokyo', lat: 35.68, lng: 139.76 }, ]); @@ -243,6 +280,20 @@ describe('MailIngestCandidatesRepository: what it reads of the trips', () => { ).toBe(false); }); + it('tells a place something still names from one nothing does', async () => { + const { user } = await makeUser(t); + const trip = await makeTrip(t, user.id, { start_date: '2027-03-01', end_date: '2027-03-01' }); + const [day] = await readTripDays(t, trip.id); + const free = await makePlace(t, trip.id); + const booked = await makePlace(t, trip.id); + const planned = await makePlace(t, trip.id); + await makeReservation(t, trip.id, { place: booked.id }); + await makeDayAssignment(t, day.id, planned.id); + expect(await candidates.placeInUse(free.id)).toBe(false); + expect(await candidates.placeInUse(booked.id)).toBe(true); + expect(await candidates.placeInUse(planned.id)).toBe(true); + }); + it('counts the payers of costs', async () => { const { user } = await makeUser(t); const trip = await makeTrip(t, user.id); diff --git a/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts b/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts index 549c83f49..f7faf44ec 100644 --- a/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts +++ b/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts @@ -56,7 +56,14 @@ async function inbox() { return { userId: user.id, addressId }; } -function store(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1) { +/** Stores a mail and answers its id; a second store of the same hash fails the test. */ +async function store(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1): Promise { + const id = await insert(box, hash, receivedAt); + if (id === null) throw new Error(`mail ${hash} was stored already`); + return id; +} + +function insert(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1) { return messages.insertIfNew({ transport: 'upload', messageIdHdr: `<${hash}@x>`, @@ -88,12 +95,11 @@ describe('MailIngestMessagesRepository', () => { it('stores a mail once and finds it by its hash', async () => { const box = await inbox(); const id = await store(box, 'm1'); - expect(id).not.toBeNull(); - expect(await store(box, 'm1')).toBeNull(); + expect(await insert(box, 'm1')).toBeNull(); expect(await messages.findIdByDedupeHash('m1')).toBe(id); expect(await messages.findIdByDedupeHash('m2')).toBeNull(); - expect(await messages.findById(id!)).toMatchObject({ status: 'queued', attempts: 0, raw_eml_enc: 'enc' }); - expect(await messages.findById(id! + 1)).toBeNull(); + expect(await messages.findById(id)).toMatchObject({ status: 'queued', attempts: 0, raw_eml_enc: 'enc' }); + expect(await messages.findById(id + 1)).toBeNull(); }); it('counts the mails of an address and the AI-read mails of a person since a time', async () => { @@ -101,7 +107,7 @@ describe('MailIngestMessagesRepository', () => { const first = await store(box, 'c1', at('2026-10-31T10:00:00Z')); await store(box, 'c2', DAY1); expect(await messages.countForAddressSince(box.addressId, at('2026-11-01T00:00:00Z'))).toBe(1); - await messages.finish(first!, DONE); + await messages.finish(first, DONE); expect(await messages.countAiReadSince(box.userId, at('2026-10-31T00:00:00Z'))).toBe(1); expect(await messages.countAiReadSince(box.userId, at('2026-11-01T00:00:00Z'))).toBe(0); }); @@ -118,14 +124,14 @@ describe('MailIngestMessagesRepository', () => { const later = at('2026-11-01T11:20:00Z'); expect(await messages.claimDue(later, at('2026-11-01T11:30:00Z'), 3)).toEqual([id]); - expect(await messages.findById(id!)).toMatchObject({ status: 'processing', attempts: 2 }); + expect(await messages.findById(id)).toMatchObject({ status: 'processing', attempts: 2 }); - await messages.retryLater(id!, at('2026-11-01T12:00:00Z')); + await messages.retryLater(id, at('2026-11-01T12:00:00Z')); expect(await messages.claimDue(later, lease, 3)).toEqual([]); expect(await messages.claimDue(at('2026-11-01T12:00:00Z'), lease, 3)).toEqual([id]); - await messages.finish(id!, DONE); - expect(await messages.findById(id!)).toMatchObject({ + await messages.finish(id, DONE); + expect(await messages.findById(id)).toMatchObject({ status: 'processed', ai_used: 1, from_verified: 1, @@ -138,10 +144,10 @@ describe('MailIngestMessagesRepository', () => { const waiting = await store(box, 'p1'); const settled = await store(box, 'p2'); const old = await store(box, 'p3', at('2026-09-01T10:00:00Z')); - await messages.finish(waiting!, DONE); - await messages.finish(settled!, DONE); + await messages.finish(waiting, DONE); + await messages.finish(settled, DONE); await candidates.insertCandidate({ - messageId: waiting!, + messageId: waiting, userId: box.userId, position: 0, itemJson: null, @@ -159,9 +165,9 @@ describe('MailIngestMessagesRepository', () => { const purgedAt = at('2026-11-20T10:00:00Z'); expect(await messages.purgeRaw(at('2026-11-10T10:00:00Z'), at('2026-10-01T10:00:00Z'), purgedAt)).toBe(2); - expect(await messages.findById(waiting!)).toMatchObject({ raw_eml_enc: 'enc', raw_purged_at: null }); - expect(await messages.findById(settled!)).toMatchObject({ raw_eml_enc: null, raw_purged_at: purgedAt }); - expect(await messages.findById(old!)).toMatchObject({ raw_eml_enc: null }); + expect(await messages.findById(waiting)).toMatchObject({ raw_eml_enc: 'enc', raw_purged_at: null }); + expect(await messages.findById(settled)).toMatchObject({ raw_eml_enc: null, raw_purged_at: purgedAt }); + expect(await messages.findById(old)).toMatchObject({ raw_eml_enc: null }); }); it('deletes settled mails without entries, and keeps queued ones and those with entries', async () => { @@ -169,10 +175,10 @@ describe('MailIngestMessagesRepository', () => { const queued = await store(box, 'x1', at('2026-08-01T10:00:00Z')); const bare = await store(box, 'x2', at('2026-08-01T10:00:00Z')); const withEntry = await store(box, 'x3', at('2026-08-01T10:00:00Z')); - await messages.finish(bare!, { ...DONE, status: 'discarded' }); - await messages.finish(withEntry!, DONE); + await messages.finish(bare, { ...DONE, status: 'discarded' }); + await messages.finish(withEntry, DONE); await candidates.insertCandidate({ - messageId: withEntry!, + messageId: withEntry, userId: box.userId, position: 0, itemJson: null, @@ -189,8 +195,8 @@ describe('MailIngestMessagesRepository', () => { }); expect(await messages.deleteSettledBefore(at('2026-09-01T00:00:00Z'))).toBe(1); - expect(await messages.findById(bare!)).toBeNull(); - expect(await messages.findById(queued!)).not.toBeNull(); - expect(await messages.findById(withEntry!)).not.toBeNull(); + expect(await messages.findById(bare)).toBeNull(); + expect(await messages.findById(queued)).not.toBeNull(); + expect(await messages.findById(withEntry)).not.toBeNull(); }); }); diff --git a/server/tests/unit/nest/llm-parse/mail-read.service.test.ts b/server/tests/unit/nest/llm-parse/mail-read.service.test.ts index 3110ee9d0..a5f7d3d48 100644 --- a/server/tests/unit/nest/llm-parse/mail-read.service.test.ts +++ b/server/tests/unit/nest/llm-parse/mail-read.service.test.ts @@ -122,11 +122,17 @@ describe('MailReadService.read', () => { }); it('reports a spent allowance as its outcome and logs no content', async () => { - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined); + const lines: string[] = []; + for (const method of ['log', 'info', 'warn', 'error'] as const) { + vi.spyOn(console, method).mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }); + } extract.mockRejectedValue(new LlmRequestError('gate', 429, 'QUOTA_SPENT')); const reading = await service.read(input(), 1, '2026-11-13'); expect(reading.outcome).toBe('quota_exhausted'); - expect(String(error.mock.calls[0][0])).not.toContain('ABC123'); + expect(lines.join(' | ')).toContain('Mail read failed'); + expect(lines.join(' | ')).not.toContain('ABC123'); }); it('reads through the local router without a trip choice', async () => { diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-checks.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-checks.test.ts new file mode 100644 index 000000000..161c4b528 --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-checks.test.ts @@ -0,0 +1,246 @@ +/** + * The server's checks on a booking read from a mail (G1 to G8) and the choice + * of its trip: the AI's pick is one input, the dates and the trip list decide, + * and every failure names the first reason the inbox shows. + */ +import { + checkInText, + chooseTrip, + contentHash, + fitsTrip, + implausible, + isCancellation, + judgeBooking, + missingCritical, + missingReference, + nearTrip, + type CheckContext, + type MailDraft, + type TripOption, +} from '../../../../src/nest/mail-ingest/mail-ingest-checks'; +import type { BookingImportPreviewItem } from '@trek/shared'; + +import { describe, expect, it } from 'vitest'; + +const NOW = new Date('2030-01-15T10:00:00Z'); +const endpoint = (role: 'from' | 'to', name: string, code: string, lat: number, lng: number) => ({ + role, + sequence: role === 'from' ? 0 : 1, + name, + code, + lat, + lng, + timezone: null, + local_time: null, + local_date: null, +}); +const HND = endpoint('from', 'Tokyo Haneda', 'HND', 35.55, 139.78); +const ITM = endpoint('to', 'Osaka Itami', 'ITM', 34.78, 135.44); +const FLIGHT: BookingImportPreviewItem = { + type: 'flight', + title: 'LH 716 Tokyo to Osaka', + reservation_time: '2030-03-02T10:00', + reservation_end_time: '2030-03-02T11:15', + confirmation_number: 'ABC123', + metadata: { flight_number: 'LH716', price: 120.5, priceCurrency: 'EUR' }, + endpoints: [HND, ITM], + source: { fileName: 'mail.eml', index: 0 }, +}; +const HOTEL: BookingImportPreviewItem = { + type: 'hotel', + title: 'Hotel Sakura', + confirmation_number: 'H-778', + _venue: { name: 'Hotel Sakura', lat: 35.01, lng: 135.76 }, + _accommodation: { check_in: '2030-03-03T15:00', check_out: '2030-03-05T11:00' }, + source: { fileName: 'mail.eml', index: 0 }, +}; +const TEXT = 'Your booking ABC123: flight LH 716 from Tokyo to Osaka. Hotel Sakura, reference H-778.'; + +function trip(id: number, start: string | null, end: string | null, over: Partial = {}): TripOption { + const days = new Map(); + if (start && end) { + for (let d = new Date(`${start}T00:00:00Z`), n = 1; d <= new Date(`${end}T00:00:00Z`); n++) { + days.set(d.toISOString().slice(0, 10), id * 100 + n); + d = new Date(d.getTime() + 86_400_000); + } + } + return { + id, + title: `Trip ${id}`, + start_date: start, + end_date: end, + ownerId: 1, + currency: 'EUR', + days, + places: [], + ...over, + }; +} +const JAPAN = trip(1, '2030-03-01', '2030-03-10'); +const context = (over: Partial = {}): CheckContext => ({ + trips: [JAPAN], + addressTripId: null, + subject: 'Your flight', + documentText: TEXT, + now: NOW, + knownAirport: (code) => ['HND', 'ITM'].includes(code), + ...over, +}); +const draft = (over: Partial = {}): MailDraft => ({ + item: FLIGHT, + source: 'kitinerary', + kind: 'new_booking', + aiTripId: null, + uncertainFields: [], + suggestedNewTrip: null, + imageOnly: false, + disagrees: false, + ...over, +}); + +describe('the single checks', () => { + it('needs a type, a start and, for a transport, both ends', () => { + expect(missingCritical(FLIGHT)).toBe(false); + expect(missingCritical({ ...FLIGHT, reservation_time: null })).toBe(true); + expect(missingCritical({ ...FLIGHT, endpoints: [HND] })).toBe(true); + expect(missingCritical(HOTEL)).toBe(false); + }); + + it('needs a booking reference for a transport, a stay or a rental only', () => { + expect(missingReference({ ...FLIGHT, confirmation_number: ' ' })).toBe(true); + expect(missingReference({ ...FLIGHT, type: 'restaurant', confirmation_number: null })).toBe(false); + }); + + it('refuses values that cannot be right (G5)', () => { + const known = (code: string) => code !== 'XXX'; + expect(implausible(FLIGHT, NOW, known)).toBe(false); + expect(implausible({ ...FLIGHT, reservation_time: 'soon' }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, reservation_time: '2034-01-01T10:00' }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, reservation_end_time: '2030-03-01T10:00' }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, reservation_end_time: '2030-03-03T10:00' }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, confirmation_number: 'A' }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, metadata: { price: 200_000 } }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, metadata: { priceCurrency: 'euro' } }, NOW, known)).toBe(true); + expect(implausible({ ...FLIGHT, metadata: { price: '1.234,50' } }, NOW, known)).toBe(false); + expect(implausible(FLIGHT, NOW, (code) => code !== 'ITM')).toBe(true); + const longStay = { ...HOTEL, _accommodation: { check_in: '2030-03-01', check_out: '2030-06-01' } }; + expect(implausible(longStay, NOW, known)).toBe(true); + }); + + it('vetoes a cancellation by kind, status or subject (G8)', () => { + const cancelled = { ...FLIGHT, metadata: { reservationStatus: 'ReservationCancelled' } }; + expect(isCancellation(FLIGHT, 'cancellation', null)).toBe(true); + expect(isCancellation(cancelled, 'new_booking', null)).toBe(true); + expect(isCancellation(FLIGHT, 'new_booking', 'Ihre Buchung wurde storniert')).toBe(true); + expect(isCancellation(FLIGHT, 'new_booking', 'Your booking at Hotel Sakura')).toBe(false); + }); + + it('wants an AI value in the mail, and only marks a free text it cannot find (G6)', () => { + expect(checkInText(FLIGHT, TEXT)).toEqual({ ok: true, uncertain: [] }); + expect(checkInText({ ...FLIGHT, confirmation_number: 'ZZZ999' }, TEXT).ok).toBe(false); + expect(checkInText({ ...FLIGHT, title: 'Paris Express', location: 'Gare du Nord' }, TEXT)).toEqual({ + ok: true, + uncertain: ['title', 'location'], + }); + }); + + it('fits a booking into a trip by its dates and the days it needs (G3)', () => { + expect(fitsTrip(FLIGHT, JAPAN)).toBe(true); + expect(fitsTrip(HOTEL, JAPAN)).toBe(true); + expect(fitsTrip(FLIGHT, trip(2, '2030-04-01', '2030-04-05'))).toBe(false); + expect(fitsTrip(FLIGHT, trip(3, null, null))).toBe(false); + // A day of slack is no day row: the booking needs the day it starts on. + expect(fitsTrip(FLIGHT, trip(4, '2030-03-03', '2030-03-05'))).toBe(false); + expect(fitsTrip({ ...FLIGHT, reservation_time: null }, JAPAN)).toBe(false); + }); + + it('confirms a booking lies near a trip by the trip’s places', () => { + const kyoto = trip(5, '2030-03-01', '2030-03-10', { + places: [{ name: 'Kyoto Station', address: null, lat: 34.98, lng: 135.75 }], + }); + expect(nearTrip(HOTEL, kyoto)).toBe(true); + expect(nearTrip(FLIGHT, kyoto)).toBe(true); + expect(nearTrip({ ...HOTEL, _venue: { name: 'Far', lat: 43.06, lng: 141.35 } }, kyoto)).toBe(false); + }); + + it('hashes what a booking says', () => { + expect(contentHash(FLIGHT)).toBe(contentHash({ ...FLIGHT, confirmation_number: 'abc 123' })); + expect(contentHash(FLIGHT)).not.toBe(contentHash({ ...FLIGHT, reservation_time: '2030-03-02T11:00' })); + }); +}); + +describe('chooseTrip', () => { + const second = trip(2, '2030-02-25', '2030-03-05', { + places: [{ name: 'Osaka', address: null, lat: 34.7, lng: 135.5 }], + }); + + it('takes the one trip whose dates fit', () => { + expect(chooseTrip(FLIGHT, null, context())).toEqual({ tripId: 1, suggestedTripId: 1, reason: null }); + }); + + it('of several, takes the AI’s pick only when the booking lies near it', () => { + const both = context({ trips: [JAPAN, second] }); + expect(chooseTrip(FLIGHT, 2, both)).toEqual({ tripId: 2, suggestedTripId: 2, reason: null }); + expect(chooseTrip(FLIGHT, 1, both)).toEqual({ tripId: null, suggestedTripId: 1, reason: 'trip_ambiguous' }); + expect(chooseTrip(FLIGHT, null, both)).toEqual({ tripId: null, suggestedTripId: 1, reason: 'trip_ambiguous' }); + }); + + it('suggests the AI’s pick when no trip fits, and names trips without dates', () => { + const later = trip(6, '2030-06-01', '2030-06-10'); + expect(chooseTrip(FLIGHT, 6, context({ trips: [later] }))).toEqual({ + tripId: null, + suggestedTripId: 6, + reason: 'no_trip_in_range', + }); + expect(chooseTrip(FLIGHT, 9, context({ trips: [trip(7, null, null)] })).reason).toBe('trip_without_dates'); + }); + + it('keeps a trip address’s booking to its trip, whatever the AI says', () => { + expect(chooseTrip(FLIGHT, 2, context({ addressTripId: 1, trips: [JAPAN, second] }))).toEqual({ + tripId: 1, + suggestedTripId: 1, + reason: null, + }); + const undated = trip(8, null, null); + expect(chooseTrip(FLIGHT, null, context({ addressTripId: 8, trips: [undated] })).tripId).toBe(8); + const elsewhen = trip(9, '2030-06-01', '2030-06-10'); + const outside = chooseTrip(FLIGHT, null, context({ addressTripId: 9, trips: [elsewhen] })); + expect(outside.reason).toBe('outside_trip_dates'); + expect(chooseTrip(FLIGHT, null, context({ addressTripId: 10 })).reason).toBe('no_edit_permission'); + }); +}); + +describe('judgeBooking', () => { + it('passes a machine-read booking that fits one trip', () => { + const verdict = judgeBooking(draft(), context()); + expect(verdict.reason).toBeNull(); + expect(verdict.checks).toMatchObject({ schema: 'pass', dates: 'pass', agreement: 'skipped', in_text: 'skipped' }); + expect(verdict.choice.tripId).toBe(1); + }); + + it('names the first reason in the order the inbox reads them', () => { + expect(judgeBooking(draft({ kind: 'change' }), context()).reason).toBe('change_or_cancellation'); + expect(judgeBooking(draft({ kind: 'cancellation' }), context()).reason).toBe('change_or_cancellation'); + expect(judgeBooking(draft(), context({ subject: 'Cancelled: your flight' })).reason).toBe('cancelled'); + expect(judgeBooking(draft({ item: { ...FLIGHT, endpoints: [] } }), context()).reason).toBe( + 'missing_critical_field', + ); + expect(judgeBooking(draft({ item: { ...FLIGHT, title: '' } }), context()).reason).toBe('missing_critical_field'); + expect(judgeBooking(draft(), context({ knownAirport: () => false })).reason).toBe('implausible'); + expect(judgeBooking(draft({ source: 'ai', disagrees: true }), context()).reason).toBe('source_mismatch'); + expect(judgeBooking(draft({ source: 'ai', imageOnly: true }), context()).reason).toBe('image_only'); + expect(judgeBooking(draft({ source: 'ai' }), context({ documentText: 'nothing' })).reason).toBe('not_in_text'); + const noReference = { ...FLIGHT, confirmation_number: null }; + expect(judgeBooking(draft({ item: noReference }), context()).reason).toBe('missing_confirmation'); + expect(judgeBooking(draft(), context({ trips: [] })).reason).toBe('no_trip_in_range'); + }); + + it('adds the fields it could not find to the AI’s own uncertain ones', () => { + const verdict = judgeBooking( + draft({ source: 'ai', uncertainFields: ['seat'], item: { ...FLIGHT, location: 'Terminal 9' } }), + context(), + ); + expect(verdict.uncertain).toEqual(['seat', 'location']); + expect(verdict.checks.in_text).toBe('pass'); + }); +}); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-helpers.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-helpers.test.ts new file mode 100644 index 000000000..8994336bd --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-helpers.test.ts @@ -0,0 +1,132 @@ +/** + * The mail import's pure pieces: what a mail is before anything reads it, a + * signed sender as a hint, a price read to the cent or not at all, the + * address token, and the sealed mail. + */ +import { walkMail } from '../../../../src/nest/llm-parse/mime-walker'; +import { addressOf, classifyMail, senderVerified } from '../../../../src/nest/mail-ingest/mail-classify'; +import { hashToken, newToken, tokenOf } from '../../../../src/nest/mail-ingest/mail-ingest-address.service'; +import { parsePriceMinor } from '../../../../src/nest/mail-ingest/mail-money'; +import { openMail, sealMail } from '../../../../src/nest/mail-ingest/mail-seal'; + +import { describe, expect, it } from 'vitest'; + +/** A mail with the given headers and body, as the walker reads it. */ +function mail(headers: Record, body: string) { + const head = Object.entries({ From: 'Mia ', Subject: 'Hello', ...headers }) + .map(([name, value]) => `${name}: ${value}`) + .join('\r\n'); + const walked = walkMail(Buffer.from(`${head}\r\nContent-Type: text/plain\r\n\r\n${body}`)); + if (!walked) throw new Error('the fixture is no mail'); + return walked; +} + +const BOOKING_BODY = 'Your booking ABC123 is confirmed. Flight LH 716 on 2030-03-02.'; + +describe('classifyMail', () => { + it('reads a booking mail', () => { + expect(classifyMail(mail({ Subject: 'Your flight' }, BOOKING_BODY))).toEqual({ kind: 'read' }); + }); + + it('shows the code of a forwarding confirmation and clicks nothing', () => { + const gmail = mail( + { From: 'forwarding-noreply@google.com', Subject: '(#123) Gmail Forwarding Confirmation' }, + 'Confirmation code: 612345678. To allow forwarding, click the link.', + ); + expect(classifyMail(gmail)).toEqual({ kind: 'forwarding_confirmation', code: '612345678' }); + }); + + it('drops an automatic reply, but not a confirmation a booking system generated', () => { + expect(classifyMail(mail({ 'Auto-Submitted': 'auto-replied' }, BOOKING_BODY))).toEqual({ + kind: 'drop', + reason: 'auto_reply', + }); + expect(classifyMail(mail({ Subject: 'Out of office' }, BOOKING_BODY)).kind).toBe('drop'); + expect(classifyMail(mail({ 'Auto-Submitted': 'auto-generated' }, BOOKING_BODY))).toEqual({ kind: 'read' }); + }); + + it('drops an empty mail', () => { + expect(classifyMail(mail({}, 'hi'))).toEqual({ kind: 'drop', reason: 'empty_mail' }); + }); + + it('drops a list mail without a booking in it, and keeps one that has one', () => { + const list = { 'List-Unsubscribe': '' }; + expect(classifyMail(mail(list, 'Our autumn deals are here, have a look at them all.'))).toEqual({ + kind: 'drop', + reason: 'newsletter', + }); + expect(classifyMail(mail(list, BOOKING_BODY))).toEqual({ kind: 'read' }); + const bulk = mail({ Precedence: 'bulk' }, 'Weekly news for travellers who love to read.'); + expect(classifyMail(bulk).kind).toBe('drop'); + }); +}); + +describe('senderVerified', () => { + const signed = { 'Authentication-Results': 'mx.example.net; dkim=pass header.d=example.com; spf=pass' }; + + it('holds for the person’s own address, signed by its domain', () => { + expect(senderVerified(mail(signed, BOOKING_BODY), 'Mia@Example.com')).toBe(true); + }); + + it('does not hold for another sender or without a signature', () => { + expect(senderVerified(mail(signed, BOOKING_BODY), 'other@example.com')).toBe(false); + expect(senderVerified(mail({}, BOOKING_BODY), 'mia@example.com')).toBe(false); + const forged = { 'Authentication-Results': 'mx.example.net; dkim=pass header.d=evil.test' }; + expect(senderVerified(mail(forged, BOOKING_BODY), 'mia@example.com')).toBe(false); + }); + + it('reads a bare address out of a From header', () => { + expect(addressOf('Mia ')).toBe('mia@example.com'); + expect(addressOf('mia@example.com')).toBe('mia@example.com'); + expect(addressOf(null)).toBeNull(); + }); +}); + +describe('parsePriceMinor', () => { + it('takes a number as it is and reads unambiguous text', () => { + expect(parsePriceMinor(120.5, 'EUR')).toBe(12050); + expect(parsePriceMinor('1.234,56', 'EUR')).toBe(123456); + expect(parsePriceMinor('1,234.56', 'USD')).toBe(123456); + expect(parsePriceMinor('€ 89,35', 'EUR')).toBe(8935); + expect(parsePriceMinor('1.234.567', 'EUR')).toBe(123456700); + expect(parsePriceMinor('12000', 'JPY')).toBe(1200000); + expect(parsePriceMinor('0,5', null)).toBe(50); + }); + + it('refuses what it would have to guess', () => { + expect(parsePriceMinor('1.234', 'EUR')).toBeNull(); + expect(parsePriceMinor('12.50', 'JPY')).toBeNull(); + expect(parsePriceMinor('1,2,3', 'EUR')).toBeNull(); + expect(parsePriceMinor('12.34,5.6', 'EUR')).toBeNull(); + expect(parsePriceMinor(Number.NaN, 'EUR')).toBeNull(); + expect(parsePriceMinor({ amount: 1 }, 'EUR')).toBeNull(); + expect(parsePriceMinor('', 'EUR')).toBeNull(); + }); +}); + +describe('the address token', () => { + it('is 24 base32 characters and looked up by its hash, whatever its case', () => { + const token = newToken(); + expect(token).toMatch(/^[a-z2-7]{24}$/); + expect(newToken()).not.toBe(token); + expect(hashToken(token.toUpperCase())).toBe(hashToken(token)); + }); + + it('is read from where the template puts it, or after a plus', () => { + const token = 'abcdefghijklmnopqrstuvwx'; + expect(tokenOf(`Trips `, 'trek+{token}@mail.example.com')).toBe(token); + expect(tokenOf(`ab12${token}@in.example.com`, 'ab12{token}@in.example.com')).toBe(token); + expect(tokenOf(`box+${token}@example.com`, null)).toBe(token); + expect(tokenOf('box+short@example.com', null)).toBeNull(); + expect(tokenOf('someone@example.com', 'trek+{token}@mail.example.com')).toBeNull(); + }); +}); + +describe('the sealed mail', () => { + it('opens to the same bytes, and a broken seal to nothing', () => { + const raw = Buffer.from([0x46, 0x72, 0x6f, 0x6d, 0x3a, 0x20, 0xe9, 0x00, 0xff]); + expect(openMail(sealMail(raw))?.equals(raw)).toBe(true); + expect(openMail(null)).toBeNull(); + expect(openMail('enc:v1:broken')).toBeNull(); + }); +}); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-notify.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-notify.test.ts new file mode 100644 index 000000000..0c0a2eb9f --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-notify.test.ts @@ -0,0 +1,86 @@ +/** + * What the mail import tells people: the forwarder gets keep or undo, the + * other members of the trip a notice, a waiting mail one note; a failed + * notification never fails the write. The notification's undo button reaches + * the inbox's undo with the person who tapped it. + */ +import { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service'; +import { MailIngestNotificationActions } from '../../../../src/nest/mail-ingest/mail-ingest-notification-actions'; +import { MailIngestNotifyService } from '../../../../src/nest/mail-ingest/mail-ingest-notify.service'; +import { NotificationsService } from '../../../../src/nest/notifications/notifications.service'; +import { Test } from '@nestjs/testing'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', () => ({ + LOG_LEVEL: 'error', + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +const notifications = { send: vi.fn(), registerResponseAction: vi.fn() }; +const actions = { undo: vi.fn(async () => ({})) }; +let notify: MailIngestNotifyService; +let buttons: MailIngestNotificationActions; + +beforeEach(async () => { + vi.clearAllMocks(); + notifications.send.mockResolvedValue({ attempted: 1, delivered: 1 }); + const moduleRef = await Test.createTestingModule({ + providers: [ + MailIngestNotifyService, + MailIngestNotificationActions, + { provide: NotificationsService, useValue: notifications }, + { provide: MailIngestActionsService, useValue: actions }, + ], + }).compile(); + notify = moduleRef.get(MailIngestNotifyService); + buttons = moduleRef.get(MailIngestNotificationActions); +}); + +const MIA = { id: 3, username: 'mia', email: 'mia@example.com', role: 'user' }; +const BOOKING = { candidateId: 11, tripId: 7, tripTitle: 'Japan', title: 'LH 716', type: 'flight' }; + +describe('MailIngestNotifyService', () => { + it('asks the forwarder to keep or undo, and tells the trip’s other members', async () => { + await notify.bookingAdded(MIA, BOOKING); + const [forwarder, trip] = notifications.send.mock.calls.map(([payload]) => payload); + expect(forwarder).toMatchObject({ + event: 'mail_booking_added', + scope: 'user', + targetId: 3, + params: { booking: 'LH 716', trip: 'Japan', tripId: '7', role: 'forwarder' }, + inApp: { + type: 'boolean', + negativeTextKey: 'notif.action.undo', + negativeCallback: { action: 'mail_ingest_undo', payload: { candidateId: 11 } }, + }, + }); + expect(trip).toMatchObject({ scope: 'trip', targetId: 7, actorId: 3, params: { role: 'traveller', actor: 'mia' } }); + }); + + it('sends one note for waiting bookings, and survives a failed send', async () => { + notifications.send.mockRejectedValue(new Error('smtp down')); + await expect(notify.bookingsPending(3, 2)).resolves.toBeUndefined(); + expect(notifications.send).toHaveBeenCalledWith( + expect.objectContaining({ event: 'mail_booking_pending', params: { count: '2' } }), + ); + }); +}); + +describe('MailIngestNotificationActions', () => { + it('registers undo and keep, and undoes the named booking for the person who tapped', async () => { + buttons.onModuleInit(); + expect(notifications.registerResponseAction.mock.calls.map(([action]) => action)).toEqual([ + 'mail_ingest_undo', + 'mail_ingest_keep', + ]); + await buttons.undo({ candidateId: 11 }, 3); + expect(actions.undo).toHaveBeenCalledWith(11, 3, false); + await expect(buttons.undo({}, 3)).rejects.toThrow('This notification names no booking'); + const keep = notifications.registerResponseAction.mock.calls[1][1] as () => Promise; + await expect(keep()).resolves.toBeUndefined(); + }); +}); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts new file mode 100644 index 000000000..5183fba04 --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts @@ -0,0 +1,91 @@ +/** + * Whether the mail import runs and how: off until switched on, the TREK Cloud + * relay as the only transport of a managed install, the address template only + * with its placeholder, and a person's switches all on until they change one. + */ +import { MailIngestPrefs } from '../../../../src/db/entities/MailIngestPrefs.entity'; +import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service'; +import { SettingsService } from '../../../../src/nest/settings/settings.service'; +import { getRepositoryToken } from '@mikro-orm/nestjs'; +import { Test } from '@nestjs/testing'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const stored = new Map(); +const settings = { getAppSetting: vi.fn(async (key: string) => stored.get(key) ?? null) }; +const env = { isManaged: vi.fn(() => false) }; +const prefs = { findForUser: vi.fn(), upsertForUser: vi.fn(async () => undefined) }; +let service: MailIngestSettingsService; + +beforeEach(async () => { + vi.clearAllMocks(); + stored.clear(); + env.isManaged.mockReturnValue(false); + prefs.findForUser.mockResolvedValue(null); + const moduleRef = await Test.createTestingModule({ + providers: [ + MailIngestSettingsService, + { provide: SettingsService, useValue: settings }, + { provide: RuntimeEnvService, useValue: env }, + { provide: getRepositoryToken(MailIngestPrefs), useValue: prefs }, + ], + }).compile(); + service = moduleRef.get(MailIngestSettingsService); +}); + +describe('MailIngestSettingsService', () => { + it('is off until switched on', async () => { + expect(await service.isAvailable()).toBe(false); + stored.set('mail_ingest_enabled', 'true'); + expect(await service.isAvailable()).toBe(true); + }); + + it('reads the transport, off for anything it does not know, the relay on a managed install', async () => { + expect(await service.transport()).toBe('off'); + stored.set('mail_ingest_transport', 'imap'); + expect(await service.transport()).toBe('imap'); + stored.set('mail_ingest_transport', 'carrier-pigeon'); + expect(await service.transport()).toBe('off'); + env.isManaged.mockReturnValue(true); + expect(await service.transport()).toBe('cloud'); + }); + + it('takes an address template only with its placeholder', async () => { + expect(await service.addressTemplate()).toBeNull(); + stored.set('mail_ingest_address_template', 'trips@example.com'); + expect(await service.addressTemplate()).toBeNull(); + stored.set('mail_ingest_address_template', ' trek+{token}@example.com '); + expect(await service.addressTemplate()).toBe('trek+{token}@example.com'); + }); + + it('reads the daily AI limit, ten when unset or broken', async () => { + expect(await service.aiDailyCap()).toBe(10); + stored.set('mail_ingest_ai_per_user_daily', '3'); + expect(await service.aiDailyCap()).toBe(3); + stored.set('mail_ingest_ai_per_user_daily', '-1'); + expect(await service.aiDailyCap()).toBe(10); + }); + + it('takes larger mails on the TREK Cloud, and the admin’s file types', async () => { + expect(service.maxMailBytes()).toBe(10 * 1024 * 1024); + env.isManaged.mockReturnValue(true); + expect(service.maxMailBytes()).toBe(25 * 1024 * 1024); + expect(await service.allowedFileTypes()).toContain('pdf'); + stored.set('allowed_file_types', 'pdf'); + expect(await service.allowedFileTypes()).toBe('pdf'); + }); + + it('has every switch on for a person who never changed one, and writes only what changes', async () => { + expect(await service.prefsOf(4)).toEqual({ enabled: true, ai_enabled: true, auto_add: true }); + prefs.findForUser.mockResolvedValue({ id: 1, user_id: 4, enabled: 1, ai_enabled: 0, auto_add: 1 }); + expect(await service.updatePrefs(4, { ai_enabled: false })).toEqual({ + enabled: true, + ai_enabled: false, + auto_add: true, + }); + expect(prefs.upsertForUser).toHaveBeenCalledWith(4, { ai_enabled: 0 }); + await service.updatePrefs(4, { enabled: true, auto_add: false }); + expect(prefs.upsertForUser).toHaveBeenLastCalledWith(4, { enabled: 1, auto_add: 0 }); + }); +}); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest.addresses.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest.addresses.test.ts new file mode 100644 index 000000000..0ce5649e9 --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest.addresses.test.ts @@ -0,0 +1,128 @@ +/** + * The forwarding addresses inside MailIngestModule over a snapshot test db: a + * person's own address and a trip's, renewing that retires the old token at + * once, and a trip address that stops working when its owner can no longer + * add bookings to the trip. + */ +import { MailIngestAddresses } from '../../../../src/db/entities/MailIngestAddresses.entity'; +import { TripMembers } from '../../../../src/db/entities/TripMembers.entity'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { MailIngestAddressService } from '../../../../src/nest/mail-ingest/mail-ingest-address.service'; +import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service'; +import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service'; +import { MailIngestModule } from '../../../../src/nest/mail-ingest/mail-ingest.module'; +import { PermissionsService } from '../../../../src/nest/permissions/permissions.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { deleteRows, findRow } from '../../../helpers/factories/rows'; +import { setAppSetting } from '../../../helpers/factories/settings'; +import { addTripMember, makeTrip } from '../../../helpers/factories/trips'; +import { makeUser } from '../../../helpers/factories/users'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { MikroORM } from '@mikro-orm/core'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const db = createSnapshotTestDb(); +let t: TestModule; +const orm = () => t.get(MikroORM); +const addresses = () => t.get(MailIngestAddressService); +const TEMPLATE = 'trek+{token}@mail.example.com'; +const permissions = { checkPermission: vi.fn(async () => true) }; + +beforeAll(async () => { + t = await createTestModule({ + db, + imports: [MailIngestModule], + overrides: [{ provide: PermissionsService, useValue: permissions }], + }); +}); +beforeEach(async () => { + resetTestDb(db); + vi.clearAllMocks(); + permissions.checkPermission.mockResolvedValue(true); + await setAppSetting(orm(), 'mail_ingest_enabled', 'true'); + await setAppSetting(orm(), 'mail_ingest_address_template', TEMPLATE); +}); +afterAll(async () => { + await t.close(); + db.close(); +}); + +const tokenIn = (address: string) => /^trek\+([a-z2-7]+)@/.exec(address)?.[1] ?? ''; +function mailTo(address: string, id: string): Buffer { + const lines = ['From: mia@example.com', `To: ${address}`, 'Subject: Booking', `Message-ID: <${id}@x>`]; + return Buffer.from(`${lines.join('\r\n')}\r\n\r\nBooking ABC123`); +} + +describe('a person’s own address', () => { + it('exists once asked for, and a renewal retires the old token', async () => { + const { user } = await makeUser(orm()); + expect(await addresses().ownAddress(user.id)).toBeNull(); + + const first = await addresses().renewOwn(user.id); + expect(first?.address).toMatch(/^trek\+[a-z2-7]{24}@mail\.example\.com$/); + expect(first?.trip_id).toBeNull(); + expect((await addresses().ownAddress(user.id))?.address).toBe(first?.address); + + const second = await addresses().renewOwn(user.id); + expect(second?.address).not.toBe(first?.address); + const intake = t.get(MailIngestIntakeService); + expect(await intake.receive(mailTo(first?.address ?? '', 'old'), 'webhook')).toEqual({ + status: 'refused', + reason: 'unknown_address', + }); + expect(await intake.receive(mailTo(second?.address ?? '', 'new'), 'webhook')).toMatchObject({ status: 'accepted' }); + const row = await findRow(orm(), MailIngestAddresses, { id: second?.id }); + expect(row?.last_used_at).not.toBeNull(); + }); + + it('cannot be made without an address template, and is not shown without one', async () => { + const { user } = await makeUser(orm()); + await addresses().ensureOwn(user.id); + await setAppSetting(orm(), 'mail_ingest_address_template', ''); + await expect(addresses().renewOwn(user.id)).rejects.toThrow(DomainError); + expect(await addresses().ownAddress(user.id)).toBeNull(); + }); + + it('takes no mail while its person has the import off', async () => { + const { user } = await makeUser(orm()); + const own = await addresses().renewOwn(user.id); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { enabled: false }); + expect(await addresses().resolve([own?.address ?? ''])).toBeNull(); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { enabled: true }); + expect((await addresses().resolve([`Mia <${own?.address}>`]))?.person.id).toBe(user.id); + expect(tokenIn(own?.address ?? '')).toHaveLength(24); + }); +}); + +describe('a trip address', () => { + it('belongs to the member who made it and stops working when they leave the trip', async () => { + const { user: owner } = await makeUser(orm()); + const { user: member } = await makeUser(orm()); + const trip = await makeTrip(orm(), owner.id); + await addTripMember(orm(), trip.id, member.id); + + const made = await addresses().renewTripAddress(member.id, trip.id); + expect(made?.trip_id).toBe(trip.id); + expect((await addresses().tripAddress(member.id, trip.id))?.address).toBe(made?.address); + expect(await addresses().tripAddress(owner.id, trip.id)).toBeNull(); + expect((await addresses().resolve([made?.address ?? '']))?.address.trip_id).toBe(trip.id); + + await deleteRows(orm(), TripMembers, { trip: trip.id, user: member.id }); + expect(await addresses().resolve([made?.address ?? ''])).toBeNull(); + await expect(addresses().tripAddress(member.id, trip.id)).rejects.toThrow('Trip not found'); + }); + + it('is refused to a member who may not add bookings, and revoked on request', async () => { + const { user: owner } = await makeUser(orm()); + const trip = await makeTrip(orm(), owner.id); + const made = await addresses().renewTripAddress(owner.id, trip.id); + await addresses().revokeTripAddress(owner.id, trip.id); + expect(await addresses().tripAddress(owner.id, trip.id)).toBeNull(); + expect(await addresses().resolve([made?.address ?? ''])).toBeNull(); + + permissions.checkPermission.mockResolvedValue(false); + await expect(addresses().renewTripAddress(owner.id, trip.id)).rejects.toThrow('Trip not found'); + }); +}); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts new file mode 100644 index 000000000..6a37c53d9 --- /dev/null +++ b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts @@ -0,0 +1,418 @@ +/** + * The mail import end to end inside MailIngestModule, over a snapshot test db + * with its real repositories and the real reservation, place and cost writes. + * Only what reaches outside is replaced: the AI reading (the walker stays + * real), kitinerary, file storage, geocoding and the notifications. + */ +import { BudgetItems } from '../../../../src/db/entities/BudgetItems.entity'; +import { MailIngestCandidates } from '../../../../src/db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../../../src/db/entities/MailIngestMessages.entity'; +import { Reservations } from '../../../../src/db/entities/Reservations.entity'; +import { importExternalId } from '../../../../src/nest/booking-import/booking-origin'; +import { BookingImportService } from '../../../../src/nest/booking-import/booking-import.service'; +import { DomainError } from '../../../../src/nest/common/domain-error'; +import { FilesService } from '../../../../src/nest/files/files.service'; +import { MailReadService } from '../../../../src/nest/llm-parse/mail-read.service'; +import type { MailReading as AiReading } from '../../../../src/nest/llm-parse/mail-read.types'; +import { walkMail } from '../../../../src/nest/llm-parse/mime-walker'; +import { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service'; +import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service'; +import { MailIngestProcessJob } from '../../../../src/nest/mail-ingest/mail-ingest-process.job'; +import { MailIngestRetentionJob } from '../../../../src/nest/mail-ingest/mail-ingest-retention.job'; +import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service'; +import { MailIngestModule } from '../../../../src/nest/mail-ingest/mail-ingest.module'; +import { MapsService } from '../../../../src/nest/maps/maps.service'; +import { NotificationsService } from '../../../../src/nest/notifications/notifications.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { countRows, findRow, findRows, updateRows } from '../../../helpers/factories/rows'; +import { setAddonEnabled, setAppSetting } from '../../../helpers/factories/settings'; +import { addTripMember, makeTrip } from '../../../helpers/factories/trips'; +import { makeUser } from '../../../helpers/factories/users'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { MikroORM } from '@mikro-orm/core'; +import type { BookingImportPreviewItem } from '@trek/shared'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../../../src/nest/audit/audit-log.logger', () => ({ + LOG_LEVEL: 'error', + logInfo: vi.fn(), + logDebug: vi.fn(), + logError: vi.fn(), + logWarn: vi.fn(), +})); + +const NOW = new Date('2030-01-15T10:00:00Z'); +const endpoint = (role: 'from' | 'to', name: string, code: string, lat: number, lng: number) => ({ + role, + sequence: role === 'from' ? 0 : 1, + name, + code, + lat, + lng, + timezone: null, + local_time: null, + local_date: null, +}); +const FLIGHT: BookingImportPreviewItem = { + type: 'flight', + title: 'LH 716 Tokyo to Osaka', + reservation_time: '2030-03-02T10:00', + reservation_end_time: '2030-03-02T11:15', + confirmation_number: 'ABC123', + metadata: { flight_number: 'LH716', price: '120,50', priceCurrency: 'EUR' }, + endpoints: [ + endpoint('from', 'Tokyo Haneda', 'HND', 35.55, 139.78), + endpoint('to', 'Osaka Itami', 'ITM', 34.78, 135.44), + ], + source: { fileName: 'mail.eml', index: 0 }, +}; +const BODY = 'Booking reference ABC123. Flight LH 716 from Tokyo to Osaka on 2030-03-02.'; + +const machine = vi.fn(async (): Promise => [FLIGHT]); +const bookingImport = { + readMailBookings: machine, + mapAiReservations: vi.fn(async (): Promise => [FLIGHT]), + externalKey: (item: BookingImportPreviewItem) => importExternalId(item), +}; +const aiRead = vi.fn(); +const reader = { walk: (raw: Buffer) => walkMail(raw), isAvailable: vi.fn(async () => false), read: aiRead }; +const files = { + createFileFromBytes: vi.fn(async () => ({ id: 77 })), + softDeleteFile: vi.fn(async () => undefined), +}; +const notifications = { send: vi.fn(async () => ({ attempted: 1, delivered: 1 })), registerResponseAction: vi.fn() }; + +const db = createSnapshotTestDb(); +let t: TestModule; +const orm = () => t.get(MikroORM); + +beforeAll(async () => { + t = await createTestModule({ + db, + imports: [MailIngestModule], + overrides: [ + { provide: BookingImportService, useValue: bookingImport }, + { provide: MailReadService, useValue: reader }, + { provide: FilesService, useValue: files }, + { provide: MapsService, useValue: { geocodeQuery: vi.fn(async () => null) } }, + { provide: NotificationsService, useValue: notifications }, + ], + }); +}); +beforeEach(async () => { + resetTestDb(db); + vi.clearAllMocks(); + machine.mockResolvedValue([FLIGHT]); + reader.isAvailable.mockResolvedValue(false); + await setAppSetting(orm(), 'mail_ingest_enabled', 'true'); + await setAddonEnabled(orm(), 'budget', true); + await setAddonEnabled(orm(), 'documents', true); +}); +afterAll(async () => { + await t.close(); + db.close(); +}); + +const intake = () => t.get(MailIngestIntakeService); +const job = () => t.get(MailIngestProcessJob); +const actions = () => t.get(MailIngestActionsService); + +function eml(id: string, subject = 'Fwd: Your flight LH 716', body = BODY): Buffer { + return Buffer.from( + [ + 'From: Mia ', + 'To: trips@example.com', + `Subject: ${subject}`, + `Message-ID: <${id}@example.com>`, + 'Date: Mon, 1 Jan 2030 10:00:00 +0000', + 'Content-Type: text/plain; charset=utf-8', + '', + body, + ].join('\r\n'), + ); +} + +async function person() { + const { user } = await makeUser(orm(), { email: 'mia@example.com', username: 'mia' }); + return user; +} + +/** Uploads a mail for the person and runs the pipeline over it; answers the mail's id. */ +async function forward(userId: number, raw: Buffer): Promise { + const taken = await intake().upload(raw, userId); + if (taken.status !== 'accepted') throw new Error(`the mail was not taken in: ${JSON.stringify(taken)}`); + expect(await job().tick(NOW)).toBe(1); + return taken.messageId; +} + +const entriesOf = (messageId: number) => + findRows(orm(), MailIngestCandidates, { message: messageId }, { position: 'asc' }); + +describe('taking a mail in', () => { + it('refuses everything while the import is off', async () => { + const user = await person(); + await setAppSetting(orm(), 'mail_ingest_enabled', 'false'); + expect(await intake().upload(eml('a'), user.id)).toEqual({ status: 'refused', reason: 'unavailable' }); + expect(await job().tick(NOW)).toBe(0); + }); + + it('stores a mail once, and refuses what is no mail', async () => { + const user = await person(); + const first = await intake().upload(eml('a'), user.id); + expect(first.status).toBe('accepted'); + expect(await intake().upload(eml('a'), user.id)).toMatchObject({ status: 'duplicate' }); + expect(await intake().upload(Buffer.from('nothing to see'), user.id)).toEqual({ + status: 'refused', + reason: 'not_a_mail', + }); + const stored = await findRow(orm(), MailIngestMessages, { user: user.id }); + expect(stored?.raw_eml_enc).toMatch(/^enc:v1:/); + }); + + it('refuses a mail to an address nobody holds', async () => { + expect(await intake().receive(eml('b'), 'webhook', ['trek+aaaaaaaaaaaaaaaaaaaaaaaa@example.com'])).toEqual({ + status: 'refused', + reason: 'unknown_address', + }); + }); +}); + +describe('a booking that passes every check', () => { + it('lands in its trip with its cost and documents, and tells the forwarder and the trip', async () => { + const user = await person(); + const { user: friend } = await makeUser(orm()); + const trip = await makeTrip(orm(), user.id, { title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }); + await addTripMember(orm(), trip.id, friend.id); + + const messageId = await forward(user.id, eml('flight')); + + const [entry] = await entriesOf(messageId); + expect(entry).toMatchObject({ + resolution: 'auto_applied', + reason: null, + resolved_trip_id: trip.id, + source: 'kitinerary', + }); + const booking = await findRow(orm(), Reservations, { trip: trip.id }); + expect(booking).toMatchObject({ + title: 'LH 716 Tokyo to Osaka', + external_source: 'mail', + external_id: importExternalId(FLIGHT), + sync_enabled: 0, + needs_review: 0, + }); + expect(await findRow(orm(), BudgetItems, { trip: trip.id })).toMatchObject({ total_price: 120.5 }); + expect(files.createFileFromBytes).toHaveBeenCalledWith( + trip.id, + expect.objectContaining({ originalname: 'mail.txt' }), + user.id, + expect.objectContaining({ reservation_id: booking?.id }), + ); + expect(JSON.parse(String((await entriesOf(messageId))[0].created_ids_json)).file_ids).toEqual([77]); + expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({ status: 'processed' }); + const events = notifications.send.mock.calls.map(([payload]) => [payload.event, payload.scope]); + expect(events).toEqual([ + ['mail_booking_added', 'user'], + ['mail_booking_added', 'trip'], + ]); + expect(t.realtime.broadcastMock).toHaveBeenCalledWith(trip.id, 'reservation:created', expect.anything()); + }); + + it('is undone by its forwarder: the booking and its cost go, the same mail again waits in the inbox', async () => { + const user = await person(); + const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + + await actions().undo(entry.id, user.id, false, NOW); + expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(0); + expect(await countRows(orm(), BudgetItems, { trip: trip.id })).toBe(0); + expect(files.softDeleteFile).toHaveBeenCalledWith(77); + await expect(actions().undo(entry.id, user.id, false, NOW)).rejects.toThrow(DomainError); + + const [again] = await entriesOf(await forward(user.id, eml('flight-again'))); + expect(again).toMatchObject({ resolution: 'pending', reason: 'possible_duplicate' }); + }); + + it('asks before undoing a booking that was edited since', async () => { + const user = await person(); + const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + await updateRows(orm(), Reservations, { trip: trip.id }, { notes: 'Window seat please' }); + + const refused = await actions() + .undo(entry.id, user.id, false, NOW) + .catch((err: unknown) => err); + expect(refused).toBeInstanceOf(DomainError); + expect((refused as DomainError).getResponse()).toMatchObject({ code: 'needs_confirmation', affects: ['edited'] }); + await actions().undo(entry.id, user.id, true, NOW); + expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(0); + }); + + it('is not written twice when the trip holds it already', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + await forward(user.id, eml('flight')); + const [second] = await entriesOf(await forward(user.id, eml('flight-forwarded-again'))); + expect(second).toMatchObject({ resolution: 'duplicate' }); + }); +}); + +describe('a booking that waits in the inbox', () => { + it('names why: no trip in its dates, and the person hears of it once', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-06-01', end_date: '2030-06-10' }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + expect(entry).toMatchObject({ resolution: 'pending', reason: 'no_trip_in_range' }); + expect(notifications.send).toHaveBeenCalledWith(expect.objectContaining({ event: 'mail_booking_pending' })); + }); + + it('asks first when the person said so', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + expect(entry).toMatchObject({ resolution: 'pending', reason: 'ask_first', suggested_trip_id: expect.any(Number) }); + }); + + it('keeps a mail without a booking for its ways out, with why the AI did not read it', async () => { + const user = await person(); + machine.mockResolvedValue([]); + const [entry] = await entriesOf(await forward(user.id, eml('nothing'))); + expect(entry).toMatchObject({ resolution: 'pending', reason: 'ai_disabled', item_json: null }); + + const prefill = await actions().prefill(entry.id, user.id); + expect(prefill.item).toMatchObject({ + type: 'flight', + confirmation_number: 'ABC123', + reservation_time: '2030-03-02', + }); + + const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const filed = await actions().fileToTrip(entry.id, user.id, trip.id, NOW); + expect(filed).toMatchObject({ resolution: 'filed', resolved_trip_id: trip.id }); + }); + + it('is added by hand to the trip the person picks, and can be discarded instead', async () => { + const user = await person(); + const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + const applied = await actions().applyToTrip(entry.id, user.id, trip.id, NOW); + expect(applied).toMatchObject({ resolution: 'applied', resolved_trip_id: trip.id }); + expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(1); + await expect(actions().discard(entry.id, user.id, NOW)).rejects.toThrow('This entry was decided already'); + }); +}); + +describe('mails that are no booking', () => { + it('drops a newsletter without reading it', async () => { + const user = await person(); + const raw = Buffer.from( + [ + 'From: news@shop.example', + 'Subject: Our autumn deals', + 'List-Unsubscribe: ', + 'Message-ID: ', + '', + 'Fresh deals for everyone who loves to shop with us.', + ].join('\r\n'), + ); + const messageId = await forward(user.id, raw); + expect(machine).not.toHaveBeenCalled(); + expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({ + status: 'discarded', + reason: 'newsletter', + }); + }); + + it('shows a forwarding confirmation’s code in the inbox', async () => { + const user = await person(); + const messageId = await forward( + user.id, + eml('confirm', 'Gmail Forwarding Confirmation', 'Confirmation code: 612345678 for forwarding.'), + ); + expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({ + forwarding_code: '612345678', + reason: 'forwarding_confirmation', + }); + expect((await entriesOf(messageId))[0]).toMatchObject({ reason: 'forwarding_confirmation' }); + }); + + it('discards a mail the AI read and found no booking in', async () => { + const user = await person(); + reader.isAvailable.mockResolvedValue(true); + machine.mockResolvedValue([]); + aiRead.mockResolvedValue({ outcome: 'empty', bookings: [], documentText: BODY, imageOnly: false } as AiReading); + const messageId = await forward(user.id, eml('chat')); + expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({ + status: 'discarded', + reason: 'not_a_booking', + ai_used: 1, + }); + }); +}); + +describe('the AI’s trip pick', () => { + it('decides between two trips that fit only where the booking lies near the pick', async () => { + const user = await person(); + reader.isAvailable.mockResolvedValue(true); + const japan = await makeTrip(orm(), user.id, { title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }); + await makeTrip(orm(), user.id, { title: 'Work', start_date: '2030-02-25', end_date: '2030-03-05' }); + aiRead.mockImplementation(async (input: { trips: Array<{ ref: string; title: string }> }) => ({ + outcome: 'ok', + documentText: BODY, + imageOnly: false, + bookings: [ + { + kind: 'new_booking', + reservations: [{ '@type': 'FlightReservation' }], + tripRef: input.trips.find((trip) => trip.title === 'Japan')?.ref ?? null, + tripConfidence: 0.9, + uncertainFields: [], + suggestedNewTrip: null, + }, + ], + })); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + // Japan has no place near Osaka the server could confirm: the inbox suggests it. + expect(entry).toMatchObject({ resolution: 'pending', reason: 'trip_ambiguous', suggested_trip_id: japan.id }); + expect(entry.source).toBe('ai'); + }); +}); + +describe('a mail that keeps failing', () => { + it('is tried again later and given up after its last try', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + machine.mockRejectedValue(new Error('kitinerary crashed')); + const taken = await intake().upload(eml('poison'), user.id); + if (taken.status !== 'accepted') throw new Error('not taken in'); + expect(await job().tick(NOW)).toBe(1); + expect(await findRow(orm(), MailIngestMessages, { id: taken.messageId })).toMatchObject({ status: 'queued' }); + expect(await job().tick(new Date(NOW.getTime() + 60 * 60 * 1000))).toBe(1); + expect(await job().tick(new Date(NOW.getTime() + 2 * 60 * 60 * 1000))).toBe(1); + expect(await findRow(orm(), MailIngestMessages, { id: taken.messageId })).toMatchObject({ + status: 'failed', + reason: 'processing_failed', + }); + }); +}); + +describe('the retention', () => { + it('purges a settled mail and keeps one an entry waits on', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const added = await forward(user.id, eml('flight')); + machine.mockResolvedValue([]); + const waiting = await forward(user.id, eml('nothing')); + + // Settled long ago: the mail with nothing waiting on it goes, the one an entry waits on stays. + const longAgo = { processed_at: '2000-01-01 00:00:00' }; + await updateRows(orm(), MailIngestMessages, { id: { $in: [added, waiting] } }, longAgo); + await t.get(MailIngestRetentionJob).tick(new Date()); + expect(await findRow(orm(), MailIngestMessages, { id: added })).toMatchObject({ raw_eml_enc: null }); + expect((await findRow(orm(), MailIngestMessages, { id: waiting }))?.raw_eml_enc).toMatch(/^enc:v1:/); + }); +}); diff --git a/server/vitest.config.ts b/server/vitest.config.ts index 45325825c..78a43107a 100644 --- a/server/vitest.config.ts +++ b/server/vitest.config.ts @@ -136,6 +136,9 @@ export default defineConfig({ // like ferratas until the first full run regenerates it. 'src/nest/journey-recordings/**/*.ts': { statements: 80, branches: 80, functions: 80, lines: 80 }, 'src/nest/llm-parse/**/*.ts': { statements: 91, branches: 85, functions: 85, lines: 94 }, + // New domain in this change (the booking mail import), unmeasured: pinned at the 80 floor + // until the first full run regenerates it. + 'src/nest/mail-ingest/**/*.ts': { statements: 80, branches: 80, functions: 80, lines: 80 }, 'src/nest/managed/**/*.ts': { statements: 100, branches: 100, functions: 100, lines: 100 }, 'src/nest/maps/**/*.ts': { statements: 93, branches: 86, functions: 97, lines: 96 }, 'src/nest/mcp-shared/**/*.ts': { statements: 84, branches: 59, functions: 99, lines: 88 }, diff --git a/shared/src/mail-ingest/mail-ingest-reason.schema.ts b/shared/src/mail-ingest/mail-ingest-reason.schema.ts index 39cf129f4..39c5c3de5 100644 --- a/shared/src/mail-ingest/mail-ingest-reason.schema.ts +++ b/shared/src/mail-ingest/mail-ingest-reason.schema.ts @@ -113,7 +113,10 @@ export const MAIL_INGEST_REASON_BY_LLM_OUTCOME: Readonly;