mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-14 00:01:03 +00:00
feat(mail-ingest): read forwarded booking mails and add what passes the checks to its trip
Addresses per person and per trip (random base32 token, stored hashed and encrypted), an intake every transport shares, and a job that reads each mail once under a lease: filters without AI, machine-readable data always, one AI request with the person's trips, the server's checks G1 to G9, then one transaction per mail that adds what passed, keeps the rest in the inbox with a reason, and notifies the forwarder and co-travellers. Undo within 7 days, manual apply, filing as documents, a prefilled form and a nightly retention complete it; on a managed install the operator's switch decides.
This commit is contained in:
Maurice
@@ -6,8 +6,15 @@ import {
|
||||
assistantChangePreviewWriteSchema,
|
||||
assistantMessagePartWriteSchema,
|
||||
assistantStoredPartsSchema,
|
||||
bookingImportPreviewItemSchema,
|
||||
ferrataSchema,
|
||||
hikingPreferencesSchema,
|
||||
mailIngestAttachmentSchema,
|
||||
mailIngestCheckResultSchema,
|
||||
mailIngestCheckSchema,
|
||||
mailIngestFieldSchema,
|
||||
mailIngestHintSchema,
|
||||
mailIngestSuggestedTripSchema,
|
||||
obstacleSnapshotSchema,
|
||||
paddleGeometrySchema,
|
||||
paddleLockSchema,
|
||||
@@ -26,9 +33,15 @@ import {
|
||||
type DawarichTrackSegment,
|
||||
type AssistantChangePreview,
|
||||
type AssistantMessagePart,
|
||||
type BookingImportPreviewItem,
|
||||
type Ferrata,
|
||||
type GeometryPoint,
|
||||
type HikingPreferences,
|
||||
type MailIngestAttachment,
|
||||
type MailIngestCheck,
|
||||
type MailIngestCheckResult,
|
||||
type MailIngestField,
|
||||
type MailIngestHint,
|
||||
type ObstacleSnapshot,
|
||||
type PaddleGeometry,
|
||||
type PaddleLock,
|
||||
@@ -531,3 +544,73 @@ export const PLACE_DETAILS_CACHE_PAYLOAD: JsonColumn<Record<string, unknown> | n
|
||||
schema: z.record(z.string(), z.unknown()).nullable(),
|
||||
fallback: () => null,
|
||||
};
|
||||
|
||||
// ── The booking mail import (mail-ingest) ──────────────────────────────────
|
||||
|
||||
/** The booking read from a mail, as the inbox and the form show it; null for an entry that stands for the mail. */
|
||||
export const MAIL_INGEST_ITEM: JsonColumn<BookingImportPreviewItem | null> = {
|
||||
column: 'mail_ingest_candidates.item_json',
|
||||
schema: bookingImportPreviewItemSchema.nullable(),
|
||||
fallback: () => null,
|
||||
};
|
||||
|
||||
/** The server's checks on the booking, by name. A broken row reads as none run. */
|
||||
export const MAIL_INGEST_CHECKS: JsonColumn<Partial<Record<MailIngestCheck, MailIngestCheckResult>>> = {
|
||||
column: 'mail_ingest_candidates.checks_json',
|
||||
schema: z.partialRecord(mailIngestCheckSchema, mailIngestCheckResultSchema),
|
||||
fallback: () => ({}),
|
||||
};
|
||||
|
||||
/** The fields of the booking marked for a second look. */
|
||||
export const MAIL_INGEST_UNCERTAIN_FIELDS: JsonColumn<MailIngestField[]> = {
|
||||
column: 'mail_ingest_candidates.uncertain_fields',
|
||||
schema: z.array(mailIngestFieldSchema),
|
||||
fallback: () => [],
|
||||
};
|
||||
|
||||
const mailIngestMatchSchema = z.object({
|
||||
/** A change or cancellation of an earlier booking rather than a new one. */
|
||||
kind: z.enum(['new_booking', 'change', 'cancellation']),
|
||||
/** The trip the AI proposed to create when none fits; never created on its own. */
|
||||
suggested_new_trip: mailIngestSuggestedTripSchema.nullable(),
|
||||
});
|
||||
export type MailIngestMatch = z.infer<typeof mailIngestMatchSchema>;
|
||||
|
||||
/** How the booking was matched to a trip, past the suggested trip's own column. */
|
||||
export const MAIL_INGEST_MATCH: JsonColumn<MailIngestMatch> = {
|
||||
column: 'mail_ingest_candidates.match_json',
|
||||
schema: mailIngestMatchSchema,
|
||||
fallback: () => ({ kind: 'new_booking', suggested_new_trip: null }),
|
||||
};
|
||||
|
||||
const idList = z.array(z.number().int().positive());
|
||||
const mailIngestCreatedSchema = z.object({
|
||||
reservation_ids: idList,
|
||||
place_ids: idList,
|
||||
budget_item_ids: idList,
|
||||
file_ids: idList,
|
||||
/** The booking as it was written, to tell an edit since then. */
|
||||
fingerprint: z.string().nullable(),
|
||||
});
|
||||
export type MailIngestCreated = z.infer<typeof mailIngestCreatedSchema>;
|
||||
|
||||
/** What adding the booking created, for its undo. A broken row undoes nothing. */
|
||||
export const MAIL_INGEST_CREATED: JsonColumn<MailIngestCreated> = {
|
||||
column: 'mail_ingest_candidates.created_ids_json',
|
||||
schema: mailIngestCreatedSchema,
|
||||
fallback: () => ({ reservation_ids: [], place_ids: [], budget_item_ids: [], file_ids: [], fingerprint: null }),
|
||||
};
|
||||
|
||||
/** What the inbox notes about a mail without holding it back. */
|
||||
export const MAIL_INGEST_HINTS: JsonColumn<MailIngestHint[]> = {
|
||||
column: 'mail_ingest_messages.hints',
|
||||
schema: z.array(mailIngestHintSchema),
|
||||
fallback: () => [],
|
||||
};
|
||||
|
||||
/** The attachments a mail carried that a booking travels in, without their bytes. */
|
||||
export const MAIL_INGEST_ATTACHMENTS: JsonColumn<MailIngestAttachment[]> = {
|
||||
column: 'mail_ingest_messages.attachments',
|
||||
schema: z.array(mailIngestAttachmentSchema),
|
||||
fallback: () => [],
|
||||
};
|
||||
|
||||
@@ -49,6 +49,11 @@ export interface MailIngestCandidateInsert {
|
||||
resolution: string;
|
||||
reason: string | null;
|
||||
createdAt: DbTimestamp;
|
||||
/** For an entry decided as it is written: the trip it went to, what it created, until when it can be undone. */
|
||||
resolvedTripId?: number | null;
|
||||
createdIdsJson?: string | null;
|
||||
resolvedAt?: DbTimestamp | null;
|
||||
undoUntil?: DbTimestamp | null;
|
||||
}
|
||||
|
||||
/** What deciding an entry writes. */
|
||||
@@ -107,6 +112,8 @@ type CandidatesKyselyDB = Pick<
|
||||
| 'places'
|
||||
| 'users'
|
||||
| 'reservations'
|
||||
| 'day_assignments'
|
||||
| 'day_accommodations'
|
||||
| 'budget_item_payers'
|
||||
>;
|
||||
|
||||
@@ -144,6 +151,10 @@ export class MailIngestCandidatesRepository extends TrekRepository<MailIngestCan
|
||||
resolution: row.resolution,
|
||||
reason: row.reason,
|
||||
created_at: row.createdAt,
|
||||
resolved_trip_id: row.resolvedTripId ?? null,
|
||||
created_ids_json: row.createdIdsJson ?? null,
|
||||
resolved_at: row.resolvedAt ?? null,
|
||||
undo_until: row.undoUntil ?? null,
|
||||
})
|
||||
.returning('id')
|
||||
.executeTakeFirstOrThrow();
|
||||
@@ -265,18 +276,41 @@ export class MailIngestCandidatesRepository extends TrekRepository<MailIngestCan
|
||||
.execute();
|
||||
}
|
||||
|
||||
/** The dated days of these trips. */
|
||||
async listDayDates(tripIds: readonly number[]): Promise<Array<{ trip_id: number; date: string }>> {
|
||||
/** One trip the person owns or is a member of and that is not archived, whatever its dates; null otherwise. */
|
||||
async findOpenTrip(userId: number, tripId: number): Promise<MailIngestTripRow | null> {
|
||||
const row = await this.db()
|
||||
.selectFrom('trips as t')
|
||||
.select(['t.id', 't.title', 't.start_date', 't.end_date', 't.user_id', 't.currency'])
|
||||
.where('t.id', '=', tripId)
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb('t.user_id', '=', userId),
|
||||
eb.exists(
|
||||
eb
|
||||
.selectFrom('trip_members as m')
|
||||
.select('m.id')
|
||||
.whereRef('m.trip_id', '=', 't.id')
|
||||
.where('m.user_id', '=', userId),
|
||||
),
|
||||
]),
|
||||
)
|
||||
.where((eb) => eb.or([eb('t.is_archived', 'is', null), eb('t.is_archived', '=', 0)]))
|
||||
.executeTakeFirst();
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
/** The dated days of these trips, by date. */
|
||||
async listDatedDays(tripIds: readonly number[]): Promise<Array<{ id: number; trip_id: number; date: string }>> {
|
||||
if (tripIds.length === 0) return [];
|
||||
const rows = await this.db()
|
||||
.selectFrom('days')
|
||||
.select(['trip_id', 'date'])
|
||||
.select(['id', 'trip_id', 'date'])
|
||||
.where('trip_id', 'in', [...tripIds])
|
||||
.where('date', 'is not', null)
|
||||
.orderBy('trip_id', 'asc')
|
||||
.orderBy('date', 'asc')
|
||||
.execute();
|
||||
return rows.flatMap((row) => (row.date === null ? [] : [{ trip_id: row.trip_id, date: row.date }]));
|
||||
return rows.flatMap((row) => (row.date === null ? [] : [{ id: row.id, trip_id: row.trip_id, date: row.date }]));
|
||||
}
|
||||
|
||||
/** The places of these trips that have a position, oldest first. */
|
||||
@@ -346,6 +380,16 @@ export class MailIngestCandidatesRepository extends TrekRepository<MailIngestCan
|
||||
return row !== undefined;
|
||||
}
|
||||
|
||||
/** Whether a booking, a day plan entry or a stay still names the place. */
|
||||
async placeInUse(placeId: number): Promise<boolean> {
|
||||
const [booking, planned, stay] = await Promise.all([
|
||||
this.db().selectFrom('reservations').select('id').where('place_id', '=', placeId).executeTakeFirst(),
|
||||
this.db().selectFrom('day_assignments').select('id').where('place_id', '=', placeId).executeTakeFirst(),
|
||||
this.db().selectFrom('day_accommodations').select('id').where('place_id', '=', placeId).executeTakeFirst(),
|
||||
]);
|
||||
return booking !== undefined || planned !== undefined || stay !== undefined;
|
||||
}
|
||||
|
||||
/** How many payers the costs name: a cost someone settled with moves balances. */
|
||||
async countPayers(budgetItemIds: readonly number[]): Promise<number> {
|
||||
if (budgetItemIds.length === 0) return 0;
|
||||
|
||||
@@ -158,22 +158,21 @@ export class MailIngestMessagesRepository extends TrekRepository<MailIngestMessa
|
||||
.orderBy('id', 'asc')
|
||||
.limit(limit)
|
||||
.execute();
|
||||
const claimed: number[] = [];
|
||||
for (const { id } of due) {
|
||||
const result = await this.db()
|
||||
.updateTable('mail_ingest_messages')
|
||||
.set((eb) => ({ status: 'processing', lease_until: leaseUntil, attempts: eb('attempts', '+', 1) }))
|
||||
.where('id', '=', id)
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb('status', '=', 'queued'),
|
||||
eb.and([eb('status', '=', 'processing'), eb('lease_until', '<', now)]),
|
||||
]),
|
||||
)
|
||||
.executeTakeFirst();
|
||||
if (Number(result.numUpdatedRows) > 0) claimed.push(id);
|
||||
}
|
||||
return claimed;
|
||||
const won = await Promise.all(due.map(({ id }) => this.claim(id, now, leaseUntil)));
|
||||
return due.flatMap(({ id }, i) => (won[i] ? [id] : []));
|
||||
}
|
||||
|
||||
/** One mail's claim: it is still queued, or its last claim ran out at `now`. */
|
||||
private async claim(id: number, now: DbTimestamp, leaseUntil: DbTimestamp): Promise<boolean> {
|
||||
const result = await this.db()
|
||||
.updateTable('mail_ingest_messages')
|
||||
.set((eb) => ({ status: 'processing', lease_until: leaseUntil, attempts: eb('attempts', '+', 1) }))
|
||||
.where('id', '=', id)
|
||||
.where((eb) =>
|
||||
eb.or([eb('status', '=', 'queued'), eb.and([eb('status', '=', 'processing'), eb('lease_until', '<', now)])]),
|
||||
)
|
||||
.executeTakeFirst();
|
||||
return Number(result.numUpdatedRows) > 0;
|
||||
}
|
||||
|
||||
/** Writes what the pipeline made of the mail and lets go of its claim. */
|
||||
|
||||
@@ -58,6 +58,7 @@ import { JourneyAccessModule } from './journey/journey-access.module';
|
||||
import { JourneyModule } from './journey/journey.module';
|
||||
import { TripJournalLinkModule } from './journey/trip-journal-link.module';
|
||||
import { LlmParseModule } from './llm-parse/llm-parse.module';
|
||||
import { MailIngestModule } from './mail-ingest/mail-ingest.module';
|
||||
import { ManagedExtModule } from './managed/managed-ext.module';
|
||||
import { MapsModule } from './maps/maps.module';
|
||||
import { McpTransportModule } from './mcp-transport/mcp-transport.module';
|
||||
@@ -209,6 +210,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
|
||||
ReservationImportModule,
|
||||
ReceiptScanModule,
|
||||
LlmParseModule,
|
||||
MailIngestModule,
|
||||
AiConnectionsModule,
|
||||
AssistantModule,
|
||||
ManagedExtModule,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { readEnv } from '../../app-config';
|
||||
import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository';
|
||||
import { aiPolicySchema } from '@trek/shared';
|
||||
import { aiPolicySchema, mailIngestTransportSettingSchema } from '@trek/shared';
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
@@ -161,6 +161,15 @@ export const APP_SETTINGS = {
|
||||
synology_capture_repair_3019: { schema: text, default: null },
|
||||
/** How many starts have tried that so far, while some photo could not be confirmed. */
|
||||
synology_capture_repair_3019_tries: { schema: intText, default: null },
|
||||
// The booking mail import (mail-ingest). Off until switched on: on a self-hosted install by its
|
||||
// admin, on a managed one by the operator, for the plans that include it and never for Start.
|
||||
mail_ingest_enabled: { schema: flag, default: 'false' },
|
||||
/** How mail reaches the install: not at all (uploads only), IMAP, a signed webhook, or the TREK Cloud relay. */
|
||||
mail_ingest_transport: { schema: mailIngestTransportSettingSchema, default: 'off' },
|
||||
/** The forwarding address, `{token}` standing where a person's token goes: `trek+{token}@example.com`. */
|
||||
mail_ingest_address_template: { schema: text, default: null },
|
||||
/** How many forwarded mails a day an AI reads for one person. */
|
||||
mail_ingest_ai_per_user_daily: { schema: intText, default: '10' },
|
||||
} as const satisfies Record<string, AppSettingDef>;
|
||||
|
||||
export type AppSettingKey = keyof typeof APP_SETTINGS;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import type { KiReservation } from '../booking-import/kitinerary.types';
|
||||
import { capImage, renderPdfPages } from './image-input';
|
||||
import { createLlmClient } from './llm-client.factory';
|
||||
@@ -96,7 +97,7 @@ export class MailReadService {
|
||||
} catch (err) {
|
||||
// The provider and the error only: the mail's content stays out of the log.
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[llm-parse] Mail read failed (provider=${config.provider}): ${reason}`);
|
||||
logError(`[llm-parse] Mail read failed (provider=${config.provider}): ${reason}`);
|
||||
return nothing(failureOutcome(err));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,7 +53,10 @@ export function normalizeNode(node: Record<string, unknown>): Record<string, unk
|
||||
* is dropped: an empty currency lets the cost form pick one, a made up one
|
||||
* (`EURials`, #2477) is saved and cannot be converted.
|
||||
*/
|
||||
export function normalizeCurrency(node: Record<string, unknown>, documentText: string | undefined): Record<string, unknown> {
|
||||
export function normalizeCurrency(
|
||||
node: Record<string, unknown>,
|
||||
documentText: string | undefined,
|
||||
): Record<string, unknown> {
|
||||
const hasPrice = node.price != null && node.price !== '';
|
||||
if (node.priceCurrency == null && !hasPrice) return node;
|
||||
const code =
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import type { WalkedMail } from '../llm-parse/mail-read.types';
|
||||
|
||||
/**
|
||||
* What a mail is before anything reads it for bookings: a forwarding
|
||||
* confirmation from Gmail or Outlook (its code is shown, nothing is clicked),
|
||||
* mail that is no booking (a newsletter, an automatic reply, an empty mail),
|
||||
* or a mail to read.
|
||||
*/
|
||||
export type MailClass =
|
||||
| { kind: 'read' }
|
||||
| { kind: 'forwarding_confirmation'; code: string | null }
|
||||
| { kind: 'drop'; reason: 'newsletter' | 'auto_reply' | 'empty_mail' };
|
||||
|
||||
const FORWARDING_SENDERS = /forwarding-noreply@google\.com|@accountprotection\.microsoft\.com|@microsoft\.com/i;
|
||||
const FORWARDING_SUBJECT =
|
||||
/(forwarding|weiterleitung|transfert|reenv[ií]o|inoltro|doorsturen).*(confirm|bestätig|verif|conferma|bevestig)/i;
|
||||
const CODE_RE = /(?:code|código|codice|kód)\D{0,40}?(\d{6,10})\b/i;
|
||||
const AUTO_REPLY_SUBJECT =
|
||||
/^(automatic reply|auto(matische)? ?(reply|antwort)|out of (the )?office|abwesenheit|r[ée]ponse automatique|risposta automatica|respuesta autom[áa]tica)/i;
|
||||
/**
|
||||
* What makes a list mail worth reading anyway. An automatic forward keeps the
|
||||
* original headers, and a booking confirmation often carries List-Unsubscribe
|
||||
* just like a newsletter; an attachment or a booking word keeps it in.
|
||||
*/
|
||||
const BOOKING_WORDS =
|
||||
/\b(booking|reservation|confirmation|itinerary|e-?ticket|boarding|check-in|pnr|buchung|reservierung|bestätigung|réservation|reserva|prenotazione|boeking|rezerwacja)\b/i;
|
||||
/** Less body than this, and no attachment, is an empty mail. */
|
||||
const EMPTY_BODY_CHARS = 20;
|
||||
|
||||
export function classifyMail(mail: WalkedMail): MailClass {
|
||||
const { envelope } = mail;
|
||||
const subject = mail.subject ?? '';
|
||||
const fromProvider = FORWARDING_SENDERS.test(envelope.from ?? '') && CODE_RE.test(mail.body);
|
||||
if (FORWARDING_SUBJECT.test(subject) || fromProvider) {
|
||||
return { kind: 'forwarding_confirmation', code: CODE_RE.exec(mail.body)?.[1] ?? null };
|
||||
}
|
||||
const autoSubmitted = (envelope.autoSubmitted ?? '').trim().toLowerCase();
|
||||
const precedence = (envelope.precedence ?? '').trim().toLowerCase();
|
||||
// `auto-replied` only (RFC 3834): a booking system marks its confirmations `auto-generated`.
|
||||
if (autoSubmitted.startsWith('auto-replied') || precedence === 'auto_reply' || AUTO_REPLY_SUBJECT.test(subject)) {
|
||||
return { kind: 'drop', reason: 'auto_reply' };
|
||||
}
|
||||
if (mail.body.trim().length < EMPTY_BODY_CHARS && mail.attachments.length === 0) {
|
||||
return { kind: 'drop', reason: 'empty_mail' };
|
||||
}
|
||||
const listMail = !!envelope.listUnsubscribe || ['bulk', 'list', 'junk'].includes(precedence);
|
||||
const bookingSignal = mail.attachments.length > 0 || BOOKING_WORDS.test(`${subject}\n${mail.body}`);
|
||||
if (listMail && !bookingSignal) return { kind: 'drop', reason: 'newsletter' };
|
||||
return { kind: 'read' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the mail came from the person themselves, signed: the From address
|
||||
* is their account's and the receiving server saw a DKIM or DMARC pass for
|
||||
* its domain. Only ever a hint shown with the mail (owner decision
|
||||
* 13.11.2026), never a reason to stop.
|
||||
*/
|
||||
export function senderVerified(mail: WalkedMail, accountEmail: string): boolean {
|
||||
const from = addressOf(mail.envelope.from);
|
||||
if (!from || from !== accountEmail.trim().toLowerCase()) return false;
|
||||
const domain = from.split('@')[1] ?? '';
|
||||
const results = (mail.envelope.authenticationResults ?? '').toLowerCase();
|
||||
const dkim = new RegExp(`dkim=pass[^;]*header\\.(d|i)=@?${escapeRegExp(domain)}\\b`);
|
||||
return dkim.test(results) || /dmarc=pass/.test(results);
|
||||
}
|
||||
|
||||
/** The bare address of a From header: `Mia <mia@example.com>` is `mia@example.com`. */
|
||||
export function addressOf(header: string | null): string | null {
|
||||
const match = /<([^>]+)>/.exec(header ?? '') ?? /([^\s<>"]+@[^\s<>"]+)/.exec(header ?? '');
|
||||
return match ? match[1].trim().toLowerCase() : null;
|
||||
}
|
||||
|
||||
function escapeRegExp(text: string): string {
|
||||
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import {
|
||||
MAIL_INGEST_CREATED,
|
||||
MAIL_INGEST_ITEM,
|
||||
MAIL_INGEST_UNCERTAIN_FIELDS,
|
||||
type MailIngestCreated,
|
||||
} from '../../db/json-columns';
|
||||
import type {
|
||||
MailIngestCandidateRow,
|
||||
MailIngestCandidatesRepository,
|
||||
} from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow, parseDbTimestamp } from '../../db/types';
|
||||
import { decodeJson, encodeJson } from '../../utils/json-column';
|
||||
import { AirportsService } from '../airports/airports.service';
|
||||
import { DemoService } from '../common/demo.service';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { MailReadService } from '../llm-parse/mail-read.service';
|
||||
import type { WalkedMail } from '../llm-parse/mail-read.types';
|
||||
import { MailIngestApplyService } from './mail-ingest-apply.service';
|
||||
import { implausible, type TripOption } from './mail-ingest-checks';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { IMPORT_SOURCES, UNDO_WINDOW_DAYS } from './mail-ingest.constants';
|
||||
import { openMail } from './mail-seal';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { BookingImportPreviewItem } from '@trek/shared';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
/** The states an added booking is undone from. */
|
||||
const ADDED = ['auto_applied', 'applied'] as const;
|
||||
const ISO_DATE_RE = /\b(20\d\d-[01]\d-[0-3]\d)\b/;
|
||||
/** A booking reference as a mail writes it: after its label, letters and digits. */
|
||||
const REFERENCE_RE =
|
||||
/(?:booking (?:reference|code|number)|confirmation (?:code|number)|reservation (?:code|number)|pnr|buchungsnummer|buchungscode|référence|localizador)\W{0,5}([A-Z0-9-]{5,20})/i;
|
||||
const TYPE_WORDS: ReadonlyArray<[RegExp, string]> = [
|
||||
[/\b(flight|flug|vol|vuelo|volo|boarding)\b/i, 'flight'],
|
||||
[/\b(train|zug|bahn|treno|tren|rail)\b/i, 'train'],
|
||||
[/\b(hotel|apartment|check-in|unterkunft|hébergement|alojamiento)\b/i, 'hotel'],
|
||||
[/\b(rental car|car hire|mietwagen|location de voiture)\b/i, 'car'],
|
||||
[/\b(restaurant|table)\b/i, 'restaurant'],
|
||||
[/\b(ticket|event|concert|konzert|museum)\b/i, 'event'],
|
||||
];
|
||||
|
||||
/**
|
||||
* What a person does with an entry of their mail inbox: add it to a trip,
|
||||
* file the mail as documents, fill the booking form in themselves, discard
|
||||
* it, or undo a booking a mail added. Every action checks again that the
|
||||
* person may add bookings to the trip it touches, and that every id belongs
|
||||
* to that trip.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestActionsService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
private readonly apply: MailIngestApplyService,
|
||||
private readonly trips: MailIngestTripsService,
|
||||
private readonly reader: MailReadService,
|
||||
private readonly airports: AirportsService,
|
||||
private readonly demo: DemoService,
|
||||
private readonly uow: UnitOfWork,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Removes exactly what adding the booking created, within the undo window,
|
||||
* for the person who forwarded it or any member who may edit the trip's
|
||||
* bookings. A booking whose cost moved balances, or that was edited since,
|
||||
* is only undone with `confirm`.
|
||||
*
|
||||
* @txIndependent the removal is one transaction; the documents go to the
|
||||
* trash after it, as storage I/O.
|
||||
*/
|
||||
async undo(
|
||||
candidateId: number,
|
||||
userId: number,
|
||||
confirm: boolean,
|
||||
now: Date = new Date(),
|
||||
): Promise<MailIngestCandidateRow> {
|
||||
const entry = await this.candidates.findById(candidateId);
|
||||
const tripId = entry?.resolved_trip_id ?? null;
|
||||
if (!entry || tripId === null || !(await this.editableBy(userId, tripId))) {
|
||||
throw new DomainError(404, 'Booking not found');
|
||||
}
|
||||
if (!(ADDED as readonly string[]).includes(entry.resolution)) {
|
||||
throw new DomainError(409, 'This booking is not added from a mail any more');
|
||||
}
|
||||
const until = parseDbTimestamp(entry.undo_until);
|
||||
if (!until || until < now) throw new DomainError(409, 'The time to undo this booking has passed');
|
||||
|
||||
const created = decodeJson(MAIL_INGEST_CREATED, entry.created_ids_json);
|
||||
const affects = await this.undoAffects(tripId, created);
|
||||
if (affects.length > 0 && !confirm) {
|
||||
throw new DomainError(409, 'Undoing this booking changes more than the booking', {
|
||||
details: { code: 'needs_confirmation', affects },
|
||||
});
|
||||
}
|
||||
const removed = await this.uow.transactional(async () => {
|
||||
const claimed = await this.candidates.resolve(entry.id, ADDED, {
|
||||
resolution: 'undone',
|
||||
reason: null,
|
||||
resolvedTripId: tripId,
|
||||
createdIdsJson: entry.created_ids_json,
|
||||
resolvedBy: userId,
|
||||
resolvedAt: dbNow(now),
|
||||
undoUntil: null,
|
||||
});
|
||||
if (!claimed) throw new DomainError(409, 'This booking was undone already');
|
||||
return this.apply.removeInTx(tripId, created);
|
||||
});
|
||||
await this.apply.announceRemoved(removed, created.file_ids);
|
||||
return this.reload(entry.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a waiting booking to the trip the inbox suggested or the person
|
||||
* picked. The person decides the trip; the server still checks that they
|
||||
* may add bookings there, that the booking is plausible and that the trip
|
||||
* does not hold it already.
|
||||
*
|
||||
* @txIndependent the booking and the entry are one transaction; the mail's
|
||||
* documents are attached after it, as storage I/O, and noted on the entry.
|
||||
*/
|
||||
async applyToTrip(
|
||||
candidateId: number,
|
||||
userId: number,
|
||||
tripId: number | null,
|
||||
now: Date = new Date(),
|
||||
): Promise<MailIngestCandidateRow> {
|
||||
const entry = await this.pendingEntry(candidateId, userId);
|
||||
const item = decodeJson(MAIL_INGEST_ITEM, entry.item_json);
|
||||
if (!item) throw new DomainError(409, 'This mail holds no booking to add; file it as a document instead');
|
||||
if (await this.demo.isDemoUserId(userId)) throw new DomainError(403, 'Demo accounts cannot add bookings');
|
||||
const trip = await this.tripFor(userId, tripId ?? entry.suggested_trip_id);
|
||||
if (implausible(item, now, (code) => this.airports.findByIata(code) !== null)) {
|
||||
throw new DomainError(422, 'This booking has values that cannot be right; fill it in yourself instead');
|
||||
}
|
||||
if (await this.candidates.tripHasExternal(trip.id, IMPORT_SOURCES, this.apply.externalKey(item))) {
|
||||
throw new DomainError(409, 'This booking is already in the trip.');
|
||||
}
|
||||
const uncertain = decodeJson(MAIL_INGEST_UNCERTAIN_FIELDS, entry.uncertain_fields);
|
||||
const prepared = await this.apply.prepare(item, trip, entry.user_id, uncertain);
|
||||
const written = await this.uow.transactional(async () => {
|
||||
const done = await this.apply.writeInTx(prepared);
|
||||
const claimed = await this.candidates.resolve(entry.id, ['pending'], {
|
||||
resolution: 'applied',
|
||||
reason: null,
|
||||
resolvedTripId: trip.id,
|
||||
createdIdsJson: encodeJson(MAIL_INGEST_CREATED, done.created),
|
||||
resolvedBy: userId,
|
||||
resolvedAt: dbNow(now),
|
||||
undoUntil: dbNow(new Date(now.getTime() + UNDO_WINDOW_DAYS * DAY_MS)),
|
||||
});
|
||||
if (!claimed) throw new DomainError(409, 'This entry was decided already');
|
||||
return done;
|
||||
});
|
||||
await this.apply.announce(written);
|
||||
const mail = await this.mailOf(entry);
|
||||
if (mail) {
|
||||
const { fileIds, error } = await this.apply.attachDocuments(trip.id, written.reservation.id, mail, userId);
|
||||
const created = { ...written.created, file_ids: fileIds };
|
||||
await this.candidates.setAttachments(entry.id, encodeJson(MAIL_INGEST_CREATED, created), error);
|
||||
}
|
||||
return this.reload(entry.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* "Add as document": the mail's attachments, or its text, as files of the
|
||||
* trip, and the entry settled as filed. Without the documents addon there
|
||||
* is nothing to file into.
|
||||
*
|
||||
* @txIndependent each document is storage I/O; the entry is settled once they are in.
|
||||
*/
|
||||
async fileToTrip(
|
||||
candidateId: number,
|
||||
userId: number,
|
||||
tripId: number,
|
||||
now: Date = new Date(),
|
||||
): Promise<MailIngestCandidateRow> {
|
||||
const entry = await this.pendingEntry(candidateId, userId);
|
||||
const trip = await this.tripFor(userId, tripId);
|
||||
const mail = await this.mailOf(entry);
|
||||
if (!mail) throw new DomainError(410, 'The mail is no longer stored');
|
||||
const { fileIds, error } = await this.apply.attachDocuments(trip.id, null, mail, userId);
|
||||
if (fileIds.length === 0) throw new DomainError(409, error ?? 'This trip takes no documents');
|
||||
const created = { reservation_ids: [], place_ids: [], budget_item_ids: [], file_ids: fileIds, fingerprint: null };
|
||||
await this.candidates.resolve(entry.id, ['pending'], {
|
||||
resolution: 'filed',
|
||||
reason: null,
|
||||
resolvedTripId: trip.id,
|
||||
createdIdsJson: encodeJson(MAIL_INGEST_CREATED, created),
|
||||
resolvedBy: userId,
|
||||
resolvedAt: dbNow(now),
|
||||
undoUntil: null,
|
||||
});
|
||||
return this.reload(entry.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* "Fill in yourself": the booking form's starting values. The booking read
|
||||
* from the mail where there is one; otherwise what the mail says plainly: a
|
||||
* type by its words, the subject as title, the first date, a reference.
|
||||
*/
|
||||
async prefill(
|
||||
candidateId: number,
|
||||
userId: number,
|
||||
): Promise<{ trip_id: number | null; item: BookingImportPreviewItem }> {
|
||||
const entry = await this.pendingEntry(candidateId, userId);
|
||||
const item = decodeJson(MAIL_INGEST_ITEM, entry.item_json);
|
||||
if (item) return { trip_id: entry.suggested_trip_id, item };
|
||||
const mail = await this.mailOf(entry);
|
||||
const message = await this.messages.findById(entry.message_id);
|
||||
return { trip_id: entry.suggested_trip_id, item: guessedItem(mail, message?.subject ?? null) };
|
||||
}
|
||||
|
||||
/** Puts a waiting entry aside; the mail goes with the retention like any other. */
|
||||
async discard(candidateId: number, userId: number, now: Date = new Date()): Promise<MailIngestCandidateRow> {
|
||||
const entry = await this.pendingEntry(candidateId, userId);
|
||||
await this.candidates.resolve(entry.id, ['pending'], {
|
||||
resolution: 'discarded',
|
||||
reason: entry.reason,
|
||||
resolvedTripId: null,
|
||||
createdIdsJson: null,
|
||||
resolvedBy: userId,
|
||||
resolvedAt: dbNow(now),
|
||||
undoUntil: null,
|
||||
});
|
||||
return this.reload(entry.id);
|
||||
}
|
||||
|
||||
/** What an undo would touch beyond the booking: costs someone settled with, and an edit since. */
|
||||
private async undoAffects(tripId: number, created: MailIngestCreated): Promise<Array<'balances' | 'edited'>> {
|
||||
const affects: Array<'balances' | 'edited'> = [];
|
||||
if ((await this.candidates.countPayers(created.budget_item_ids)) > 0) affects.push('balances');
|
||||
if (await this.apply.editedSince(tripId, created)) affects.push('edited');
|
||||
return affects;
|
||||
}
|
||||
|
||||
/** The person's own waiting entry; anyone else's answers as missing. */
|
||||
private async pendingEntry(candidateId: number, userId: number): Promise<MailIngestCandidateRow> {
|
||||
const entry = await this.candidates.findById(candidateId);
|
||||
if (!entry || entry.user_id !== userId) throw new DomainError(404, 'Entry not found');
|
||||
if (entry.resolution !== 'pending') throw new DomainError(409, 'This entry was decided already');
|
||||
return entry;
|
||||
}
|
||||
|
||||
private async tripFor(userId: number, tripId: number | null): Promise<TripOption> {
|
||||
const person = await this.trips.person(userId);
|
||||
const trip = person && tripId !== null ? await this.trips.editableTrip(person, tripId) : null;
|
||||
if (!trip) throw new DomainError(404, 'Trip not found');
|
||||
return trip;
|
||||
}
|
||||
|
||||
private async editableBy(userId: number, tripId: number): Promise<boolean> {
|
||||
const person = await this.trips.person(userId);
|
||||
return person !== null && (await this.trips.editableTrip(person, tripId)) !== null;
|
||||
}
|
||||
|
||||
/** The mail an entry came from, as long as it is stored. */
|
||||
private async mailOf(entry: MailIngestCandidateRow): Promise<WalkedMail | null> {
|
||||
const message = await this.messages.findById(entry.message_id);
|
||||
const raw = openMail(message?.raw_eml_enc ?? null);
|
||||
return raw ? this.reader.walk(raw) : null;
|
||||
}
|
||||
|
||||
private async reload(id: number): Promise<MailIngestCandidateRow> {
|
||||
const row = await this.candidates.findById(id);
|
||||
if (!row) throw new DomainError(404, 'Entry not found');
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
/** A booking form started from what a mail says in plain words, for a mail no reading made sense of. */
|
||||
function guessedItem(mail: WalkedMail | null, storedSubject: string | null): BookingImportPreviewItem {
|
||||
const subject = mail?.subject ?? storedSubject ?? '';
|
||||
const text = `${subject}\n${mail?.body ?? ''}`;
|
||||
const type = TYPE_WORDS.find(([words]) => words.test(text))?.[1] ?? 'other';
|
||||
const date = ISO_DATE_RE.exec(text)?.[1] ?? null;
|
||||
const reference = REFERENCE_RE.exec(text)?.[1] ?? null;
|
||||
return {
|
||||
type,
|
||||
title: subject.trim().slice(0, 200) || 'Booking',
|
||||
reservation_time: date,
|
||||
confirmation_number: reference ? reference.toUpperCase() : null,
|
||||
source: { fileName: 'mail', index: 0 },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import { MailIngestAddresses } from '../../db/entities/MailIngestAddresses.entity';
|
||||
import type {
|
||||
MailIngestAddressesRepository,
|
||||
MailIngestAddressRow,
|
||||
} from '../../db/repositories/MailIngestAddresses.repository';
|
||||
import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import { dbNow, utcSuffix } from '../../db/types';
|
||||
import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
import { DomainError } from '../common/domain-error';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { TOKEN_LENGTH } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { MailIngestAddress } from '@trek/shared';
|
||||
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
|
||||
/** RFC 4648 base32, lower case: a mail's local part keeps it as it is. */
|
||||
const BASE32 = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
|
||||
/** A live address and the person it belongs to, as a mail that reached it is filed under. */
|
||||
export interface ResolvedAddress {
|
||||
address: MailIngestAddressRow;
|
||||
person: MailIngestPersonRow;
|
||||
}
|
||||
|
||||
/**
|
||||
* The forwarding addresses. A person has one of their own and may make one
|
||||
* per trip they can add bookings to; each address is a random token behind
|
||||
* the admin's catch-all, stored as its sha256 for the lookup and encrypted
|
||||
* so its owner can see it again. Renewing replaces the token at once.
|
||||
*
|
||||
* A trip address belongs to the member who made it, and stops working the
|
||||
* moment they can no longer add bookings to the trip: every mail and every
|
||||
* write checks that again.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestAddressService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestAddresses) private readonly addresses: MailIngestAddressesRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly trips: MailIngestTripsService,
|
||||
private readonly uow: UnitOfWork,
|
||||
) {}
|
||||
|
||||
/** The person's own address; null until they asked for one or while no address template is set. */
|
||||
async ownAddress(userId: number): Promise<MailIngestAddress | null> {
|
||||
return this.view(await this.addresses.findLive(userId, null));
|
||||
}
|
||||
|
||||
/** A new own address for the person, the old one stops working. */
|
||||
async renewOwn(userId: number): Promise<MailIngestAddress | null> {
|
||||
await this.requireTemplate();
|
||||
return this.view(await this.rotate(userId, null));
|
||||
}
|
||||
|
||||
/** The person's address for the trip, if they made one and may still add bookings there. */
|
||||
async tripAddress(userId: number, tripId: number): Promise<MailIngestAddress | null> {
|
||||
await this.requireTripEdit(userId, tripId);
|
||||
return this.view(await this.addresses.findLive(userId, tripId));
|
||||
}
|
||||
|
||||
/** A new address for the trip, for a person who may add bookings to it; an old one stops working. */
|
||||
async renewTripAddress(userId: number, tripId: number): Promise<MailIngestAddress | null> {
|
||||
await this.requireTripEdit(userId, tripId);
|
||||
await this.requireTemplate();
|
||||
return this.view(await this.rotate(userId, tripId));
|
||||
}
|
||||
|
||||
/** The person's address for the trip stops working. */
|
||||
async revokeTripAddress(userId: number, tripId: number): Promise<void> {
|
||||
await this.addresses.revokeLive(userId, tripId, dbNow());
|
||||
}
|
||||
|
||||
/**
|
||||
* The live address a mail was sent to, from the recipients in the order the
|
||||
* receiving server names them (Delivered-To, X-Original-To, To). Null for a
|
||||
* token nobody holds, a revoked one, a person who switched the import off,
|
||||
* and a trip address whose owner may no longer add bookings to the trip.
|
||||
*/
|
||||
async resolve(recipients: readonly string[]): Promise<ResolvedAddress | null> {
|
||||
const template = await this.settings.addressTemplate();
|
||||
const tokens = recipients
|
||||
.map((recipient) => tokenOf(recipient, template))
|
||||
.filter((token): token is string => token !== null);
|
||||
const found = await Promise.all(tokens.map((token) => this.addresses.findByTokenHash(hashToken(token))));
|
||||
const address = found.find((row): row is MailIngestAddressRow => row !== null);
|
||||
if (!address || !(await this.isLive(address))) return null;
|
||||
return this.withPerson(address);
|
||||
}
|
||||
|
||||
/** The person's own live address, made now when they have none: an uploaded mail goes through it. */
|
||||
async ensureOwn(userId: number): Promise<MailIngestAddressRow> {
|
||||
return (await this.addresses.findLive(userId, null)) ?? (await this.rotate(userId, null));
|
||||
}
|
||||
|
||||
/** Whether an address takes mail: not revoked, its person's import on, a trip address's owner still an editor. */
|
||||
async isLive(address: MailIngestAddressRow): Promise<boolean> {
|
||||
if (address.revoked_at) return false;
|
||||
if (!(await this.settings.prefsOf(address.user_id)).enabled) return false;
|
||||
if (address.trip_id === null) return true;
|
||||
const person = await this.trips.person(address.user_id);
|
||||
return person !== null && (await this.trips.editableTrip(person, address.trip_id)) !== null;
|
||||
}
|
||||
|
||||
async findById(id: number): Promise<MailIngestAddressRow | null> {
|
||||
return this.addresses.findById(id);
|
||||
}
|
||||
|
||||
/** Notes that a mail came in on the address. */
|
||||
async touch(addressId: number): Promise<void> {
|
||||
await this.addresses.touchUsed(addressId, dbNow());
|
||||
}
|
||||
|
||||
private async withPerson(address: MailIngestAddressRow): Promise<ResolvedAddress | null> {
|
||||
const person = await this.trips.person(address.user_id);
|
||||
return person ? { address, person } : null;
|
||||
}
|
||||
|
||||
private async rotate(userId: number, tripId: number | null): Promise<MailIngestAddressRow> {
|
||||
const token = newToken();
|
||||
const id = await this.uow.transactional(async () => {
|
||||
const now = dbNow();
|
||||
await this.addresses.revokeLive(userId, tripId, now);
|
||||
return this.addresses.insertAddress({
|
||||
userId,
|
||||
tripId,
|
||||
tokenHash: hashToken(token),
|
||||
tokenEnc: encrypt_api_key(token),
|
||||
createdAt: now,
|
||||
});
|
||||
});
|
||||
const row = await this.addresses.findById(id);
|
||||
if (!row) throw new Error(`mail address ${id} vanished after its insert`);
|
||||
return row;
|
||||
}
|
||||
|
||||
private async view(row: MailIngestAddressRow | null): Promise<MailIngestAddress | null> {
|
||||
if (!row) return null;
|
||||
const template = await this.settings.addressTemplate();
|
||||
const token = decrypt_api_key(row.token_enc);
|
||||
if (!template || !token) return null;
|
||||
return {
|
||||
id: row.id,
|
||||
address: template.replace('{token}', token),
|
||||
trip_id: row.trip_id,
|
||||
created_at: utcSuffix(row.created_at) ?? row.created_at,
|
||||
last_used_at: utcSuffix(row.last_used_at),
|
||||
};
|
||||
}
|
||||
|
||||
private async requireTemplate(): Promise<void> {
|
||||
if (!(await this.settings.addressTemplate())) {
|
||||
throw new DomainError(409, 'No forwarding address is set up on this server');
|
||||
}
|
||||
}
|
||||
|
||||
private async requireTripEdit(userId: number, tripId: number): Promise<void> {
|
||||
const person = await this.trips.person(userId);
|
||||
if (!person || !(await this.trips.editableTrip(person, tripId))) {
|
||||
throw new DomainError(404, 'Trip not found');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A random token of {@link TOKEN_LENGTH} base32 characters, from the system's cryptographic source. */
|
||||
export function newToken(): string {
|
||||
const bytes = randomBytes(TOKEN_LENGTH);
|
||||
return [...bytes].map((byte) => BASE32[byte % 32]).join('');
|
||||
}
|
||||
|
||||
/** The sha256 a token is looked up by; tokens are case-insensitive, as a mail's local part is in practice. */
|
||||
export function hashToken(token: string): string {
|
||||
return createHash('sha256').update(token.toLowerCase()).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* The token a recipient address carries: what stands where `{token}` stands
|
||||
* in the template, or, for a box with sub-addressing, what follows the `+`.
|
||||
* Null for an address that holds none.
|
||||
*/
|
||||
export function tokenOf(recipient: string, template: string | null): string | null {
|
||||
const address = recipient.replace(/^.*</, '').replace(/>.*$/, '').trim().toLowerCase();
|
||||
const local = address.split('@')[0] ?? '';
|
||||
if (template) {
|
||||
const [prefix, suffix] = template.toLowerCase().split('@')[0].split('{token}');
|
||||
if (suffix !== undefined && local.startsWith(prefix) && local.endsWith(suffix)) {
|
||||
const token = local.slice(prefix.length, local.length - suffix.length);
|
||||
if (isToken(token)) return token;
|
||||
}
|
||||
}
|
||||
const plus = local.split('+')[1];
|
||||
return plus && isToken(plus) ? plus : null;
|
||||
}
|
||||
|
||||
function isToken(value: string): boolean {
|
||||
return value.length >= 20 && /^[a-z2-7]+$/.test(value);
|
||||
}
|
||||
@@ -0,0 +1,386 @@
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import type { MailIngestCreated } from '../../db/json-columns';
|
||||
import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import { BookingImportService } from '../booking-import/booking-import.service';
|
||||
import { BudgetService } from '../budget/budget.service';
|
||||
import { isUploadTypeAllowed } from '../files/files.constants';
|
||||
import { FilesService } from '../files/files.service';
|
||||
import type { WalkedMail } from '../llm-parse/mail-read.types';
|
||||
import { MapsService } from '../maps/maps.service';
|
||||
import { PlacesService, type PlaceCreateInput } from '../places/places.service';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import { ReservationsService, type CostEvent, type CreateReservationData } from '../reservations/reservations.service';
|
||||
import { contentHash, dateOf, type TripOption } from './mail-ingest-checks';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MAIL_EXTERNAL_SOURCE } from './mail-ingest.constants';
|
||||
import { parsePriceMinor } from './mail-money';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
normalizePlaceWebsite,
|
||||
typeToCostCategory,
|
||||
type BookingImportPreviewItem,
|
||||
type MailIngestField,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
type CostEntry = Parameters<ReservationsService['createWithCostInTx']>[2];
|
||||
|
||||
/** What a booking's fingerprint covers: the fields a person edits. */
|
||||
const FINGERPRINT_KEYS = [
|
||||
'title',
|
||||
'type',
|
||||
'status',
|
||||
'reservation_time',
|
||||
'reservation_end_time',
|
||||
'location',
|
||||
'confirmation_number',
|
||||
'notes',
|
||||
'metadata',
|
||||
] as const;
|
||||
type Written = Awaited<ReturnType<ReservationsService['createWithCostInTx']>>;
|
||||
|
||||
/** A booking ready to write: everything that needs the network (geocoding, the frozen rate) is done. */
|
||||
export interface PreparedBooking {
|
||||
tripId: number;
|
||||
data: CreateReservationData;
|
||||
venue: PlaceCreateInput | null;
|
||||
cost: CostEntry;
|
||||
travelerIds: number[];
|
||||
/** The fields marked for a second look, the price among them when no cost could be written. */
|
||||
uncertain: MailIngestField[];
|
||||
}
|
||||
|
||||
/** What writing a booking created, for the announcement after the commit and for its undo. */
|
||||
export interface WrittenBooking {
|
||||
tripId: number;
|
||||
reservation: Written['reservation'];
|
||||
place: { id: number } | null;
|
||||
accommodationCreated: boolean;
|
||||
costEvents: CostEvent[];
|
||||
stayMirror: Written['stayMirror'];
|
||||
created: MailIngestCreated;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a booking read from a mail into its trip, with every field the mail
|
||||
* gave: the booking or transport with its endpoints and legs, a stay for a
|
||||
* hotel, the venue as a place (an existing one when the trip has it), the
|
||||
* forwarder as traveller, and the cost when the price reads to the cent. It
|
||||
* goes through the same services a booking made by hand goes through, and
|
||||
* records exactly what it created, so an undo removes that and nothing else.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestApplyService {
|
||||
constructor(
|
||||
private readonly reservations: ReservationsService,
|
||||
private readonly places: PlacesService,
|
||||
private readonly budget: BudgetService,
|
||||
private readonly addons: AddonsService,
|
||||
private readonly files: FilesService,
|
||||
private readonly maps: MapsService,
|
||||
private readonly realtime: RealtimeService,
|
||||
private readonly bookingImport: BookingImportService,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
) {}
|
||||
|
||||
/** The key a mail booking is stored under: its reference, or, without one, what it says. */
|
||||
externalKey(item: BookingImportPreviewItem): string {
|
||||
return this.bookingImport.externalKey(item) ?? `content:${contentHash(item)}`;
|
||||
}
|
||||
|
||||
/** The network half of a write: locate the venue and freeze the cost's rate. Writes nothing. */
|
||||
async prepare(
|
||||
item: BookingImportPreviewItem,
|
||||
trip: TripOption,
|
||||
forwarderId: number,
|
||||
uncertain: readonly MailIngestField[],
|
||||
): Promise<PreparedBooking> {
|
||||
const marked = new Set(uncertain);
|
||||
const cost = await this.costOf(item, trip);
|
||||
if (cost === null) marked.add('price');
|
||||
const venue = await this.venueOf(item);
|
||||
return {
|
||||
tripId: trip.id,
|
||||
data: this.reservationData(item, trip, marked),
|
||||
venue,
|
||||
cost: cost ?? undefined,
|
||||
travelerIds: [forwarderId],
|
||||
uncertain: [...marked],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The write itself, inside the caller's transaction: the place (matched or
|
||||
* new), the booking with its stay and cost, its traveller. Nothing is
|
||||
* announced here; the caller announces once the transaction committed.
|
||||
*/
|
||||
async writeInTx(prepared: PreparedBooking): Promise<WrittenBooking> {
|
||||
const tripId = String(prepared.tripId);
|
||||
const matched = prepared.venue ? await this.places.findMatchingPlaceId(tripId, prepared.venue) : null;
|
||||
const place = prepared.venue && matched === null ? await this.places.create(tripId, prepared.venue) : null;
|
||||
const placeId = matched ?? place?.id ?? undefined;
|
||||
const stay = prepared.data.create_accommodation;
|
||||
const written = await this.reservations.createWithCostInTx(
|
||||
tripId,
|
||||
{ ...prepared.data, place_id: placeId, ...(stay && { create_accommodation: { ...stay, place_id: placeId } }) },
|
||||
prepared.cost,
|
||||
);
|
||||
await this.reservations.setReservationTravelers(written.reservation.id, tripId, prepared.travelerIds);
|
||||
// Read back once everything is written: the fingerprint an undo compares against is the booking as it stands.
|
||||
const stored = await this.reservations.getReservationWithJoins(written.reservation.id, tripId);
|
||||
return {
|
||||
tripId: prepared.tripId,
|
||||
reservation: written.reservation,
|
||||
place: place ? { id: place.id } : null,
|
||||
accommodationCreated: written.accommodationCreated,
|
||||
costEvents: written.costEvents,
|
||||
stayMirror: written.stayMirror,
|
||||
created: {
|
||||
reservation_ids: [written.reservation.id],
|
||||
place_ids: place ? [place.id] : [],
|
||||
budget_item_ids: createdItemIds(written.costEvents),
|
||||
file_ids: [],
|
||||
fingerprint: fingerprintOf(stored ?? written.reservation),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** What a committed write tells the trip, to everyone, the forwarder's devices included. */
|
||||
async announce(written: WrittenBooking): Promise<void> {
|
||||
const { tripId } = written;
|
||||
if (written.place) this.realtime.broadcast(tripId, 'place:created', { place: written.place });
|
||||
this.realtime.broadcast(tripId, 'reservation:created', { reservation: written.reservation });
|
||||
if (written.accommodationCreated) this.realtime.broadcast(tripId, 'accommodation:created', {});
|
||||
await this.reservations.announceStayMirror(tripId, written.stayMirror);
|
||||
this.reservations.announceCost(tripId, written.costEvents, undefined);
|
||||
}
|
||||
|
||||
/**
|
||||
* The mail's documents as trip files on the booking (or on the trip, for a
|
||||
* mail filed without one), when the documents addon is on: each attachment
|
||||
* of a type the trip takes, or the mail's text when none is. A failure
|
||||
* leaves the booking standing and is reported, never thrown.
|
||||
*
|
||||
* @txIndependent one file per attachment on purpose: each upload is storage
|
||||
* I/O, and a file that failed leaves the ones before it in place.
|
||||
*/
|
||||
async attachDocuments(
|
||||
tripId: number,
|
||||
reservationId: number | null,
|
||||
mail: WalkedMail,
|
||||
userId: number,
|
||||
): Promise<{ fileIds: number[]; error: string | null }> {
|
||||
if (!(await this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS))) return { fileIds: [], error: null };
|
||||
const allowed = await this.settings.allowedFileTypes();
|
||||
const kept = mail.attachments.filter((file) => isUploadTypeAllowed(file.fileName, file.mimeType, allowed));
|
||||
const uploads = kept.length
|
||||
? kept.map((file) => ({ originalname: file.fileName, mimetype: file.mimeType, bytes: file.data }))
|
||||
: [{ originalname: 'mail.txt', mimetype: 'text/plain', bytes: Buffer.from(mailText(mail), 'utf8') }];
|
||||
const fileIds: number[] = [];
|
||||
try {
|
||||
await uploads.reduce(async (previous, upload) => {
|
||||
await previous;
|
||||
const file = await this.files.createFileFromBytes(tripId, upload, userId, { reservation_id: reservationId });
|
||||
fileIds.push(file.id);
|
||||
this.realtime.broadcast(tripId, 'file:created', { file });
|
||||
}, Promise.resolve());
|
||||
return { fileIds, error: null };
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
logWarn(`[mail-ingest] Attaching a mail's documents to trip ${tripId} failed: ${reason}`);
|
||||
return { fileIds, error: reason.slice(0, 300) };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes what adding a booking created, inside the caller's transaction:
|
||||
* the booking with its stay and costs, and a place it made that nothing
|
||||
* else names any more. Answers what to announce once that committed.
|
||||
*/
|
||||
async removeInTx(tripId: number, created: MailIngestCreated): Promise<RemovedBooking> {
|
||||
const removed: RemovedBooking = {
|
||||
tripId,
|
||||
reservationIds: [],
|
||||
accommodationIds: [],
|
||||
budgetItemIds: [],
|
||||
placeIds: [],
|
||||
};
|
||||
await created.reservation_ids.reduce(async (previous, id) => {
|
||||
await previous;
|
||||
const gone = await this.reservations.remove(id, tripId);
|
||||
if (!gone.deleted) return;
|
||||
removed.reservationIds.push(id);
|
||||
removed.budgetItemIds.push(...gone.deletedBudgetItemIds);
|
||||
if (gone.accommodationDeleted && gone.deleted.accommodation_id) {
|
||||
removed.accommodationIds.push(gone.deleted.accommodation_id);
|
||||
}
|
||||
}, Promise.resolve());
|
||||
await created.place_ids.reduce(async (previous, placeId) => {
|
||||
await previous;
|
||||
if (await this.candidates.placeInUse(placeId)) return;
|
||||
if ((await this.places.remove(String(tripId), String(placeId))).deleted) removed.placeIds.push(placeId);
|
||||
}, Promise.resolve());
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** What an undo tells the trip once it committed, and the documents it moves to the trash. */
|
||||
async announceRemoved(removed: RemovedBooking, fileIds: readonly number[]): Promise<void> {
|
||||
const { tripId } = removed;
|
||||
for (const accommodationId of removed.accommodationIds) {
|
||||
this.realtime.broadcast(tripId, 'accommodation:deleted', { accommodationId });
|
||||
}
|
||||
for (const itemId of removed.budgetItemIds) this.realtime.broadcast(tripId, 'budget:deleted', { itemId });
|
||||
for (const reservationId of removed.reservationIds) {
|
||||
this.realtime.broadcast(tripId, 'reservation:deleted', { reservationId });
|
||||
}
|
||||
for (const placeId of removed.placeIds) this.realtime.broadcast(tripId, 'place:deleted', { placeId });
|
||||
await Promise.all(fileIds.map((fileId) => this.files.softDeleteFile(fileId)));
|
||||
for (const fileId of fileIds) this.realtime.broadcast(tripId, 'file:deleted', { fileId });
|
||||
}
|
||||
|
||||
/** Whether the booking was changed since it was written, by its fingerprint then and now. */
|
||||
async editedSince(tripId: number, created: MailIngestCreated): Promise<boolean> {
|
||||
if (created.fingerprint === null) return false;
|
||||
const [reservationId] = created.reservation_ids;
|
||||
if (reservationId === undefined) return true;
|
||||
const now = await this.reservations.getReservationWithJoins(reservationId, tripId);
|
||||
return !now || fingerprintOf(now) !== created.fingerprint;
|
||||
}
|
||||
|
||||
private reservationData(
|
||||
item: BookingImportPreviewItem,
|
||||
trip: TripOption,
|
||||
marked: Set<MailIngestField>,
|
||||
): CreateReservationData {
|
||||
const { _venue, _accommodation, source: _source, type_guessed: _guessed, needs_review: _review, ...fields } = item;
|
||||
const endpoints = (fields.endpoints ?? []).flatMap((endpoint) =>
|
||||
endpoint.lat != null && endpoint.lng != null ? [{ ...endpoint, lat: endpoint.lat, lng: endpoint.lng }] : [],
|
||||
);
|
||||
if (endpoints.length < (fields.endpoints ?? []).length) marked.add('endpoints');
|
||||
return {
|
||||
title: fields.title,
|
||||
type: fields.type,
|
||||
status: statusOf(item),
|
||||
reservation_time: fields.reservation_time ?? undefined,
|
||||
reservation_end_time: fields.reservation_end_time ?? undefined,
|
||||
location: fields.location ?? undefined,
|
||||
confirmation_number: fields.confirmation_number ?? undefined,
|
||||
metadata: fields.metadata,
|
||||
endpoints,
|
||||
needs_review: marked.size > 0,
|
||||
external_source: MAIL_EXTERNAL_SOURCE,
|
||||
external_id: this.externalKey(item),
|
||||
sync_enabled: false,
|
||||
...(item.type === 'hotel' && _accommodation && { create_accommodation: stayOf(_accommodation, trip) }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The cost a price becomes: only with the costs addon on, a price read to
|
||||
* the cent and, in a foreign currency, a rate frozen now. Undefined where
|
||||
* there is no cost to write, null where the mail names a price that cannot
|
||||
* be written without guessing (the field is marked instead).
|
||||
*/
|
||||
private async costOf(item: BookingImportPreviewItem, trip: TripOption): Promise<CostEntry | null> {
|
||||
const meta = item.metadata ?? {};
|
||||
if (meta.price == null || meta.price === '') return undefined;
|
||||
if (!(await this.addons.isAddonEnabled(ADDON_IDS.BUDGET))) return undefined;
|
||||
const currency = typeof meta.priceCurrency === 'string' ? meta.priceCurrency.toUpperCase() : null;
|
||||
const minor = parsePriceMinor(meta.price, currency ?? trip.currency);
|
||||
if (minor === null || minor <= 0) return null;
|
||||
const entry: NonNullable<CostEntry> = {
|
||||
total_price: minor / 100,
|
||||
category: typeToCostCategory(item.type),
|
||||
currency,
|
||||
};
|
||||
try {
|
||||
await this.budget.freezeForeignRate(trip.id, entry);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const abroad = currency !== null && currency !== trip.currency;
|
||||
return abroad && entry.exchange_rate === undefined ? null : entry;
|
||||
}
|
||||
|
||||
/** The venue as a place: its position from the mail, or looked up by its name and address. */
|
||||
private async venueOf(item: BookingImportPreviewItem): Promise<PlaceCreateInput | null> {
|
||||
const venue = item._venue;
|
||||
if (!venue?.name) return null;
|
||||
let position = venue.lat != null && venue.lng != null ? { lat: venue.lat, lng: venue.lng } : null;
|
||||
if (!position) {
|
||||
const query = venue.address ? `${venue.name} ${venue.address}` : venue.name;
|
||||
position = await this.maps.geocodeQuery(query).catch(() => null);
|
||||
}
|
||||
return {
|
||||
name: venue.name,
|
||||
lat: position?.lat,
|
||||
lng: position?.lng,
|
||||
address: venue.address,
|
||||
website: normalizePlaceWebsite(venue.website) ?? undefined,
|
||||
phone: venue.phone,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** What an undo removed, for the announcement after it committed. */
|
||||
export interface RemovedBooking {
|
||||
tripId: number;
|
||||
reservationIds: number[];
|
||||
accommodationIds: string[];
|
||||
budgetItemIds: number[];
|
||||
placeIds: number[];
|
||||
}
|
||||
|
||||
/** The status a booking is written with: confirmed when the mail said so, pending otherwise. */
|
||||
function statusOf(item: BookingImportPreviewItem): string {
|
||||
const status = item.metadata?.reservationStatus;
|
||||
return typeof status === 'string' && /confirm/i.test(status) ? 'confirmed' : 'pending';
|
||||
}
|
||||
|
||||
/** A hotel's stay: its days by check-in and check-out date, its times of day alone. */
|
||||
function stayOf(
|
||||
stay: NonNullable<BookingImportPreviewItem['_accommodation']>,
|
||||
trip: TripOption,
|
||||
): NonNullable<CreateReservationData['create_accommodation']> {
|
||||
const checkIn = dateOf(stay.check_in);
|
||||
const checkOut = dateOf(stay.check_out);
|
||||
return {
|
||||
start_day_id: checkIn ? trip.days.get(checkIn) : undefined,
|
||||
end_day_id: checkOut ? trip.days.get(checkOut) : undefined,
|
||||
check_in: timeOf(stay.check_in),
|
||||
check_out: timeOf(stay.check_out),
|
||||
confirmation: stay.confirmation,
|
||||
};
|
||||
}
|
||||
|
||||
/** The time of day of an ISO date-time, `HH:MM`; undefined for a bare date. */
|
||||
function timeOf(value: string | undefined): string | undefined {
|
||||
const match = /T(\d{2}:\d{2})/.exec(value ?? '');
|
||||
return match ? match[1] : undefined;
|
||||
}
|
||||
|
||||
/** The ids of the costs a write created. */
|
||||
function createdItemIds(events: readonly CostEvent[]): number[] {
|
||||
return events.flatMap((event) => {
|
||||
if (event.event !== 'budget:created') return [];
|
||||
const item = event.payload.item;
|
||||
return item && typeof item === 'object' && 'id' in item && typeof item.id === 'number' ? [item.id] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** A booking's content as one hash: an edit after the write changes it. */
|
||||
function fingerprintOf(reservation: Record<string, unknown>): string {
|
||||
const content = FINGERPRINT_KEYS.map((key) => reservation[key] ?? null);
|
||||
return createHash('sha256').update(JSON.stringify(content)).digest('hex');
|
||||
}
|
||||
|
||||
/** A mail as a text file: its subject and its readable body, without any header. */
|
||||
function mailText(mail: WalkedMail): string {
|
||||
return `${mail.subject ?? ''}\n\n${mail.body}`.trim();
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
import { haversineKm } from '../common/geo';
|
||||
import { NEAR_TRIP_KM } from './mail-ingest.constants';
|
||||
import {
|
||||
bookingImportPreviewItemSchema,
|
||||
type BookingImportPreviewItem,
|
||||
type MailIngestCheck,
|
||||
type MailIngestCheckResult,
|
||||
type MailIngestField,
|
||||
type MailIngestReason,
|
||||
type MailIngestSource,
|
||||
} from '@trek/shared';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
/**
|
||||
* The server's checks on one booking read from a mail (plan 5.3, G1 to G8),
|
||||
* and the choice of its trip. Pure: everything they weigh is handed in, so a
|
||||
* mail cannot steer them, and a test can name every case.
|
||||
*/
|
||||
|
||||
/** A trip a booking may go to, as the checks see it. */
|
||||
export interface TripOption {
|
||||
id: number;
|
||||
title: string;
|
||||
start_date: string | null;
|
||||
end_date: string | null;
|
||||
ownerId: number;
|
||||
/** The trip's currency, upper case. */
|
||||
currency: string;
|
||||
/** The trip's day ids by date. */
|
||||
days: ReadonlyMap<string, number>;
|
||||
places: ReadonlyArray<{ name: string; address: string | null; lat: number; lng: number }>;
|
||||
}
|
||||
|
||||
/** A booking read from a mail, before the checks. */
|
||||
export interface MailDraft {
|
||||
item: BookingImportPreviewItem;
|
||||
source: MailIngestSource;
|
||||
kind: 'new_booking' | 'change' | 'cancellation';
|
||||
/** The trip the AI picked, already mapped from its reference to an id of the list; null when it picked none. */
|
||||
aiTripId: number | null;
|
||||
uncertainFields: MailIngestField[];
|
||||
suggestedNewTrip: { destination: string; start_date: string | null; end_date: string | null } | null;
|
||||
/** Read from page images alone: nothing can be checked against text. */
|
||||
imageOnly: boolean;
|
||||
/** The AI read a critical field differently from the machine-readable data (G4). */
|
||||
disagrees: boolean;
|
||||
}
|
||||
|
||||
/** What the mail around the booking adds to the checks. */
|
||||
export interface CheckContext {
|
||||
trips: readonly TripOption[];
|
||||
/** The trip whose address the mail came in on; null for the person's own address. */
|
||||
addressTripId: number | null;
|
||||
subject: string | null;
|
||||
/** Everything the mail and its attachments say, for the in-text check. */
|
||||
documentText: string;
|
||||
now: Date;
|
||||
knownAirport: (code: string) => boolean;
|
||||
}
|
||||
|
||||
export interface TripChoice {
|
||||
/** The trip the booking goes to; null when none is certain. */
|
||||
tripId: number | null;
|
||||
/** The trip to offer in the inbox. */
|
||||
suggestedTripId: number | null;
|
||||
reason: MailIngestReason | null;
|
||||
}
|
||||
|
||||
export interface BookingVerdict {
|
||||
checks: Partial<Record<MailIngestCheck, MailIngestCheckResult>>;
|
||||
/** The first reason the booking cannot be added on its own; null when every check passed. */
|
||||
reason: MailIngestReason | null;
|
||||
uncertain: MailIngestField[];
|
||||
choice: TripChoice;
|
||||
}
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
/** Types that run from one place to another and need both to be added. */
|
||||
const ROUTED_TYPES = new Set(['flight', 'train', 'bus', 'ferry', 'cruise']);
|
||||
/** Types that are only added on their own with a booking reference: a mail without one is too vague. */
|
||||
const REFERENCE_TYPES = new Set(['flight', 'train', 'bus', 'ferry', 'cruise', 'car', 'hotel']);
|
||||
const MAX_HOURS: Readonly<Record<string, number>> = { flight: 20, train: 48, bus: 48 };
|
||||
const MAX_STAY_NIGHTS = 60;
|
||||
const MAX_PRICE = 100_000;
|
||||
const REFERENCE_RE = /^[A-Z0-9-]{3,40}$/;
|
||||
const CURRENCY_RE = /^[A-Z]{3}$/;
|
||||
/**
|
||||
* Words that mark a cancellation in the subject. Only the subject: every
|
||||
* hotel confirmation speaks of its free cancellation in the body.
|
||||
*/
|
||||
const CANCEL_RE = /\b(cancel+ed|cancel+ation|storniert|stornierung|annul[ée]e?|annulation|cancelad[ao]|annullat[ao]|geannuleerd|anulowan[ay]|отмен)/i;
|
||||
|
||||
/** The date part of an ISO date or date-time, null for anything else. */
|
||||
export function dateOf(value: string | null | undefined): string | null {
|
||||
const date = value?.slice(0, 10) ?? '';
|
||||
return /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : null;
|
||||
}
|
||||
|
||||
/** When the booking starts: its time, or a stay's check-in. */
|
||||
export function bookingStart(item: BookingImportPreviewItem): string | null {
|
||||
return item.reservation_time || item._accommodation?.check_in || null;
|
||||
}
|
||||
|
||||
/** When the booking ends: its end time, or a stay's check-out. */
|
||||
export function bookingEnd(item: BookingImportPreviewItem): string | null {
|
||||
return item.reservation_end_time || item._accommodation?.check_out || null;
|
||||
}
|
||||
|
||||
/** A booking reference as an operator prints it, without the spacing and case a mail adds. */
|
||||
export function normalizeReference(value: string | null | undefined): string {
|
||||
return (value ?? '').replace(/\s+/g, '').toUpperCase();
|
||||
}
|
||||
|
||||
/** No type, no start, or a transport without a named start and end. */
|
||||
export function missingCritical(item: BookingImportPreviewItem): boolean {
|
||||
if (!item.type || !dateOf(bookingStart(item))) return true;
|
||||
if (!ROUTED_TYPES.has(item.type)) return false;
|
||||
return (item.endpoints ?? []).filter((endpoint) => endpoint.name.trim() !== '').length < 2;
|
||||
}
|
||||
|
||||
/** A transport, stay or rental without a booking reference. */
|
||||
export function missingReference(item: BookingImportPreviewItem): boolean {
|
||||
return REFERENCE_TYPES.has(item.type) && normalizeReference(item.confirmation_number) === '';
|
||||
}
|
||||
|
||||
/** G5: a value that cannot be right. */
|
||||
export function implausible(
|
||||
item: BookingImportPreviewItem,
|
||||
now: Date,
|
||||
knownAirport: (code: string) => boolean,
|
||||
): boolean {
|
||||
const start = Date.parse(bookingStart(item) ?? '');
|
||||
if (!Number.isFinite(start)) return true;
|
||||
if (start < now.getTime() - 366 * DAY_MS || start > now.getTime() + 3 * 366 * DAY_MS) return true;
|
||||
const end = Date.parse(bookingEnd(item) ?? '');
|
||||
if (Number.isFinite(end) && (end < start || tooLong(item.type, end - start))) return true;
|
||||
if (item.type === 'flight' && (item.endpoints ?? []).some((e) => e.code && !knownAirport(e.code))) return true;
|
||||
const reference = normalizeReference(item.confirmation_number);
|
||||
if (reference && !REFERENCE_RE.test(reference)) return true;
|
||||
return !plausibleMoney(item.metadata);
|
||||
}
|
||||
|
||||
function tooLong(type: string, ms: number): boolean {
|
||||
if (type === 'hotel') return ms > MAX_STAY_NIGHTS * DAY_MS;
|
||||
const hours = MAX_HOURS[type];
|
||||
return hours !== undefined && ms > hours * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
function plausibleMoney(metadata: Record<string, unknown> | undefined): boolean {
|
||||
const currency = metadata?.priceCurrency;
|
||||
if (currency != null && !(typeof currency === 'string' && CURRENCY_RE.test(currency.toUpperCase()))) return false;
|
||||
const price = metadata?.price;
|
||||
if (price == null || price === '') return true;
|
||||
const amount = Number(price);
|
||||
// A price written with separators is left to the money parser, which marks rather than refuses it.
|
||||
return !Number.isFinite(amount) || (amount > 0 && amount < MAX_PRICE);
|
||||
}
|
||||
|
||||
/** G8: the booking, or the mail it came in, is a cancellation. */
|
||||
export function isCancellation(
|
||||
item: BookingImportPreviewItem,
|
||||
kind: MailDraft['kind'],
|
||||
subject: string | null,
|
||||
): boolean {
|
||||
if (kind === 'cancellation') return true;
|
||||
const status = item.metadata?.reservationStatus;
|
||||
if (typeof status === 'string' && /cancel/i.test(status)) return true;
|
||||
return CANCEL_RE.test(subject ?? '');
|
||||
}
|
||||
|
||||
/** Letters and digits only, upper case: how a reference or a flight number is looked for in the text. */
|
||||
function compact(text: string): string {
|
||||
return text.toUpperCase().replace(/[^\p{L}\p{N}]/gu, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* G6: what an AI reading names has to be in the mail. A reference or a
|
||||
* service number that is not refuses the booking; a free text that is not
|
||||
* (its title, its place) only marks the field for a second look.
|
||||
*/
|
||||
export function checkInText(
|
||||
item: BookingImportPreviewItem,
|
||||
documentText: string,
|
||||
): { ok: boolean; uncertain: MailIngestField[] } {
|
||||
const text = compact(documentText);
|
||||
const reference = compact(item.confirmation_number ?? '');
|
||||
const number = compact(String(item.metadata?.flight_number ?? item.metadata?.train_number ?? ''));
|
||||
const ok = (!reference || text.includes(reference)) && (!number || text.includes(number));
|
||||
const uncertain: MailIngestField[] = [];
|
||||
if (!freeTextIn(item.title, documentText)) uncertain.push('title');
|
||||
if (item.location && !freeTextIn(item.location, documentText)) uncertain.push('location');
|
||||
return { ok, uncertain };
|
||||
}
|
||||
|
||||
/** Whether most words of a free text appear in the document, case and accents aside. */
|
||||
function freeTextIn(value: string, documentText: string): boolean {
|
||||
const fold = (s: string) => s.normalize('NFKD').replace(/\p{M}/gu, '').toLowerCase();
|
||||
const haystack = fold(documentText);
|
||||
const words = fold(value)
|
||||
.split(/[^\p{L}\p{N}]+/u)
|
||||
.filter((word) => word.length >= 3);
|
||||
if (words.length === 0) return true;
|
||||
return words.filter((word) => haystack.includes(word)).length * 2 >= words.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* G3: every date of the booking lies within the trip (a day of slack on each
|
||||
* side) and the days it needs exist: a stay its check-in and check-out day,
|
||||
* anything else the day it starts on.
|
||||
*/
|
||||
export function fitsTrip(item: BookingImportPreviewItem, trip: TripOption): boolean {
|
||||
if (!trip.start_date || !trip.end_date) return false;
|
||||
const from = shiftDate(trip.start_date, -1);
|
||||
const to = shiftDate(trip.end_date, 1);
|
||||
const start = dateOf(bookingStart(item));
|
||||
const end = dateOf(bookingEnd(item));
|
||||
const dates = [start, end].filter((date): date is string => date !== null);
|
||||
if (dates.length === 0 || dates.some((date) => date < from || date > to)) return false;
|
||||
const needed = item.type === 'hotel' ? dates : [start];
|
||||
return needed.every((date) => date !== null && trip.days.has(date));
|
||||
}
|
||||
|
||||
function shiftDate(date: string, days: number): string {
|
||||
return new Date(Date.parse(`${date}T00:00:00Z`) + days * DAY_MS).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/** Whether the booking's destination lies near one of the trip's places, by coordinates the server holds. */
|
||||
export function nearTrip(item: BookingImportPreviewItem, trip: TripOption): boolean {
|
||||
const points = [...(item.endpoints ?? []).slice(-1), ...(item._venue ? [item._venue] : [])].filter(
|
||||
(point): point is typeof point & { lat: number; lng: number } => point.lat != null && point.lng != null,
|
||||
);
|
||||
return points.some((point) =>
|
||||
trip.places.some((place) => haversineKm(point.lat, point.lng, place.lat, place.lng) <= NEAR_TRIP_KM),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* G2 and G7: the trip the booking goes to. The server decides; the AI's pick
|
||||
* is one input. A trip address names its trip. Otherwise the booking goes
|
||||
* where exactly one trip fits its dates, or, of several that do, to the AI's
|
||||
* pick when the server confirms the booking lies near it.
|
||||
*/
|
||||
export function chooseTrip(item: BookingImportPreviewItem, aiTripId: number | null, context: CheckContext): TripChoice {
|
||||
const { trips, addressTripId } = context;
|
||||
if (addressTripId !== null) {
|
||||
const trip = trips.find((candidate) => candidate.id === addressTripId);
|
||||
if (!trip) return { tripId: null, suggestedTripId: null, reason: 'no_edit_permission' };
|
||||
if (trip.start_date && trip.end_date && !fitsTrip(item, trip)) {
|
||||
return { tripId: null, suggestedTripId: trip.id, reason: 'outside_trip_dates' };
|
||||
}
|
||||
return { tripId: trip.id, suggestedTripId: trip.id, reason: null };
|
||||
}
|
||||
const passing = trips.filter((trip) => fitsTrip(item, trip));
|
||||
const picked = passing.find((trip) => trip.id === aiTripId);
|
||||
if (passing.length === 1) return { tripId: passing[0].id, suggestedTripId: passing[0].id, reason: null };
|
||||
if (passing.length > 1) {
|
||||
if (picked && nearTrip(item, picked)) return { tripId: picked.id, suggestedTripId: picked.id, reason: null };
|
||||
return { tripId: null, suggestedTripId: (picked ?? passing[0]).id, reason: 'trip_ambiguous' };
|
||||
}
|
||||
const suggested = trips.some((trip) => trip.id === aiTripId) ? aiTripId : null;
|
||||
const undatedOnly = trips.length > 0 && trips.every((trip) => !trip.start_date || !trip.end_date);
|
||||
return { tripId: null, suggestedTripId: suggested, reason: undatedOnly ? 'trip_without_dates' : 'no_trip_in_range' };
|
||||
}
|
||||
|
||||
/**
|
||||
* Every check on one booking, and the first reason it cannot be added on its
|
||||
* own. The trip is chosen even for a booking that fails, so the inbox can
|
||||
* suggest it.
|
||||
*/
|
||||
export function judgeBooking(draft: MailDraft, context: CheckContext): BookingVerdict {
|
||||
const { item } = draft;
|
||||
const choice = chooseTrip(item, draft.aiTripId, context);
|
||||
const fromAi = draft.source === 'ai';
|
||||
const inText = fromAi ? checkInText(item, context.documentText) : { ok: true, uncertain: [] };
|
||||
const checks: BookingVerdict['checks'] = {
|
||||
schema: bookingImportPreviewItemSchema.safeParse(item).success ? 'pass' : 'fail',
|
||||
trip: choice.reason === 'no_edit_permission' ? 'fail' : 'pass',
|
||||
dates: choice.tripId !== null || choice.reason === 'trip_ambiguous' ? 'pass' : 'fail',
|
||||
agreement: draft.disagrees ? 'fail' : fromAi ? 'pass' : 'skipped',
|
||||
plausible: implausible(item, context.now, context.knownAirport) ? 'fail' : 'pass',
|
||||
in_text: !fromAi ? 'skipped' : inText.ok && !draft.imageOnly ? 'pass' : 'fail',
|
||||
assignment: choice.tripId !== null ? 'pass' : 'fail',
|
||||
cancellation: isCancellation(item, draft.kind, context.subject) ? 'fail' : 'pass',
|
||||
};
|
||||
const uncertain = [...new Set([...draft.uncertainFields, ...inText.uncertain])];
|
||||
return { checks, reason: firstReason(draft, checks, choice), uncertain, choice };
|
||||
}
|
||||
|
||||
function firstReason(draft: MailDraft, checks: BookingVerdict['checks'], choice: TripChoice): MailIngestReason | null {
|
||||
if (draft.kind === 'change') return 'change_or_cancellation';
|
||||
if (checks.cancellation === 'fail') return draft.kind === 'cancellation' ? 'change_or_cancellation' : 'cancelled';
|
||||
if (checks.schema === 'fail' || missingCritical(draft.item)) return 'missing_critical_field';
|
||||
if (checks.plausible === 'fail') return 'implausible';
|
||||
if (checks.agreement === 'fail') return 'source_mismatch';
|
||||
if (checks.in_text === 'fail') return draft.imageOnly ? 'image_only' : 'not_in_text';
|
||||
if (missingReference(draft.item)) return 'missing_confirmation';
|
||||
return choice.reason;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the booking says, as one hash: two mails about the same booking with
|
||||
* the same content give the same hash, and a different time or place a
|
||||
* different one.
|
||||
*/
|
||||
export function contentHash(item: BookingImportPreviewItem): string {
|
||||
const fingerprint = {
|
||||
type: item.type,
|
||||
title: item.title,
|
||||
start: item.reservation_time ?? null,
|
||||
end: item.reservation_end_time ?? null,
|
||||
reference: normalizeReference(item.confirmation_number),
|
||||
location: item.location ?? null,
|
||||
stay: item._accommodation ? [item._accommodation.check_in ?? null, item._accommodation.check_out ?? null] : null,
|
||||
endpoints: (item.endpoints ?? []).map((endpoint) => [endpoint.code ?? null, endpoint.name]),
|
||||
};
|
||||
return createHash('sha256').update(JSON.stringify(fingerprint)).digest('hex');
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import type { MailIngestAddressRow } from '../../db/repositories/MailIngestAddresses.repository';
|
||||
import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { MailReadService } from '../llm-parse/mail-read.service';
|
||||
import { MailIngestAddressService } from './mail-ingest-address.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { sealMail } from './mail-seal';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { MailIngestTransport } from '@trek/shared';
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
/** Why a mail was not taken in. A transport answers each the way its protocol does (a 550, a 404, a 413). */
|
||||
export type MailIntakeRefusal =
|
||||
| 'unavailable'
|
||||
| 'too_large'
|
||||
| 'not_a_mail'
|
||||
| 'unknown_address'
|
||||
| 'address_daily_limit'
|
||||
| 'inbox_full';
|
||||
|
||||
export type MailIntakeResult =
|
||||
| { status: 'accepted' | 'duplicate'; messageId: number }
|
||||
| { status: 'refused'; reason: MailIntakeRefusal };
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Where every mail comes in, whatever carried it: an IMAP box, the signed
|
||||
* webhook, the TREK Cloud relay, or an .eml the person uploaded. It only
|
||||
* stores the mail, encrypted and queued, once per address and Message-ID,
|
||||
* and answers at once; the processing job reads it later. So a transport can
|
||||
* acknowledge as soon as this returns, and a mail delivered twice is read once.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestIntakeService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly addresses: MailIngestAddressService,
|
||||
private readonly reader: MailReadService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* A mail a transport received. `envelopeRecipients` are the RCPT TO
|
||||
* addresses where the transport knows them; otherwise the headers name the
|
||||
* address, in the order the receiving server adds them.
|
||||
*/
|
||||
async receive(
|
||||
raw: Buffer,
|
||||
transport: Exclude<MailIngestTransport, 'upload'>,
|
||||
envelopeRecipients: readonly string[] = [],
|
||||
): Promise<MailIntakeResult> {
|
||||
const refused = await this.precheck(raw);
|
||||
if (refused) return { status: 'refused', reason: refused };
|
||||
const mail = this.reader.walk(raw);
|
||||
if (!mail) return { status: 'refused', reason: 'not_a_mail' };
|
||||
const { deliveredTo, xOriginalTo, to } = mail.envelope;
|
||||
const headerRecipients = [deliveredTo, xOriginalTo, ...(to ?? '').split(',')].filter(
|
||||
(value): value is string => !!value && value.trim() !== '',
|
||||
);
|
||||
const resolved = await this.addresses.resolve([...envelopeRecipients, ...headerRecipients]);
|
||||
if (!resolved) return { status: 'refused', reason: 'unknown_address' };
|
||||
return this.store(raw, transport, resolved.address, mail.envelope.messageId);
|
||||
}
|
||||
|
||||
/** An .eml the person uploaded, read as if they had forwarded it to their own address. */
|
||||
async upload(raw: Buffer, userId: number): Promise<MailIntakeResult> {
|
||||
const refused = await this.precheck(raw);
|
||||
if (refused) return { status: 'refused', reason: refused };
|
||||
const mail = this.reader.walk(raw);
|
||||
if (!mail) return { status: 'refused', reason: 'not_a_mail' };
|
||||
const address = await this.addresses.ensureOwn(userId);
|
||||
return this.store(raw, 'upload', address, mail.envelope.messageId);
|
||||
}
|
||||
|
||||
private async precheck(raw: Buffer): Promise<MailIntakeRefusal | null> {
|
||||
if (!(await this.settings.isAvailable())) return 'unavailable';
|
||||
if (raw.length > this.settings.maxMailBytes()) return 'too_large';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** @txIndependent the mail is stored by itself; noting the address's last use stands on its own. */
|
||||
private async store(
|
||||
raw: Buffer,
|
||||
transport: MailIngestTransport,
|
||||
address: MailIngestAddressRow,
|
||||
messageIdHdr: string | null,
|
||||
): Promise<MailIntakeResult> {
|
||||
const dedupeHash = createHash('sha256')
|
||||
.update(`${address.id}\n${messageIdHdr ?? createHash('sha256').update(raw).digest('hex')}`)
|
||||
.digest('hex');
|
||||
const known = await this.messages.findIdByDedupeHash(dedupeHash);
|
||||
if (known !== null) return { status: 'duplicate', messageId: known };
|
||||
|
||||
const now = new Date();
|
||||
const today = await this.messages.countForAddressSince(address.id, dbNow(new Date(now.getTime() - DAY_MS)));
|
||||
if (today >= MAIL_INGEST_LIMITS.mailsPerAddressPerDay) return { status: 'refused', reason: 'address_daily_limit' };
|
||||
if ((await this.candidates.countPending(address.user_id)) >= MAIL_INGEST_LIMITS.pendingPerUser) {
|
||||
return { status: 'refused', reason: 'inbox_full' };
|
||||
}
|
||||
|
||||
const id = await this.messages.insertIfNew({
|
||||
transport,
|
||||
messageIdHdr: messageIdHdr?.slice(0, 500) ?? null,
|
||||
dedupeHash,
|
||||
addressId: address.id,
|
||||
userId: address.user_id,
|
||||
tripId: address.trip_id,
|
||||
size: raw.length,
|
||||
rawEnc: sealMail(raw),
|
||||
receivedAt: dbNow(now),
|
||||
});
|
||||
// Another delivery of the same mail won the insert in between: it is in, once.
|
||||
if (id === null) {
|
||||
const winner = await this.messages.findIdByDedupeHash(dedupeHash);
|
||||
return winner === null ? { status: 'refused', reason: 'not_a_mail' } : { status: 'duplicate', messageId: winner };
|
||||
}
|
||||
await this.addresses.touch(address.id);
|
||||
return { status: 'accepted', messageId: id };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { NotificationsService } from '../notifications/notifications.service';
|
||||
import { MailIngestActionsService } from './mail-ingest-actions.service';
|
||||
import { KEEP_ACTION, UNDO_ACTION } from './mail-ingest.constants';
|
||||
import { Injectable, type OnModuleInit } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* What the forwarder's notification buttons do. Undo takes the booking back
|
||||
* out unless its cost moved balances or it was edited since, which needs the
|
||||
* inbox's own confirmation; keep only answers the question. The handlers run
|
||||
* in the request that answered the notification, under its context.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestNotificationActions implements OnModuleInit {
|
||||
constructor(
|
||||
private readonly notifications: NotificationsService,
|
||||
private readonly actions: MailIngestActionsService,
|
||||
) {}
|
||||
|
||||
onModuleInit(): void {
|
||||
this.notifications.registerResponseAction(UNDO_ACTION, (payload, userId) => this.undo(payload, userId));
|
||||
this.notifications.registerResponseAction(KEEP_ACTION, async () => undefined);
|
||||
}
|
||||
|
||||
async undo(payload: Record<string, unknown>, userId: number): Promise<void> {
|
||||
const candidateId = Number(payload.candidateId);
|
||||
if (!Number.isInteger(candidateId) || candidateId <= 0) throw new Error('This notification names no booking');
|
||||
await this.actions.undo(candidateId, userId, false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import { NotificationsService } from '../notifications/notifications.service';
|
||||
import { KEEP_ACTION, UNDO_ACTION } from './mail-ingest.constants';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/** A booking a mail added, as its notifications name it. */
|
||||
export interface AddedBooking {
|
||||
candidateId: number;
|
||||
tripId: number;
|
||||
tripTitle: string;
|
||||
title: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* What the mail import tells people. The forwarder hears of every booking it
|
||||
* added, with keep and undo; the other members of a shared trip hear that a
|
||||
* booking came in (owner decision 13.11.2026); and a mail whose bookings wait
|
||||
* in the inbox sends one note for the lot. A notification never fails the
|
||||
* write it reports.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestNotifyService {
|
||||
constructor(private readonly notifications: NotificationsService) {}
|
||||
|
||||
/** @txStandalone a notice reports a write that committed; a failed one undoes nothing. */
|
||||
async bookingAdded(forwarder: MailIngestPersonRow, booking: AddedBooking): Promise<void> {
|
||||
const params = {
|
||||
booking: booking.title,
|
||||
type: booking.type,
|
||||
trip: booking.tripTitle,
|
||||
tripId: String(booking.tripId),
|
||||
actor: forwarder.username,
|
||||
};
|
||||
await this.quietly(
|
||||
this.notifications.send({
|
||||
event: 'mail_booking_added',
|
||||
actorId: null,
|
||||
scope: 'user',
|
||||
targetId: forwarder.id,
|
||||
params: { ...params, role: 'forwarder' },
|
||||
inApp: {
|
||||
type: 'boolean',
|
||||
positiveTextKey: 'notif.action.keep',
|
||||
negativeTextKey: 'notif.action.undo',
|
||||
positiveCallback: { action: KEEP_ACTION, payload: { candidateId: booking.candidateId } },
|
||||
negativeCallback: { action: UNDO_ACTION, payload: { candidateId: booking.candidateId } },
|
||||
},
|
||||
}),
|
||||
);
|
||||
// Trip scope reaches every member but the actor, who heard above.
|
||||
await this.quietly(
|
||||
this.notifications.send({
|
||||
event: 'mail_booking_added',
|
||||
actorId: forwarder.id,
|
||||
scope: 'trip',
|
||||
targetId: booking.tripId,
|
||||
params: { ...params, role: 'traveller' },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** @txStandalone one note for the bookings of a mail that wait in the inbox. */
|
||||
async bookingsPending(userId: number, pending: number): Promise<void> {
|
||||
await this.quietly(
|
||||
this.notifications.send({
|
||||
event: 'mail_booking_pending',
|
||||
actorId: null,
|
||||
scope: 'user',
|
||||
targetId: userId,
|
||||
params: { count: String(pending) },
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
private async quietly(sent: Promise<unknown>): Promise<void> {
|
||||
try {
|
||||
await sent;
|
||||
} catch (err) {
|
||||
logError(`[mail-ingest] A notification failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import type { MailIngestCursorRepository } from '../../db/repositories/MailIngestCursor.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import { withRequestContext } from '../database/request-context';
|
||||
import { CronRegistrarService, LEASE_OWNER } from '../scheduling/cron-registrar.service';
|
||||
import { MailIngestProcessService } from './mail-ingest-process.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
/** How long a run holds the pipeline and its mails before another run may take them over. */
|
||||
const LEASE_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Reads the queued mails every two minutes, at most three per run, one after
|
||||
* the other. The pipeline's own lease on its cursor row keeps two processes
|
||||
* on one database (or a run and an upload that asked for one now) from
|
||||
* reading at once; each mail has its claim as well, so a run that died hands
|
||||
* its mail on when the claim runs out. Every mail is read in a request
|
||||
* context of its own, so one mail's identity map never leaks into the next.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestProcessJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
@InjectRepository(MailIngestCursor) private readonly cursor: MailIngestCursorRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly process: MailIngestProcessService,
|
||||
private readonly registrar: CronRegistrarService,
|
||||
private readonly orm: MikroORM,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('mail-ingest-process', '*/2 * * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* One run; answers how many mails it read. Nothing happens while the import
|
||||
* is off, which is how a plan without it costs nothing.
|
||||
*
|
||||
* @txIndependent every mail is read and written on its own: one that fails
|
||||
* leaves the others, and its retry is its own claim's business.
|
||||
*/
|
||||
async tick(now: Date = new Date()): Promise<number> {
|
||||
if (!(await this.settings.isAvailable())) return 0;
|
||||
const until = dbNow(new Date(now.getTime() + LEASE_MS));
|
||||
if (!(await this.cursor.acquireLease('pipeline', '', LEASE_OWNER, until, dbNow(now)))) return 0;
|
||||
try {
|
||||
const ids = await this.messages.claimDue(dbNow(now), until, MAIL_INGEST_LIMITS.mailsPerRun);
|
||||
await ids.reduce(async (previous, id) => {
|
||||
await previous;
|
||||
await withRequestContext(this.orm, () => this.process.process(id, now));
|
||||
}, Promise.resolve());
|
||||
await this.cursor.recordOk('pipeline', '', dbNow(new Date()));
|
||||
return ids.length;
|
||||
} catch (err) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
logError(`[mail-ingest] A processing run failed: ${reason}`);
|
||||
await this.cursor.recordError('pipeline', '', reason);
|
||||
return 0;
|
||||
} finally {
|
||||
await this.cursor.releaseLease('pipeline', '', LEASE_OWNER);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import {
|
||||
MAIL_INGEST_ATTACHMENTS,
|
||||
MAIL_INGEST_CHECKS,
|
||||
MAIL_INGEST_CREATED,
|
||||
MAIL_INGEST_HINTS,
|
||||
MAIL_INGEST_ITEM,
|
||||
MAIL_INGEST_MATCH,
|
||||
MAIL_INGEST_UNCERTAIN_FIELDS,
|
||||
} from '../../db/json-columns';
|
||||
import type {
|
||||
MailIngestCandidatesRepository,
|
||||
MailIngestPersonRow,
|
||||
} from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type {
|
||||
MailIngestMessageOutcome,
|
||||
MailIngestMessageRow,
|
||||
MailIngestMessagesRepository,
|
||||
} from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { encodeJson } from '../../utils/json-column';
|
||||
import { AirportsService } from '../airports/airports.service';
|
||||
import { logError } from '../audit/audit-log.logger';
|
||||
import { DemoService } from '../common/demo.service';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { MailReadService } from '../llm-parse/mail-read.service';
|
||||
import type { WalkedMail } from '../llm-parse/mail-read.types';
|
||||
import { classifyMail, senderVerified } from './mail-classify';
|
||||
import { MailIngestApplyService, type PreparedBooking, type WrittenBooking } from './mail-ingest-apply.service';
|
||||
import {
|
||||
bookingStart,
|
||||
contentHash,
|
||||
judgeBooking,
|
||||
normalizeReference,
|
||||
type BookingVerdict,
|
||||
type CheckContext,
|
||||
type MailDraft,
|
||||
type TripOption,
|
||||
} from './mail-ingest-checks';
|
||||
import { MailIngestAddressService } from './mail-ingest-address.service';
|
||||
import { MailIngestNotifyService } from './mail-ingest-notify.service';
|
||||
import { MailIngestReadingService } from './mail-ingest-reading.service';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { IMPORT_SOURCES, MAIL_INGEST_LIMITS, UNDO_WINDOW_DAYS } from './mail-ingest.constants';
|
||||
import { openMail } from './mail-seal';
|
||||
import { UniqueConstraintViolationException } from '@mikro-orm/core';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { MailIngestHint, MailIngestPrefs, MailIngestReason, MailIngestResolution } from '@trek/shared';
|
||||
|
||||
/** One booking of a mail with the decision on it. */
|
||||
interface PlannedBooking {
|
||||
position: number;
|
||||
draft: MailDraft;
|
||||
verdict: BookingVerdict;
|
||||
key: string;
|
||||
hash: string;
|
||||
resolution: MailIngestResolution;
|
||||
reason: MailIngestReason | null;
|
||||
/** Set for a booking that is added on its own, with the trip it goes to. */
|
||||
prepared: PreparedBooking | null;
|
||||
trip: TripOption | null;
|
||||
}
|
||||
|
||||
/** A booking the write added, with its entry, for what follows the commit. */
|
||||
interface AddedEntry {
|
||||
candidateId: number;
|
||||
trip: TripOption;
|
||||
written: WrittenBooking;
|
||||
}
|
||||
|
||||
const MINUTE_MS = 60 * 1000;
|
||||
const DAY_MS = 24 * 60 * MINUTE_MS;
|
||||
/** How long a failed mail waits per try before the next. */
|
||||
const RETRY_STEP_MS = 5 * MINUTE_MS;
|
||||
/** A booking an hour either side of one in the trip is the same booking, as far as adding it on its own goes. */
|
||||
const SIMILAR_WINDOW_MS = 60 * MINUTE_MS;
|
||||
|
||||
/**
|
||||
* Reads one forwarded mail to its end (plan section 5). It filters what is no
|
||||
* booking without AI, reads the bookings (machine-readable data always, the
|
||||
* AI where allowed), checks each one, adds those that pass to their trip in
|
||||
* one transaction with the mail's own status, and keeps the rest in the
|
||||
* inbox with a reason. A mail is hostile: it can pick neither a person, nor a
|
||||
* trip outside the person's own list, nor anything but adding a booking.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestProcessService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly addresses: MailIngestAddressService,
|
||||
private readonly trips: MailIngestTripsService,
|
||||
private readonly reading: MailIngestReadingService,
|
||||
private readonly apply: MailIngestApplyService,
|
||||
private readonly notify: MailIngestNotifyService,
|
||||
private readonly reader: MailReadService,
|
||||
private readonly airports: AirportsService,
|
||||
private readonly demo: DemoService,
|
||||
private readonly uow: UnitOfWork,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reads a mail the caller claimed. A failure is not the caller's: the mail
|
||||
* is tried again later, and given up after its last try.
|
||||
*
|
||||
* @txIndependent the mail's own write is one transaction; the retry note on
|
||||
* a failure, and what follows the commit, stand on their own.
|
||||
*/
|
||||
async process(messageId: number, now: Date = new Date()): Promise<void> {
|
||||
const message = await this.messages.findById(messageId);
|
||||
if (!message || message.status !== 'processing') return;
|
||||
try {
|
||||
await this.read(message, now);
|
||||
} catch (err) {
|
||||
await this.failed(message, err, now);
|
||||
}
|
||||
}
|
||||
|
||||
/** @txIndependent each branch settles the mail in one write; the notices after it stand on their own. */
|
||||
private async read(message: MailIngestMessageRow, now: Date): Promise<void> {
|
||||
const raw = openMail(message.raw_eml_enc);
|
||||
const mail = raw ? this.reader.walk(raw) : null;
|
||||
const person = await this.trips.person(message.user_id);
|
||||
if (!raw || !mail || !person) {
|
||||
await this.keepForInbox(message, null, 'mail_unreadable', now);
|
||||
return;
|
||||
}
|
||||
const outcome = this.outcomeBase(mail, person, now);
|
||||
const address = message.address_id === null ? null : await this.addresses.findById(message.address_id);
|
||||
if (!address || !(await this.addresses.isLive(address))) {
|
||||
await this.messages.finish(message.id, { ...outcome, status: 'discarded', reason: 'address_invalid' });
|
||||
return;
|
||||
}
|
||||
const kind = classifyMail(mail);
|
||||
if (kind.kind === 'drop') {
|
||||
await this.messages.finish(message.id, { ...outcome, status: 'discarded', reason: kind.reason });
|
||||
return;
|
||||
}
|
||||
if (kind.kind === 'forwarding_confirmation') {
|
||||
await this.keepForInbox(message, { ...outcome, forwardingCode: kind.code }, 'forwarding_confirmation', now);
|
||||
await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id));
|
||||
return;
|
||||
}
|
||||
await this.readBookings(message, raw, mail, person, outcome, now);
|
||||
}
|
||||
|
||||
/**
|
||||
* @txIndependent the bookings and the mail's status are one transaction;
|
||||
* announcements, documents and notices follow it.
|
||||
*/
|
||||
private async readBookings(
|
||||
message: MailIngestMessageRow,
|
||||
raw: Buffer,
|
||||
mail: WalkedMail,
|
||||
person: MailIngestPersonRow,
|
||||
outcome: MailIngestMessageOutcome,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
const prefs = await this.settings.prefsOf(person.id);
|
||||
const machine = await this.reading.readMachine(raw, mail, person.id);
|
||||
const trips = await this.tripsFor(message, person, now, this.reading.hintDates(mail, machine));
|
||||
const read = await this.reading.read({ mail, machine, person, prefs, trips, now });
|
||||
const done = { ...outcome, aiUsed: read.aiUsed };
|
||||
if (read.drafts.length === 0) {
|
||||
if (read.emptyReason === 'not_a_booking') {
|
||||
await this.messages.finish(message.id, { ...done, status: 'discarded', reason: 'not_a_booking' });
|
||||
return;
|
||||
}
|
||||
await this.keepForInbox(message, done, read.emptyReason, now);
|
||||
await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id));
|
||||
return;
|
||||
}
|
||||
const context: CheckContext = {
|
||||
trips,
|
||||
addressTripId: message.trip_id,
|
||||
subject: mail.subject,
|
||||
documentText: read.documentText,
|
||||
now,
|
||||
knownAirport: (code) => this.airports.findByIata(code) !== null,
|
||||
};
|
||||
const planned = await this.plan(read.drafts, context, person, prefs);
|
||||
const added = await this.write(message, planned, done, now);
|
||||
await this.afterCommit(added, planned, mail, person);
|
||||
}
|
||||
|
||||
/** The trips a booking of this mail may go to: the address's trip alone, or the person's editable ones. */
|
||||
private async tripsFor(
|
||||
message: MailIngestMessageRow,
|
||||
person: MailIngestPersonRow,
|
||||
now: Date,
|
||||
hintDates: string[],
|
||||
): Promise<TripOption[]> {
|
||||
if (message.trip_id === null) return this.trips.editableTrips(person, now, hintDates);
|
||||
const trip = await this.trips.editableTrip(person, message.trip_id);
|
||||
return trip ? [trip] : [];
|
||||
}
|
||||
|
||||
/** Every booking with its checks, the checks that read its trip and, for one added on its own, its network half. */
|
||||
private async plan(
|
||||
drafts: readonly MailDraft[],
|
||||
context: CheckContext,
|
||||
person: MailIngestPersonRow,
|
||||
prefs: MailIngestPrefs,
|
||||
): Promise<PlannedBooking[]> {
|
||||
const tooMany = drafts.length > MAIL_INGEST_LIMITS.autoBookings;
|
||||
const demo = await this.demo.isDemoUserId(person.id);
|
||||
const judged = await Promise.all(drafts.map((draft, position) => this.judge(draft, position, context)));
|
||||
const seen = new Set<string>();
|
||||
for (const booking of judged) {
|
||||
const tripKey = `${booking.verdict.choice.tripId}|${booking.key}`;
|
||||
if (booking.resolution === 'pending' && booking.reason === null && seen.has(tripKey)) {
|
||||
booking.resolution = 'duplicate';
|
||||
}
|
||||
seen.add(tripKey);
|
||||
if (booking.resolution !== 'pending' || booking.reason !== null) continue;
|
||||
if (tooMany) booking.reason = 'too_many_bookings';
|
||||
else if (demo) booking.reason = 'demo_mode';
|
||||
else if (!prefs.auto_add) booking.reason = 'ask_first';
|
||||
}
|
||||
return judged.reduce<Promise<PlannedBooking[]>>(async (previous, booking) => {
|
||||
const done = await previous;
|
||||
const trip = context.trips.find((candidate) => candidate.id === booking.verdict.choice.tripId);
|
||||
if (booking.resolution === 'pending' && booking.reason === null && trip) {
|
||||
booking.prepared = await this.apply.prepare(booking.draft.item, trip, person.id, booking.verdict.uncertain);
|
||||
booking.trip = trip;
|
||||
}
|
||||
return [...done, booking];
|
||||
}, Promise.resolve([]));
|
||||
}
|
||||
|
||||
/** One booking's checks, and the checks that read the trip it would go to (G9, an undone twin, a look-alike). */
|
||||
private async judge(draft: MailDraft, position: number, context: CheckContext): Promise<PlannedBooking> {
|
||||
const verdict = judgeBooking(draft, context);
|
||||
const planned: PlannedBooking = {
|
||||
position,
|
||||
draft,
|
||||
verdict,
|
||||
key: this.apply.externalKey(draft.item),
|
||||
hash: contentHash(draft.item),
|
||||
resolution: 'pending',
|
||||
reason: verdict.reason,
|
||||
prepared: null,
|
||||
trip: null,
|
||||
};
|
||||
const tripId = verdict.choice.tripId;
|
||||
if (tripId === null || verdict.reason !== null) return planned;
|
||||
if (await this.candidates.tripHasExternal(tripId, IMPORT_SOURCES, planned.key)) {
|
||||
verdict.checks.duplicate = 'fail';
|
||||
return { ...planned, resolution: 'duplicate' };
|
||||
}
|
||||
verdict.checks.duplicate = 'pass';
|
||||
if ((await this.candidates.wasUndoneInTrip(tripId, planned.hash)) || (await this.looksAlike(tripId, draft))) {
|
||||
return { ...planned, reason: 'possible_duplicate' };
|
||||
}
|
||||
return planned;
|
||||
}
|
||||
|
||||
private async looksAlike(tripId: number, draft: MailDraft): Promise<boolean> {
|
||||
const start = Date.parse(bookingStart(draft.item) ?? '');
|
||||
const window = (offset: number) =>
|
||||
Number.isFinite(start) ? new Date(start + offset).toISOString().slice(0, 16) : null;
|
||||
return this.candidates.tripHasSimilar(tripId, {
|
||||
type: draft.item.type,
|
||||
startFrom: window(-SIMILAR_WINDOW_MS),
|
||||
startTo: window(SIMILAR_WINDOW_MS),
|
||||
confirmation: normalizeReference(draft.item.confirmation_number) || null,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One transaction per mail: every entry, the bookings added on their own
|
||||
* (each in a savepoint, so one the unique index refuses is a duplicate and
|
||||
* the others still land), and the mail's own status. A crash before the
|
||||
* commit leaves the mail to be read again, never half added.
|
||||
*/
|
||||
private async write(
|
||||
message: MailIngestMessageRow,
|
||||
planned: readonly PlannedBooking[],
|
||||
outcome: MailIngestMessageOutcome,
|
||||
now: Date,
|
||||
): Promise<AddedEntry[]> {
|
||||
const createdAt = dbNow(now);
|
||||
const undoUntil = dbNow(new Date(now.getTime() + UNDO_WINDOW_DAYS * DAY_MS));
|
||||
return this.uow.transactional(async () => {
|
||||
const added = await planned.reduce<Promise<AddedEntry[]>>(async (previous, booking) => {
|
||||
const done = await previous;
|
||||
const written = booking.prepared ? await this.tryWrite(booking) : null;
|
||||
const candidateId = await this.candidates.insertCandidate({
|
||||
messageId: message.id,
|
||||
userId: message.user_id,
|
||||
position: booking.position,
|
||||
itemJson: encodeJson(MAIL_INGEST_ITEM, booking.draft.item),
|
||||
source: booking.draft.source,
|
||||
checksJson: encodeJson(MAIL_INGEST_CHECKS, booking.verdict.checks),
|
||||
uncertainFields: encodeJson(
|
||||
MAIL_INGEST_UNCERTAIN_FIELDS,
|
||||
booking.prepared?.uncertain ?? booking.verdict.uncertain,
|
||||
),
|
||||
matchJson: encodeJson(MAIL_INGEST_MATCH, {
|
||||
kind: booking.draft.kind,
|
||||
suggested_new_trip: booking.draft.suggestedNewTrip,
|
||||
}),
|
||||
suggestedTripId: booking.verdict.choice.suggestedTripId,
|
||||
dedupeKey: booking.key,
|
||||
contentHash: booking.hash,
|
||||
resolution: written ? 'auto_applied' : booking.prepared ? 'duplicate' : booking.resolution,
|
||||
reason: written || booking.prepared ? null : booking.reason,
|
||||
createdAt,
|
||||
...(written && {
|
||||
resolvedTripId: written.tripId,
|
||||
createdIdsJson: encodeJson(MAIL_INGEST_CREATED, written.created),
|
||||
resolvedAt: createdAt,
|
||||
undoUntil,
|
||||
}),
|
||||
});
|
||||
return written && booking.trip ? [...done, { candidateId, trip: booking.trip, written }] : done;
|
||||
}, Promise.resolve([]));
|
||||
await this.messages.finish(message.id, { ...outcome, status: 'processed', reason: null });
|
||||
return added;
|
||||
});
|
||||
}
|
||||
|
||||
/** A savepoint around one booking: the unique index refusing it makes it a duplicate, not a failed mail. */
|
||||
private async tryWrite(booking: PlannedBooking): Promise<WrittenBooking | null> {
|
||||
const prepared = booking.prepared;
|
||||
if (!prepared) return null;
|
||||
try {
|
||||
return await this.uow.transactional(() => this.apply.writeInTx(prepared));
|
||||
} catch (err) {
|
||||
if (err instanceof UniqueConstraintViolationException) return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* After the commit: what the trips are told, the mail's documents on the
|
||||
* first booking of each trip, and the notifications.
|
||||
*
|
||||
* @txIndependent each step stands on its own after the commit: a document
|
||||
* that fails to attach leaves the booking, and is noted on its entry.
|
||||
*/
|
||||
private async afterCommit(
|
||||
added: readonly AddedEntry[],
|
||||
planned: readonly PlannedBooking[],
|
||||
mail: WalkedMail,
|
||||
person: MailIngestPersonRow,
|
||||
): Promise<void> {
|
||||
const attachedTrips = new Set<number>();
|
||||
await added.reduce(async (previous, entry) => {
|
||||
await previous;
|
||||
await this.apply.announce(entry.written);
|
||||
if (!attachedTrips.has(entry.written.tripId)) {
|
||||
attachedTrips.add(entry.written.tripId);
|
||||
const { fileIds, error } = await this.apply.attachDocuments(
|
||||
entry.written.tripId,
|
||||
entry.written.reservation.id,
|
||||
mail,
|
||||
person.id,
|
||||
);
|
||||
const created = { ...entry.written.created, file_ids: fileIds };
|
||||
await this.candidates.setAttachments(entry.candidateId, encodeJson(MAIL_INGEST_CREATED, created), error);
|
||||
}
|
||||
await this.notify.bookingAdded(person, {
|
||||
candidateId: entry.candidateId,
|
||||
tripId: entry.written.tripId,
|
||||
tripTitle: entry.trip.title,
|
||||
title: entry.written.reservation.title,
|
||||
type: entry.written.reservation.type,
|
||||
});
|
||||
}, Promise.resolve());
|
||||
if (planned.some((booking) => booking.resolution === 'pending' && booking.prepared === null)) {
|
||||
await this.notify.bookingsPending(person.id, await this.candidates.countPending(person.id));
|
||||
}
|
||||
}
|
||||
|
||||
/** The entry standing for a mail that gave no booking, with the reason, so the ways out still apply. */
|
||||
private async keepForInbox(
|
||||
message: MailIngestMessageRow,
|
||||
outcome: MailIngestMessageOutcome | null,
|
||||
reason: MailIngestReason | null,
|
||||
now: Date,
|
||||
): Promise<void> {
|
||||
const createdAt = dbNow(now);
|
||||
await this.uow.transactional(async () => {
|
||||
await this.candidates.insertCandidate({
|
||||
messageId: message.id,
|
||||
userId: message.user_id,
|
||||
position: 0,
|
||||
itemJson: null,
|
||||
source: null,
|
||||
checksJson: null,
|
||||
uncertainFields: null,
|
||||
matchJson: null,
|
||||
suggestedTripId: message.trip_id,
|
||||
dedupeKey: null,
|
||||
contentHash: null,
|
||||
resolution: 'pending',
|
||||
reason,
|
||||
createdAt,
|
||||
});
|
||||
await this.messages.finish(message.id, {
|
||||
...(outcome ?? emptyOutcome(createdAt)),
|
||||
status: 'processed',
|
||||
reason,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** What the mail says about itself, before any booking in it is read. */
|
||||
private outcomeBase(mail: WalkedMail, person: MailIngestPersonRow, now: Date): MailIngestMessageOutcome {
|
||||
const hints: MailIngestHint[] = [];
|
||||
if (!senderVerified(mail, person.email)) hints.push('sender_unverified');
|
||||
if (mail.skippedAttachments > 0) hints.push('attachments_skipped');
|
||||
return {
|
||||
status: 'processed',
|
||||
reason: null,
|
||||
subject: mail.subject?.slice(0, 300) ?? null,
|
||||
fromAddr: mail.envelope.from?.slice(0, 320) ?? null,
|
||||
fromVerified: !hints.includes('sender_unverified'),
|
||||
authResults: mail.envelope.authenticationResults?.slice(0, 2000) ?? null,
|
||||
hints: encodeJson(MAIL_INGEST_HINTS, hints),
|
||||
attachments: encodeJson(
|
||||
MAIL_INGEST_ATTACHMENTS,
|
||||
mail.attachments.map((file) => ({ file_name: file.fileName, kind: file.kind, size: file.data.length })),
|
||||
),
|
||||
forwardingCode: null,
|
||||
aiUsed: false,
|
||||
processedAt: dbNow(now),
|
||||
};
|
||||
}
|
||||
|
||||
/** A try that failed: again later, or given up after the last one. The mail's content stays out of the log. */
|
||||
private async failed(message: MailIngestMessageRow, err: unknown, now: Date): Promise<void> {
|
||||
logError(`[mail-ingest] Reading mail ${message.id} failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
if (message.attempts < MAIL_INGEST_LIMITS.maxAttempts) {
|
||||
await this.messages.retryLater(message.id, dbNow(new Date(now.getTime() + RETRY_STEP_MS * message.attempts)));
|
||||
return;
|
||||
}
|
||||
await this.messages.finish(message.id, {
|
||||
...emptyOutcome(dbNow(now)),
|
||||
status: 'failed',
|
||||
reason: 'processing_failed',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function emptyOutcome(at: MailIngestMessageOutcome['processedAt']): MailIngestMessageOutcome {
|
||||
return {
|
||||
status: 'processed',
|
||||
reason: null,
|
||||
subject: null,
|
||||
fromAddr: null,
|
||||
fromVerified: false,
|
||||
authResults: null,
|
||||
hints: null,
|
||||
attachments: null,
|
||||
forwardingCode: null,
|
||||
aiUsed: false,
|
||||
processedAt: at,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import type { MailIngestPersonRow } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { BookingImportService } from '../booking-import/booking-import.service';
|
||||
import { MailReadService } from '../llm-parse/mail-read.service';
|
||||
import type { MailReadBooking, MailReadTrip, WalkedMail } from '../llm-parse/mail-read.types';
|
||||
import { bookingStart, dateOf, normalizeReference, type MailDraft, type TripOption } from './mail-ingest-checks';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
MAIL_INGEST_REASON_BY_LLM_OUTCOME,
|
||||
type BookingImportPreviewItem,
|
||||
type LlmParseOutcome,
|
||||
type MailIngestPrefs,
|
||||
type MailIngestReason,
|
||||
} from '@trek/shared';
|
||||
|
||||
/** What reading a mail gave. */
|
||||
export interface MailReading {
|
||||
drafts: MailDraft[];
|
||||
/** Whether a request went to the AI, which counts against the person's daily limit. */
|
||||
aiUsed: boolean;
|
||||
/** Why the mail gave no booking; null when it gave some. */
|
||||
emptyReason: MailIngestReason | null;
|
||||
/** Everything the mail and its attachments say, for the in-text check. */
|
||||
documentText: string;
|
||||
}
|
||||
|
||||
/** The outcomes of a reading that never reached a model. */
|
||||
const NOT_SENT: ReadonlySet<LlmParseOutcome> = new Set(['ai_disabled', 'no_text', 'user_cap']);
|
||||
const ISO_DATE_RE = /\b(20\d\d-[01]\d-[0-3]\d)\b/g;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Reads the bookings out of a forwarded mail. The machine-readable parts
|
||||
* (JSON-LD, calendar files, passes, through kitinerary) are read always and
|
||||
* cost nothing. The AI reads the mail when the person allows it and is under
|
||||
* their daily limit, and picks a trip from the list it is given; where both
|
||||
* name a booking, the machine-readable values win, and a critical field they
|
||||
* disagree on sends the booking to the inbox (G4).
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestReadingService {
|
||||
constructor(
|
||||
private readonly bookingImport: BookingImportService,
|
||||
private readonly reader: MailReadService,
|
||||
private readonly settings: MailIngestSettingsService,
|
||||
private readonly trips: MailIngestTripsService,
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
) {}
|
||||
|
||||
/** The bookings the machine-readable parts of the mail name. */
|
||||
async readMachine(raw: Buffer, mail: WalkedMail, userId: number): Promise<BookingImportPreviewItem[]> {
|
||||
const attachments = mail.attachments.map(({ fileName, data }) => ({ fileName, data }));
|
||||
return this.bookingImport.readMailBookings(raw, attachments, userId);
|
||||
}
|
||||
|
||||
/** The dates the mail names: they rank the trips the AI is offered. */
|
||||
hintDates(mail: WalkedMail, machine: readonly BookingImportPreviewItem[]): string[] {
|
||||
const named = [...`${mail.subject ?? ''}\n${mail.body}`.matchAll(ISO_DATE_RE)].map((match) => match[1]);
|
||||
const read = machine.map((item) => dateOf(bookingStart(item))).filter((date): date is string => date !== null);
|
||||
return [...new Set([...read, ...named])];
|
||||
}
|
||||
|
||||
async read(args: {
|
||||
mail: WalkedMail;
|
||||
machine: BookingImportPreviewItem[];
|
||||
person: MailIngestPersonRow;
|
||||
prefs: MailIngestPrefs;
|
||||
trips: readonly TripOption[];
|
||||
now: Date;
|
||||
}): Promise<MailReading> {
|
||||
const { mail, machine, person, prefs, trips, now } = args;
|
||||
const skipped = await this.aiSkipped(person.id, prefs, now);
|
||||
const refs = trips.map((trip, i) => ({ trip, ref: `t${i + 1}` }));
|
||||
const reading = skipped
|
||||
? null
|
||||
: await this.reader.read(
|
||||
{
|
||||
subject: mail.subject,
|
||||
date: mail.envelope.date,
|
||||
body: mail.body,
|
||||
attachments: mail.attachments,
|
||||
hints: machine.map(hintOf),
|
||||
trips: refs.map(({ trip, ref }) => this.tripForAi(trip, ref)),
|
||||
},
|
||||
person.id,
|
||||
now.toISOString().slice(0, 10),
|
||||
);
|
||||
const documentText = reading?.documentText ?? `${mail.subject ?? ''}\n${mail.body}`;
|
||||
const aiUsed = reading !== null && !NOT_SENT.has(reading.outcome);
|
||||
|
||||
if (reading && reading.bookings.length > 0) {
|
||||
const tripIdOf = new Map(refs.map(({ trip, ref }) => [ref, trip.id]));
|
||||
const drafts = await this.fromAi(reading.bookings, machine, tripIdOf, reading.imageOnly);
|
||||
return { drafts, aiUsed, emptyReason: null, documentText };
|
||||
}
|
||||
if (machine.length > 0) return { drafts: machine.map(fromMachine), aiUsed, emptyReason: null, documentText };
|
||||
const emptyReason = reading ? MAIL_INGEST_REASON_BY_LLM_OUTCOME[reading.outcome] : skipped;
|
||||
return { drafts: [], aiUsed, emptyReason: emptyReason ?? 'not_a_booking', documentText };
|
||||
}
|
||||
|
||||
/** Why the AI does not read this mail: switched off, not set up, or the person's daily limit is reached. */
|
||||
private async aiSkipped(userId: number, prefs: MailIngestPrefs, now: Date): Promise<MailIngestReason | null> {
|
||||
if (!prefs.ai_enabled || !(await this.reader.isAvailable(userId))) return 'ai_disabled';
|
||||
const since = dbNow(new Date(now.getTime() - DAY_MS));
|
||||
if ((await this.messages.countAiReadSince(userId, since)) >= (await this.settings.aiDailyCap())) {
|
||||
return 'ai_user_cap';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private tripForAi(trip: TripOption, ref: string): MailReadTrip {
|
||||
return {
|
||||
ref,
|
||||
title: trip.title.slice(0, 80),
|
||||
start_date: trip.start_date,
|
||||
end_date: trip.end_date,
|
||||
destinations: this.trips.destinationsOf(trip),
|
||||
};
|
||||
}
|
||||
|
||||
private async fromAi(
|
||||
bookings: readonly MailReadBooking[],
|
||||
machine: readonly BookingImportPreviewItem[],
|
||||
tripIdOf: ReadonlyMap<string, number>,
|
||||
imageOnly: boolean,
|
||||
): Promise<MailDraft[]> {
|
||||
const mapped = await Promise.all(
|
||||
bookings.map((booking) => this.bookingImport.mapAiReservations(booking.reservations, 'mail')),
|
||||
);
|
||||
return bookings
|
||||
.flatMap((booking, i) => mapped[i].map((item) => ({ booking, item })))
|
||||
.slice(0, MAIL_INGEST_LIMITS.storedBookings)
|
||||
.map(({ booking, item }) => {
|
||||
const match = machine.find((candidate) => sameBooking(candidate, item));
|
||||
const disagrees = match !== undefined && criticalDiffers(match, item);
|
||||
return {
|
||||
item: match && !disagrees ? match : item,
|
||||
source: 'ai' as const,
|
||||
kind: booking.kind,
|
||||
aiTripId: booking.tripRef === null ? null : (tripIdOf.get(booking.tripRef) ?? null),
|
||||
uncertainFields: booking.uncertainFields,
|
||||
suggestedNewTrip: booking.suggestedNewTrip,
|
||||
imageOnly: imageOnly && match === undefined,
|
||||
disagrees,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function fromMachine(item: BookingImportPreviewItem): MailDraft {
|
||||
return {
|
||||
item,
|
||||
source: 'kitinerary',
|
||||
kind: 'new_booking',
|
||||
aiTripId: null,
|
||||
uncertainFields: [],
|
||||
suggestedNewTrip: null,
|
||||
imageOnly: false,
|
||||
disagrees: false,
|
||||
};
|
||||
}
|
||||
|
||||
/** A booking as a hint for the AI: what it is, when, its reference and route, no more. */
|
||||
function hintOf(item: BookingImportPreviewItem): Record<string, unknown> {
|
||||
return {
|
||||
type: item.type,
|
||||
title: item.title,
|
||||
start: bookingStart(item),
|
||||
reference: item.confirmation_number ?? null,
|
||||
route: (item.endpoints ?? []).map((endpoint) => endpoint.code ?? endpoint.name),
|
||||
};
|
||||
}
|
||||
|
||||
/** The same booking read twice: one type and one reference, or one type starting on one day. */
|
||||
function sameBooking(a: BookingImportPreviewItem, b: BookingImportPreviewItem): boolean {
|
||||
if (a.type !== b.type) return false;
|
||||
const reference = normalizeReference(a.confirmation_number);
|
||||
if (reference && reference === normalizeReference(b.confirmation_number)) return true;
|
||||
const day = dateOf(bookingStart(a));
|
||||
return day !== null && day === dateOf(bookingStart(b));
|
||||
}
|
||||
|
||||
/** G4: a critical field both readings give, read differently: the start to the minute, the reference, the route. */
|
||||
function criticalDiffers(machine: BookingImportPreviewItem, ai: BookingImportPreviewItem): boolean {
|
||||
const differs = (x: string | null, y: string | null) => x !== null && y !== null && x !== y;
|
||||
const minute = (item: BookingImportPreviewItem) => bookingStart(item)?.slice(0, 16) ?? null;
|
||||
const reference = (item: BookingImportPreviewItem) => normalizeReference(item.confirmation_number) || null;
|
||||
const route = (item: BookingImportPreviewItem) => {
|
||||
const codes = (item.endpoints ?? []).map((endpoint) => endpoint.code).filter((code): code is string => !!code);
|
||||
return codes.length >= 2 ? `${codes[0]}-${codes[codes.length - 1]}`.toUpperCase() : null;
|
||||
};
|
||||
return (
|
||||
differs(minute(machine), minute(ai)) ||
|
||||
differs(reference(machine), reference(ai)) ||
|
||||
differs(route(machine), route(ai))
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import type { MailIngestCandidatesRepository } from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository';
|
||||
import { dbNow } from '../../db/types';
|
||||
import { logError, logInfo } from '../audit/audit-log.logger';
|
||||
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
/** A mail is kept this long after it was settled, for the ways out of the inbox and an undo. */
|
||||
const RAW_AFTER_DONE_DAYS = 7;
|
||||
/** And never longer than this after it came in. */
|
||||
const RAW_MAX_DAYS = 30;
|
||||
/** Decided entries, and mails without any, are kept this long, for the person's history and the admin's counts. */
|
||||
const DECIDED_DAYS = 90;
|
||||
|
||||
/**
|
||||
* The nightly sweep of the mail import (plan section 7): the stored mail goes
|
||||
* 7 days after it was settled and at the latest 30 days after it came in;
|
||||
* decided entries and the mails left without one go after 90 days. Waiting
|
||||
* entries stay until the person decides. Bookings the import added are
|
||||
* ordinary bookings and are not touched.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestRetentionJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository,
|
||||
@InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository,
|
||||
private readonly registrar: CronRegistrarService,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
this.registrar.register('mail-ingest-retention', '20 4 * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/** @txIndependent a retention sweep in passes, each complete on its own. */
|
||||
async tick(now: Date = new Date()): Promise<void> {
|
||||
const before = (days: number) => dbNow(new Date(now.getTime() - days * DAY_MS));
|
||||
try {
|
||||
const purged = await this.messages.purgeRaw(before(RAW_AFTER_DONE_DAYS), before(RAW_MAX_DAYS), dbNow(now));
|
||||
const entries = await this.candidates.deleteResolvedBefore(before(DECIDED_DAYS));
|
||||
const mails = await this.messages.deleteSettledBefore(before(DECIDED_DAYS));
|
||||
if (purged + entries + mails > 0) {
|
||||
logInfo(`Mail import retention: ${purged} mail(s) purged, ${entries} entr(ies) and ${mails} mail(s) removed`);
|
||||
}
|
||||
} catch (err) {
|
||||
logError(`Mail import retention: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity';
|
||||
import type { MailIngestPrefsPatch, MailIngestPrefsRepository } from '../../db/repositories/MailIngestPrefs.repository';
|
||||
import { RuntimeEnvService } from '../app-config/runtime-env.service';
|
||||
import { DEFAULT_ALLOWED_EXTENSIONS } from '../files/files.constants';
|
||||
import { SettingsService } from '../settings/settings.service';
|
||||
import { MAIL_INGEST_LIMITS } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
mailIngestTransportSettingSchema,
|
||||
type MailIngestPrefs as MailIngestPrefsView,
|
||||
type MailIngestPrefsUpdateRequest,
|
||||
type MailIngestTransportSetting,
|
||||
} from '@trek/shared';
|
||||
|
||||
/** The daily AI cap when the setting is missing or unreadable. */
|
||||
const DEFAULT_AI_PER_USER_DAILY = 10;
|
||||
|
||||
/**
|
||||
* Whether the mail import runs on this install, how mail reaches it, and the
|
||||
* person's own switches.
|
||||
*
|
||||
* The import is off until `mail_ingest_enabled` says otherwise. On a managed
|
||||
* install that setting is the operator's: it is written for the plans that
|
||||
* include the import and never for Start, so on Start every surface reports
|
||||
* the import as unavailable and nothing is read.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestSettingsService {
|
||||
constructor(
|
||||
private readonly settings: SettingsService,
|
||||
private readonly env: RuntimeEnvService,
|
||||
@InjectRepository(MailIngestPrefs) private readonly prefs: MailIngestPrefsRepository,
|
||||
) {}
|
||||
|
||||
async isAvailable(): Promise<boolean> {
|
||||
return (await this.settings.getAppSetting('mail_ingest_enabled')) === 'true';
|
||||
}
|
||||
|
||||
/** How mail reaches the install; a managed one takes it from the TREK Cloud relay alone. */
|
||||
async transport(): Promise<MailIngestTransportSetting> {
|
||||
if (this.env.isManaged()) return 'cloud';
|
||||
const stored = await this.settings.getAppSetting('mail_ingest_transport');
|
||||
const parsed = mailIngestTransportSettingSchema.safeParse(stored);
|
||||
return parsed.success ? parsed.data : 'off';
|
||||
}
|
||||
|
||||
/** The forwarding address with `{token}` where a person's token goes; null until an admin set one. */
|
||||
async addressTemplate(): Promise<string | null> {
|
||||
const template = (await this.settings.getAppSetting('mail_ingest_address_template'))?.trim();
|
||||
return template && template.includes('{token}') ? template : null;
|
||||
}
|
||||
|
||||
async aiDailyCap(): Promise<number> {
|
||||
const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily'));
|
||||
return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY;
|
||||
}
|
||||
|
||||
/** The largest mail taken in: more on the TREK Cloud, whose relay already capped it. */
|
||||
maxMailBytes(): number {
|
||||
return this.env.isManaged() ? MAIL_INGEST_LIMITS.cloudMailBytes : MAIL_INGEST_LIMITS.selfHostedMailBytes;
|
||||
}
|
||||
|
||||
/** The file types a trip takes, as the admin set them, for the documents a mail leaves behind. */
|
||||
async allowedFileTypes(): Promise<string> {
|
||||
return (await this.settings.getAppSetting('allowed_file_types')) || DEFAULT_ALLOWED_EXTENSIONS;
|
||||
}
|
||||
|
||||
/** The person's switches; a person who never changed one has every one on. */
|
||||
async prefsOf(userId: number): Promise<MailIngestPrefsView> {
|
||||
const row = await this.prefs.findForUser(userId);
|
||||
return {
|
||||
enabled: row ? row.enabled === 1 : true,
|
||||
ai_enabled: row ? row.ai_enabled === 1 : true,
|
||||
auto_add: row ? row.auto_add === 1 : true,
|
||||
};
|
||||
}
|
||||
|
||||
async updatePrefs(userId: number, update: MailIngestPrefsUpdateRequest): Promise<MailIngestPrefsView> {
|
||||
const patch: MailIngestPrefsPatch = {};
|
||||
if (update.enabled !== undefined) patch.enabled = update.enabled ? 1 : 0;
|
||||
if (update.ai_enabled !== undefined) patch.ai_enabled = update.ai_enabled ? 1 : 0;
|
||||
if (update.auto_add !== undefined) patch.auto_add = update.auto_add ? 1 : 0;
|
||||
await this.prefs.upsertForUser(userId, patch);
|
||||
return this.prefsOf(userId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import type {
|
||||
MailIngestCandidatesRepository,
|
||||
MailIngestPersonRow,
|
||||
MailIngestTripRow,
|
||||
} from '../../db/repositories/MailIngestCandidates.repository';
|
||||
import { PermissionsService } from '../permissions/permissions.service';
|
||||
import type { TripOption } from './mail-ingest-checks';
|
||||
import { MAX_TRIP_CANDIDATES, TRIP_LOOKBACK_DAYS } from './mail-ingest.constants';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/** Open trips read before the permission filter, so trips a person may not edit cannot crowd out the rest. */
|
||||
const TRIPS_READ = 60;
|
||||
/** The destinations a trip names to the AI. */
|
||||
const DESTINATIONS_PER_TRIP = 5;
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* The trips a booking from a mail may go to: those the person may add
|
||||
* bookings to (`reservation_edit`), not archived, that ended at most 30 days
|
||||
* ago or have no dates, at most 20 of them. The list is the mail import's
|
||||
* whole world: the AI picks from it, the checks weigh it, and nothing outside
|
||||
* it is ever written to.
|
||||
*/
|
||||
@Injectable()
|
||||
export class MailIngestTripsService {
|
||||
constructor(
|
||||
@InjectRepository(MailIngestCandidates) private readonly reads: MailIngestCandidatesRepository,
|
||||
private readonly permissions: PermissionsService,
|
||||
) {}
|
||||
|
||||
async person(userId: number): Promise<MailIngestPersonRow | null> {
|
||||
return this.reads.findPerson(userId);
|
||||
}
|
||||
|
||||
/**
|
||||
* The trips the person may add a booking to, those whose dates hold one of
|
||||
* `hintDates` first (the dates the mail names), then by start, newest first.
|
||||
*/
|
||||
async editableTrips(person: MailIngestPersonRow, now: Date, hintDates: readonly string[]): Promise<TripOption[]> {
|
||||
const cutoff = new Date(now.getTime() - TRIP_LOOKBACK_DAYS * DAY_MS).toISOString().slice(0, 10);
|
||||
const rows = await this.reads.listOpenTrips(person.id, cutoff, TRIPS_READ);
|
||||
const allowed = await Promise.all(rows.map((row) => this.mayEdit(person, row)));
|
||||
const ranked = rows
|
||||
.filter((_, i) => allowed[i])
|
||||
.map((row, order) => ({ row, order, hit: hintDates.some((date) => covers(row, date)) }))
|
||||
.sort((a, b) => Number(b.hit) - Number(a.hit) || a.order - b.order)
|
||||
.slice(0, MAX_TRIP_CANDIDATES)
|
||||
.map(({ row }) => row);
|
||||
return this.withDaysAndPlaces(ranked);
|
||||
}
|
||||
|
||||
/** One trip the person may add a booking to, whatever its dates; null when they may not. */
|
||||
async editableTrip(person: MailIngestPersonRow, tripId: number): Promise<TripOption | null> {
|
||||
const row = await this.reads.findOpenTrip(person.id, tripId);
|
||||
if (!row || !(await this.mayEdit(person, row))) return null;
|
||||
return (await this.withDaysAndPlaces([row]))[0];
|
||||
}
|
||||
|
||||
/** The names a trip is known by, for the AI's list: its places' names, the first few. */
|
||||
destinationsOf(trip: TripOption): string[] {
|
||||
return [...new Set(trip.places.map((place) => place.name))].slice(0, DESTINATIONS_PER_TRIP);
|
||||
}
|
||||
|
||||
private async mayEdit(person: MailIngestPersonRow, trip: MailIngestTripRow): Promise<boolean> {
|
||||
const isMember = trip.user_id !== person.id;
|
||||
return this.permissions.checkPermission('reservation_edit', person.role, trip.user_id, person.id, isMember);
|
||||
}
|
||||
|
||||
private async withDaysAndPlaces(rows: readonly MailIngestTripRow[]): Promise<TripOption[]> {
|
||||
const ids = rows.map((row) => row.id);
|
||||
const days = await this.reads.listDatedDays(ids);
|
||||
const places = await this.reads.listPlacesWithPosition(ids);
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
title: row.title,
|
||||
start_date: row.start_date,
|
||||
end_date: row.end_date,
|
||||
ownerId: row.user_id,
|
||||
currency: (row.currency || 'EUR').toUpperCase(),
|
||||
days: new Map(days.filter((day) => day.trip_id === row.id).map((day) => [day.date, day.id])),
|
||||
places: places.filter((place) => place.trip_id === row.id),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
function covers(trip: MailIngestTripRow, date: string): boolean {
|
||||
return !!trip.start_date && !!trip.end_date && trip.start_date <= date && date <= trip.end_date;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/** The `external_source` of a booking the mail import wrote. */
|
||||
export const MAIL_EXTERNAL_SOURCE = 'mail';
|
||||
|
||||
/** The sources whose keys say a booking is in a trip already: the file import's and the mail import's. */
|
||||
export const IMPORT_SOURCES: readonly string[] = ['import', MAIL_EXTERNAL_SOURCE];
|
||||
|
||||
/** The limits of the import (plan section 11). */
|
||||
export const MAIL_INGEST_LIMITS = {
|
||||
/** The largest mail a self-hosted install takes in, and the largest the TREK Cloud relay hands on. */
|
||||
selfHostedMailBytes: 10 * 1024 * 1024,
|
||||
cloudMailBytes: 25 * 1024 * 1024,
|
||||
/** Bookings stored per mail, and bookings added on their own per mail. */
|
||||
storedBookings: 10,
|
||||
autoBookings: 5,
|
||||
/** Mails one run of the pipeline reads. */
|
||||
mailsPerRun: 3,
|
||||
/** Mails one address takes in a day. */
|
||||
mailsPerAddressPerDay: 50,
|
||||
/** Entries waiting in one person's inbox before new mails stop being read. */
|
||||
pendingPerUser: 200,
|
||||
/** Tries before a mail that keeps failing is given up. */
|
||||
maxAttempts: 3,
|
||||
} as const;
|
||||
|
||||
/** How long an added booking can be undone in one tap. */
|
||||
export const UNDO_WINDOW_DAYS = 7;
|
||||
|
||||
/** Trips that ended longer ago than this are not offered for a new booking. */
|
||||
export const TRIP_LOOKBACK_DAYS = 30;
|
||||
|
||||
/** The most trips a mail is weighed against. */
|
||||
export const MAX_TRIP_CANDIDATES = 20;
|
||||
|
||||
/** Random base32 characters in a token: 24 of them are 120 bits. */
|
||||
export const TOKEN_LENGTH = 24;
|
||||
|
||||
/** A booking whose destination lies within this of one of a trip's places belongs to that trip. */
|
||||
export const NEAR_TRIP_KM = 150;
|
||||
|
||||
/** The in-app actions of the forwarder's notification. */
|
||||
export const UNDO_ACTION = 'mail_ingest_undo';
|
||||
export const KEEP_ACTION = 'mail_ingest_keep';
|
||||
@@ -0,0 +1,85 @@
|
||||
import { MailIngestAddresses } from '../../db/entities/MailIngestAddresses.entity';
|
||||
import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestCursor } from '../../db/entities/MailIngestCursor.entity';
|
||||
import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity';
|
||||
import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity';
|
||||
import { AddonsModule } from '../addons/addons.module';
|
||||
import { AirportsModule } from '../airports/airports.module';
|
||||
import { AppConfigModule } from '../app-config/app-config.module';
|
||||
import { BookingImportModule } from '../booking-import/booking-import.module';
|
||||
import { BudgetModule } from '../budget/budget.module';
|
||||
import { DemoModule } from '../common/demo.module';
|
||||
import { FilesModule } from '../files/files.module';
|
||||
import { LlmParseModule } from '../llm-parse/llm-parse.module';
|
||||
import { MapsModule } from '../maps/maps.module';
|
||||
import { NotificationsModule } from '../notifications/notifications.module';
|
||||
import { PermissionsModule } from '../permissions/permissions.module';
|
||||
import { PlacesModule } from '../places/places.module';
|
||||
import { ReservationsModule } from '../reservations/reservations.module';
|
||||
import { SchedulingModule } from '../scheduling/scheduling.module';
|
||||
import { SettingsModule } from '../settings/settings.module';
|
||||
import { MailIngestActionsService } from './mail-ingest-actions.service';
|
||||
import { MailIngestAddressService } from './mail-ingest-address.service';
|
||||
import { MailIngestApplyService } from './mail-ingest-apply.service';
|
||||
import { MailIngestIntakeService } from './mail-ingest-intake.service';
|
||||
import { MailIngestNotificationActions } from './mail-ingest-notification-actions';
|
||||
import { MailIngestNotifyService } from './mail-ingest-notify.service';
|
||||
import { MailIngestProcessJob } from './mail-ingest-process.job';
|
||||
import { MailIngestProcessService } from './mail-ingest-process.service';
|
||||
import { MailIngestReadingService } from './mail-ingest-reading.service';
|
||||
import { MailIngestRetentionJob } from './mail-ingest-retention.job';
|
||||
import { MailIngestSettingsService } from './mail-ingest-settings.service';
|
||||
import { MailIngestTripsService } from './mail-ingest-trips.service';
|
||||
import { MikroOrmModule } from '@mikro-orm/nestjs';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* The booking mail import: a person forwards a booking mail to an address of
|
||||
* their own, and the booking lands in the matching trip, or waits in their
|
||||
* inbox with a reason. Transports (IMAP, the signed webhook, the TREK Cloud
|
||||
* relay, an .eml upload) only hand mail to MailIngestIntakeService; the
|
||||
* processing job reads it; everything a person does with an entry goes
|
||||
* through MailIngestActionsService.
|
||||
*/
|
||||
@Module({
|
||||
imports: [
|
||||
AppConfigModule,
|
||||
DemoModule,
|
||||
SettingsModule,
|
||||
AddonsModule,
|
||||
PermissionsModule,
|
||||
ReservationsModule,
|
||||
PlacesModule,
|
||||
BudgetModule,
|
||||
FilesModule,
|
||||
MapsModule,
|
||||
AirportsModule,
|
||||
NotificationsModule,
|
||||
BookingImportModule,
|
||||
LlmParseModule,
|
||||
SchedulingModule,
|
||||
MikroOrmModule.forFeature([
|
||||
MailIngestAddresses,
|
||||
MailIngestPrefs,
|
||||
MailIngestCursor,
|
||||
MailIngestMessages,
|
||||
MailIngestCandidates,
|
||||
]),
|
||||
],
|
||||
providers: [
|
||||
MailIngestSettingsService,
|
||||
MailIngestTripsService,
|
||||
MailIngestAddressService,
|
||||
MailIngestIntakeService,
|
||||
MailIngestReadingService,
|
||||
MailIngestApplyService,
|
||||
MailIngestNotifyService,
|
||||
MailIngestProcessService,
|
||||
MailIngestActionsService,
|
||||
MailIngestNotificationActions,
|
||||
MailIngestProcessJob,
|
||||
MailIngestRetentionJob,
|
||||
],
|
||||
exports: [MailIngestSettingsService, MailIngestAddressService, MailIngestIntakeService, MailIngestActionsService],
|
||||
})
|
||||
export class MailIngestModule {}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { currencyDecimals, toMinor } from '@trek/shared';
|
||||
|
||||
/**
|
||||
* A price from a mail as whole hundredths, or null when it cannot be read to
|
||||
* the cent without guessing. A number is taken as it is. A text decides its
|
||||
* decimal mark by itself: with both a dot and a comma the last one is the
|
||||
* decimal mark; with one of them, two digits after it make it the decimal
|
||||
* mark, three are a thousands group or a third decimal, so the reading is
|
||||
* left alone. More decimals than the currency has are no price either.
|
||||
*/
|
||||
export function parsePriceMinor(value: unknown, currency: string | null): number | null {
|
||||
const decimals = currency ? currencyDecimals(currency) : 2;
|
||||
if (typeof value === 'number') return Number.isFinite(value) ? toMinor(value) : null;
|
||||
if (typeof value !== 'string') return null;
|
||||
const text = value.replace(/[^\d.,]/g, '');
|
||||
const parts = splitAmount(text);
|
||||
if (!parts) return null;
|
||||
const [whole, fraction] = parts;
|
||||
if (fraction.length > decimals) return null;
|
||||
return toMinor(Number(`${whole}.${fraction || '0'}`));
|
||||
}
|
||||
|
||||
/** The whole and the fractional digits of an amount, or null when its separators are ambiguous. */
|
||||
function splitAmount(text: string): [string, string] | null {
|
||||
if (/^\d+$/.test(text)) return [text, ''];
|
||||
const lastDot = text.lastIndexOf('.');
|
||||
const lastComma = text.lastIndexOf(',');
|
||||
if (lastDot >= 0 && lastComma >= 0) {
|
||||
const decimalAt = Math.max(lastDot, lastComma);
|
||||
const thousands = text[Math.min(lastDot, lastComma)];
|
||||
const whole = text.slice(0, decimalAt);
|
||||
return grouped(whole, thousands) ? [digits(whole), text.slice(decimalAt + 1)] : null;
|
||||
}
|
||||
const mark = lastDot >= 0 ? '.' : ',';
|
||||
const pieces = text.split(mark);
|
||||
if (pieces.length > 2) return grouped(text, mark) ? [digits(text), ''] : null;
|
||||
const [whole, fraction] = pieces;
|
||||
if (!/^\d+$/.test(whole) || !/^\d+$/.test(fraction) || fraction.length === 3) return null;
|
||||
return [whole, fraction];
|
||||
}
|
||||
|
||||
/** Whether `text` is digits in groups of three behind its first group, split by `separator`. */
|
||||
function grouped(text: string, separator: string): boolean {
|
||||
const [head, ...groups] = text.split(separator);
|
||||
return /^\d{1,3}$/.test(head) && groups.length > 0 && groups.every((group) => /^\d{3}$/.test(group));
|
||||
}
|
||||
|
||||
function digits(text: string): string {
|
||||
return text.replace(/\D/g, '');
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { decrypt_api_key, encrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
|
||||
/**
|
||||
* The stored form of a forwarded mail: encrypted the way API keys are. A mail
|
||||
* is bytes; latin1 maps each byte to one character and back, so nothing is
|
||||
* lost on the way through the text column.
|
||||
*/
|
||||
export function sealMail(raw: Buffer): string {
|
||||
return encrypt_api_key(raw.toString('latin1'));
|
||||
}
|
||||
|
||||
/** The mail a sealed text holds; null for one that does not open (another key, a broken row). */
|
||||
export function openMail(sealed: string | null): Buffer | null {
|
||||
if (!sealed) return null;
|
||||
const text = decrypt_api_key(sealed);
|
||||
return text === null ? null : Buffer.from(text, 'latin1');
|
||||
}
|
||||
@@ -15,9 +15,10 @@ import type { MailIngestMessagesRepository } from '../../../../src/db/repositori
|
||||
import { dbNow } from '../../../../src/db/types';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { addBudgetItemPayer, makeBudgetItem } from '../../../helpers/factories/budget';
|
||||
import { makeDayAssignment } from '../../../helpers/factories/itinerary';
|
||||
import { makePlace } from '../../../helpers/factories/places';
|
||||
import { makeReservation } from '../../../helpers/factories/reservations';
|
||||
import { addTripMember, makeTrip } from '../../../helpers/factories/trips';
|
||||
import { addTripMember, makeTrip, readTripDays } from '../../../helpers/factories/trips';
|
||||
import { makeUser } from '../../../helpers/factories/users';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
@@ -67,7 +68,8 @@ async function mailOf(userId: number): Promise<number> {
|
||||
rawEnc: 'enc',
|
||||
receivedAt: DAY1,
|
||||
});
|
||||
return id!;
|
||||
if (id === null) throw new Error('the mail was stored already');
|
||||
return id;
|
||||
}
|
||||
|
||||
function entry(messageId: number, userId: number, over: Partial<MailIngestCandidateInsert> = {}) {
|
||||
@@ -128,6 +130,26 @@ describe('MailIngestCandidatesRepository: the entries', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('writes an entry decided as it is written with its trip, ids and undo window', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const trip = await makeTrip(t, user.id);
|
||||
const until = at('2026-11-08T10:00:00Z');
|
||||
const id = await entry(await mailOf(user.id), user.id, {
|
||||
resolution: 'auto_applied',
|
||||
reason: null,
|
||||
resolvedTripId: trip.id,
|
||||
createdIdsJson: '{"reservation_ids":[3]}',
|
||||
resolvedAt: DAY1,
|
||||
undoUntil: until,
|
||||
});
|
||||
expect(await candidates.findById(id)).toMatchObject({
|
||||
resolved_trip_id: trip.id,
|
||||
created_ids_json: '{"reservation_ids":[3]}',
|
||||
resolved_at: DAY1,
|
||||
undo_until: until,
|
||||
});
|
||||
});
|
||||
|
||||
it('remembers a booking undone in a trip by its content', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const trip = await makeTrip(t, user.id);
|
||||
@@ -184,17 +206,32 @@ describe('MailIngestCandidatesRepository: what it reads of the trips', () => {
|
||||
expect(await candidates.listOpenTrips(user.id, '2026-10-01', 1)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('finds one open trip of the person whatever its dates, and none of a stranger', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const { user: other } = await makeUser(t);
|
||||
const ended = await makeTrip(t, user.id, { start_date: '2025-01-01', end_date: '2025-01-02' });
|
||||
const archived = await makeTrip(t, user.id, { is_archived: 1 });
|
||||
const foreign = await makeTrip(t, other.id);
|
||||
expect(await candidates.findOpenTrip(user.id, ended.id)).toMatchObject({ id: ended.id, user_id: user.id });
|
||||
expect(await candidates.findOpenTrip(user.id, archived.id)).toBeNull();
|
||||
expect(await candidates.findOpenTrip(user.id, foreign.id)).toBeNull();
|
||||
await addTripMember(t, foreign.id, user.id);
|
||||
expect(await candidates.findOpenTrip(user.id, foreign.id)).toMatchObject({ id: foreign.id });
|
||||
});
|
||||
|
||||
it('reads the dated days and the placed places of trips', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const trip = await makeTrip(t, user.id, { start_date: '2027-03-01', end_date: '2027-03-02' });
|
||||
await makePlace(t, trip.id, { name: 'Hotel Sakura', address: 'Tokyo', lat: 35.68, lng: 139.76 });
|
||||
await makePlace(t, trip.id, { name: 'Somewhere', lat: null, lng: null });
|
||||
|
||||
expect(await candidates.listDayDates([trip.id])).toEqual([
|
||||
{ trip_id: trip.id, date: '2027-03-01' },
|
||||
{ trip_id: trip.id, date: '2027-03-02' },
|
||||
const days = await candidates.listDatedDays([trip.id]);
|
||||
expect(days.map(({ trip_id, date }) => [trip_id, date])).toEqual([
|
||||
[trip.id, '2027-03-01'],
|
||||
[trip.id, '2027-03-02'],
|
||||
]);
|
||||
expect(await candidates.listDayDates([])).toEqual([]);
|
||||
expect(days.every((day) => day.id > 0)).toBe(true);
|
||||
expect(await candidates.listDatedDays([])).toEqual([]);
|
||||
expect(await candidates.listPlacesWithPosition([trip.id])).toEqual([
|
||||
{ trip_id: trip.id, name: 'Hotel Sakura', address: 'Tokyo', lat: 35.68, lng: 139.76 },
|
||||
]);
|
||||
@@ -243,6 +280,20 @@ describe('MailIngestCandidatesRepository: what it reads of the trips', () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('tells a place something still names from one nothing does', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const trip = await makeTrip(t, user.id, { start_date: '2027-03-01', end_date: '2027-03-01' });
|
||||
const [day] = await readTripDays(t, trip.id);
|
||||
const free = await makePlace(t, trip.id);
|
||||
const booked = await makePlace(t, trip.id);
|
||||
const planned = await makePlace(t, trip.id);
|
||||
await makeReservation(t, trip.id, { place: booked.id });
|
||||
await makeDayAssignment(t, day.id, planned.id);
|
||||
expect(await candidates.placeInUse(free.id)).toBe(false);
|
||||
expect(await candidates.placeInUse(booked.id)).toBe(true);
|
||||
expect(await candidates.placeInUse(planned.id)).toBe(true);
|
||||
});
|
||||
|
||||
it('counts the payers of costs', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const trip = await makeTrip(t, user.id);
|
||||
|
||||
@@ -56,7 +56,14 @@ async function inbox() {
|
||||
return { userId: user.id, addressId };
|
||||
}
|
||||
|
||||
function store(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1) {
|
||||
/** Stores a mail and answers its id; a second store of the same hash fails the test. */
|
||||
async function store(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1): Promise<number> {
|
||||
const id = await insert(box, hash, receivedAt);
|
||||
if (id === null) throw new Error(`mail ${hash} was stored already`);
|
||||
return id;
|
||||
}
|
||||
|
||||
function insert(box: { userId: number; addressId: number }, hash: string, receivedAt = DAY1) {
|
||||
return messages.insertIfNew({
|
||||
transport: 'upload',
|
||||
messageIdHdr: `<${hash}@x>`,
|
||||
@@ -88,12 +95,11 @@ describe('MailIngestMessagesRepository', () => {
|
||||
it('stores a mail once and finds it by its hash', async () => {
|
||||
const box = await inbox();
|
||||
const id = await store(box, 'm1');
|
||||
expect(id).not.toBeNull();
|
||||
expect(await store(box, 'm1')).toBeNull();
|
||||
expect(await insert(box, 'm1')).toBeNull();
|
||||
expect(await messages.findIdByDedupeHash('m1')).toBe(id);
|
||||
expect(await messages.findIdByDedupeHash('m2')).toBeNull();
|
||||
expect(await messages.findById(id!)).toMatchObject({ status: 'queued', attempts: 0, raw_eml_enc: 'enc' });
|
||||
expect(await messages.findById(id! + 1)).toBeNull();
|
||||
expect(await messages.findById(id)).toMatchObject({ status: 'queued', attempts: 0, raw_eml_enc: 'enc' });
|
||||
expect(await messages.findById(id + 1)).toBeNull();
|
||||
});
|
||||
|
||||
it('counts the mails of an address and the AI-read mails of a person since a time', async () => {
|
||||
@@ -101,7 +107,7 @@ describe('MailIngestMessagesRepository', () => {
|
||||
const first = await store(box, 'c1', at('2026-10-31T10:00:00Z'));
|
||||
await store(box, 'c2', DAY1);
|
||||
expect(await messages.countForAddressSince(box.addressId, at('2026-11-01T00:00:00Z'))).toBe(1);
|
||||
await messages.finish(first!, DONE);
|
||||
await messages.finish(first, DONE);
|
||||
expect(await messages.countAiReadSince(box.userId, at('2026-10-31T00:00:00Z'))).toBe(1);
|
||||
expect(await messages.countAiReadSince(box.userId, at('2026-11-01T00:00:00Z'))).toBe(0);
|
||||
});
|
||||
@@ -118,14 +124,14 @@ describe('MailIngestMessagesRepository', () => {
|
||||
|
||||
const later = at('2026-11-01T11:20:00Z');
|
||||
expect(await messages.claimDue(later, at('2026-11-01T11:30:00Z'), 3)).toEqual([id]);
|
||||
expect(await messages.findById(id!)).toMatchObject({ status: 'processing', attempts: 2 });
|
||||
expect(await messages.findById(id)).toMatchObject({ status: 'processing', attempts: 2 });
|
||||
|
||||
await messages.retryLater(id!, at('2026-11-01T12:00:00Z'));
|
||||
await messages.retryLater(id, at('2026-11-01T12:00:00Z'));
|
||||
expect(await messages.claimDue(later, lease, 3)).toEqual([]);
|
||||
expect(await messages.claimDue(at('2026-11-01T12:00:00Z'), lease, 3)).toEqual([id]);
|
||||
|
||||
await messages.finish(id!, DONE);
|
||||
expect(await messages.findById(id!)).toMatchObject({
|
||||
await messages.finish(id, DONE);
|
||||
expect(await messages.findById(id)).toMatchObject({
|
||||
status: 'processed',
|
||||
ai_used: 1,
|
||||
from_verified: 1,
|
||||
@@ -138,10 +144,10 @@ describe('MailIngestMessagesRepository', () => {
|
||||
const waiting = await store(box, 'p1');
|
||||
const settled = await store(box, 'p2');
|
||||
const old = await store(box, 'p3', at('2026-09-01T10:00:00Z'));
|
||||
await messages.finish(waiting!, DONE);
|
||||
await messages.finish(settled!, DONE);
|
||||
await messages.finish(waiting, DONE);
|
||||
await messages.finish(settled, DONE);
|
||||
await candidates.insertCandidate({
|
||||
messageId: waiting!,
|
||||
messageId: waiting,
|
||||
userId: box.userId,
|
||||
position: 0,
|
||||
itemJson: null,
|
||||
@@ -159,9 +165,9 @@ describe('MailIngestMessagesRepository', () => {
|
||||
|
||||
const purgedAt = at('2026-11-20T10:00:00Z');
|
||||
expect(await messages.purgeRaw(at('2026-11-10T10:00:00Z'), at('2026-10-01T10:00:00Z'), purgedAt)).toBe(2);
|
||||
expect(await messages.findById(waiting!)).toMatchObject({ raw_eml_enc: 'enc', raw_purged_at: null });
|
||||
expect(await messages.findById(settled!)).toMatchObject({ raw_eml_enc: null, raw_purged_at: purgedAt });
|
||||
expect(await messages.findById(old!)).toMatchObject({ raw_eml_enc: null });
|
||||
expect(await messages.findById(waiting)).toMatchObject({ raw_eml_enc: 'enc', raw_purged_at: null });
|
||||
expect(await messages.findById(settled)).toMatchObject({ raw_eml_enc: null, raw_purged_at: purgedAt });
|
||||
expect(await messages.findById(old)).toMatchObject({ raw_eml_enc: null });
|
||||
});
|
||||
|
||||
it('deletes settled mails without entries, and keeps queued ones and those with entries', async () => {
|
||||
@@ -169,10 +175,10 @@ describe('MailIngestMessagesRepository', () => {
|
||||
const queued = await store(box, 'x1', at('2026-08-01T10:00:00Z'));
|
||||
const bare = await store(box, 'x2', at('2026-08-01T10:00:00Z'));
|
||||
const withEntry = await store(box, 'x3', at('2026-08-01T10:00:00Z'));
|
||||
await messages.finish(bare!, { ...DONE, status: 'discarded' });
|
||||
await messages.finish(withEntry!, DONE);
|
||||
await messages.finish(bare, { ...DONE, status: 'discarded' });
|
||||
await messages.finish(withEntry, DONE);
|
||||
await candidates.insertCandidate({
|
||||
messageId: withEntry!,
|
||||
messageId: withEntry,
|
||||
userId: box.userId,
|
||||
position: 0,
|
||||
itemJson: null,
|
||||
@@ -189,8 +195,8 @@ describe('MailIngestMessagesRepository', () => {
|
||||
});
|
||||
|
||||
expect(await messages.deleteSettledBefore(at('2026-09-01T00:00:00Z'))).toBe(1);
|
||||
expect(await messages.findById(bare!)).toBeNull();
|
||||
expect(await messages.findById(queued!)).not.toBeNull();
|
||||
expect(await messages.findById(withEntry!)).not.toBeNull();
|
||||
expect(await messages.findById(bare)).toBeNull();
|
||||
expect(await messages.findById(queued)).not.toBeNull();
|
||||
expect(await messages.findById(withEntry)).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -122,11 +122,17 @@ describe('MailReadService.read', () => {
|
||||
});
|
||||
|
||||
it('reports a spent allowance as its outcome and logs no content', async () => {
|
||||
const error = vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
const lines: string[] = [];
|
||||
for (const method of ['log', 'info', 'warn', 'error'] as const) {
|
||||
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
|
||||
lines.push(args.map(String).join(' '));
|
||||
});
|
||||
}
|
||||
extract.mockRejectedValue(new LlmRequestError('gate', 429, 'QUOTA_SPENT'));
|
||||
const reading = await service.read(input(), 1, '2026-11-13');
|
||||
expect(reading.outcome).toBe('quota_exhausted');
|
||||
expect(String(error.mock.calls[0][0])).not.toContain('ABC123');
|
||||
expect(lines.join(' | ')).toContain('Mail read failed');
|
||||
expect(lines.join(' | ')).not.toContain('ABC123');
|
||||
});
|
||||
|
||||
it('reads through the local router without a trip choice', async () => {
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* The server's checks on a booking read from a mail (G1 to G8) and the choice
|
||||
* of its trip: the AI's pick is one input, the dates and the trip list decide,
|
||||
* and every failure names the first reason the inbox shows.
|
||||
*/
|
||||
import {
|
||||
checkInText,
|
||||
chooseTrip,
|
||||
contentHash,
|
||||
fitsTrip,
|
||||
implausible,
|
||||
isCancellation,
|
||||
judgeBooking,
|
||||
missingCritical,
|
||||
missingReference,
|
||||
nearTrip,
|
||||
type CheckContext,
|
||||
type MailDraft,
|
||||
type TripOption,
|
||||
} from '../../../../src/nest/mail-ingest/mail-ingest-checks';
|
||||
import type { BookingImportPreviewItem } from '@trek/shared';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const NOW = new Date('2030-01-15T10:00:00Z');
|
||||
const endpoint = (role: 'from' | 'to', name: string, code: string, lat: number, lng: number) => ({
|
||||
role,
|
||||
sequence: role === 'from' ? 0 : 1,
|
||||
name,
|
||||
code,
|
||||
lat,
|
||||
lng,
|
||||
timezone: null,
|
||||
local_time: null,
|
||||
local_date: null,
|
||||
});
|
||||
const HND = endpoint('from', 'Tokyo Haneda', 'HND', 35.55, 139.78);
|
||||
const ITM = endpoint('to', 'Osaka Itami', 'ITM', 34.78, 135.44);
|
||||
const FLIGHT: BookingImportPreviewItem = {
|
||||
type: 'flight',
|
||||
title: 'LH 716 Tokyo to Osaka',
|
||||
reservation_time: '2030-03-02T10:00',
|
||||
reservation_end_time: '2030-03-02T11:15',
|
||||
confirmation_number: 'ABC123',
|
||||
metadata: { flight_number: 'LH716', price: 120.5, priceCurrency: 'EUR' },
|
||||
endpoints: [HND, ITM],
|
||||
source: { fileName: 'mail.eml', index: 0 },
|
||||
};
|
||||
const HOTEL: BookingImportPreviewItem = {
|
||||
type: 'hotel',
|
||||
title: 'Hotel Sakura',
|
||||
confirmation_number: 'H-778',
|
||||
_venue: { name: 'Hotel Sakura', lat: 35.01, lng: 135.76 },
|
||||
_accommodation: { check_in: '2030-03-03T15:00', check_out: '2030-03-05T11:00' },
|
||||
source: { fileName: 'mail.eml', index: 0 },
|
||||
};
|
||||
const TEXT = 'Your booking ABC123: flight LH 716 from Tokyo to Osaka. Hotel Sakura, reference H-778.';
|
||||
|
||||
function trip(id: number, start: string | null, end: string | null, over: Partial<TripOption> = {}): TripOption {
|
||||
const days = new Map<string, number>();
|
||||
if (start && end) {
|
||||
for (let d = new Date(`${start}T00:00:00Z`), n = 1; d <= new Date(`${end}T00:00:00Z`); n++) {
|
||||
days.set(d.toISOString().slice(0, 10), id * 100 + n);
|
||||
d = new Date(d.getTime() + 86_400_000);
|
||||
}
|
||||
}
|
||||
return {
|
||||
id,
|
||||
title: `Trip ${id}`,
|
||||
start_date: start,
|
||||
end_date: end,
|
||||
ownerId: 1,
|
||||
currency: 'EUR',
|
||||
days,
|
||||
places: [],
|
||||
...over,
|
||||
};
|
||||
}
|
||||
const JAPAN = trip(1, '2030-03-01', '2030-03-10');
|
||||
const context = (over: Partial<CheckContext> = {}): CheckContext => ({
|
||||
trips: [JAPAN],
|
||||
addressTripId: null,
|
||||
subject: 'Your flight',
|
||||
documentText: TEXT,
|
||||
now: NOW,
|
||||
knownAirport: (code) => ['HND', 'ITM'].includes(code),
|
||||
...over,
|
||||
});
|
||||
const draft = (over: Partial<MailDraft> = {}): MailDraft => ({
|
||||
item: FLIGHT,
|
||||
source: 'kitinerary',
|
||||
kind: 'new_booking',
|
||||
aiTripId: null,
|
||||
uncertainFields: [],
|
||||
suggestedNewTrip: null,
|
||||
imageOnly: false,
|
||||
disagrees: false,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('the single checks', () => {
|
||||
it('needs a type, a start and, for a transport, both ends', () => {
|
||||
expect(missingCritical(FLIGHT)).toBe(false);
|
||||
expect(missingCritical({ ...FLIGHT, reservation_time: null })).toBe(true);
|
||||
expect(missingCritical({ ...FLIGHT, endpoints: [HND] })).toBe(true);
|
||||
expect(missingCritical(HOTEL)).toBe(false);
|
||||
});
|
||||
|
||||
it('needs a booking reference for a transport, a stay or a rental only', () => {
|
||||
expect(missingReference({ ...FLIGHT, confirmation_number: ' ' })).toBe(true);
|
||||
expect(missingReference({ ...FLIGHT, type: 'restaurant', confirmation_number: null })).toBe(false);
|
||||
});
|
||||
|
||||
it('refuses values that cannot be right (G5)', () => {
|
||||
const known = (code: string) => code !== 'XXX';
|
||||
expect(implausible(FLIGHT, NOW, known)).toBe(false);
|
||||
expect(implausible({ ...FLIGHT, reservation_time: 'soon' }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, reservation_time: '2034-01-01T10:00' }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, reservation_end_time: '2030-03-01T10:00' }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, reservation_end_time: '2030-03-03T10:00' }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, confirmation_number: 'A' }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, metadata: { price: 200_000 } }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, metadata: { priceCurrency: 'euro' } }, NOW, known)).toBe(true);
|
||||
expect(implausible({ ...FLIGHT, metadata: { price: '1.234,50' } }, NOW, known)).toBe(false);
|
||||
expect(implausible(FLIGHT, NOW, (code) => code !== 'ITM')).toBe(true);
|
||||
const longStay = { ...HOTEL, _accommodation: { check_in: '2030-03-01', check_out: '2030-06-01' } };
|
||||
expect(implausible(longStay, NOW, known)).toBe(true);
|
||||
});
|
||||
|
||||
it('vetoes a cancellation by kind, status or subject (G8)', () => {
|
||||
const cancelled = { ...FLIGHT, metadata: { reservationStatus: 'ReservationCancelled' } };
|
||||
expect(isCancellation(FLIGHT, 'cancellation', null)).toBe(true);
|
||||
expect(isCancellation(cancelled, 'new_booking', null)).toBe(true);
|
||||
expect(isCancellation(FLIGHT, 'new_booking', 'Ihre Buchung wurde storniert')).toBe(true);
|
||||
expect(isCancellation(FLIGHT, 'new_booking', 'Your booking at Hotel Sakura')).toBe(false);
|
||||
});
|
||||
|
||||
it('wants an AI value in the mail, and only marks a free text it cannot find (G6)', () => {
|
||||
expect(checkInText(FLIGHT, TEXT)).toEqual({ ok: true, uncertain: [] });
|
||||
expect(checkInText({ ...FLIGHT, confirmation_number: 'ZZZ999' }, TEXT).ok).toBe(false);
|
||||
expect(checkInText({ ...FLIGHT, title: 'Paris Express', location: 'Gare du Nord' }, TEXT)).toEqual({
|
||||
ok: true,
|
||||
uncertain: ['title', 'location'],
|
||||
});
|
||||
});
|
||||
|
||||
it('fits a booking into a trip by its dates and the days it needs (G3)', () => {
|
||||
expect(fitsTrip(FLIGHT, JAPAN)).toBe(true);
|
||||
expect(fitsTrip(HOTEL, JAPAN)).toBe(true);
|
||||
expect(fitsTrip(FLIGHT, trip(2, '2030-04-01', '2030-04-05'))).toBe(false);
|
||||
expect(fitsTrip(FLIGHT, trip(3, null, null))).toBe(false);
|
||||
// A day of slack is no day row: the booking needs the day it starts on.
|
||||
expect(fitsTrip(FLIGHT, trip(4, '2030-03-03', '2030-03-05'))).toBe(false);
|
||||
expect(fitsTrip({ ...FLIGHT, reservation_time: null }, JAPAN)).toBe(false);
|
||||
});
|
||||
|
||||
it('confirms a booking lies near a trip by the trip’s places', () => {
|
||||
const kyoto = trip(5, '2030-03-01', '2030-03-10', {
|
||||
places: [{ name: 'Kyoto Station', address: null, lat: 34.98, lng: 135.75 }],
|
||||
});
|
||||
expect(nearTrip(HOTEL, kyoto)).toBe(true);
|
||||
expect(nearTrip(FLIGHT, kyoto)).toBe(true);
|
||||
expect(nearTrip({ ...HOTEL, _venue: { name: 'Far', lat: 43.06, lng: 141.35 } }, kyoto)).toBe(false);
|
||||
});
|
||||
|
||||
it('hashes what a booking says', () => {
|
||||
expect(contentHash(FLIGHT)).toBe(contentHash({ ...FLIGHT, confirmation_number: 'abc 123' }));
|
||||
expect(contentHash(FLIGHT)).not.toBe(contentHash({ ...FLIGHT, reservation_time: '2030-03-02T11:00' }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('chooseTrip', () => {
|
||||
const second = trip(2, '2030-02-25', '2030-03-05', {
|
||||
places: [{ name: 'Osaka', address: null, lat: 34.7, lng: 135.5 }],
|
||||
});
|
||||
|
||||
it('takes the one trip whose dates fit', () => {
|
||||
expect(chooseTrip(FLIGHT, null, context())).toEqual({ tripId: 1, suggestedTripId: 1, reason: null });
|
||||
});
|
||||
|
||||
it('of several, takes the AI’s pick only when the booking lies near it', () => {
|
||||
const both = context({ trips: [JAPAN, second] });
|
||||
expect(chooseTrip(FLIGHT, 2, both)).toEqual({ tripId: 2, suggestedTripId: 2, reason: null });
|
||||
expect(chooseTrip(FLIGHT, 1, both)).toEqual({ tripId: null, suggestedTripId: 1, reason: 'trip_ambiguous' });
|
||||
expect(chooseTrip(FLIGHT, null, both)).toEqual({ tripId: null, suggestedTripId: 1, reason: 'trip_ambiguous' });
|
||||
});
|
||||
|
||||
it('suggests the AI’s pick when no trip fits, and names trips without dates', () => {
|
||||
const later = trip(6, '2030-06-01', '2030-06-10');
|
||||
expect(chooseTrip(FLIGHT, 6, context({ trips: [later] }))).toEqual({
|
||||
tripId: null,
|
||||
suggestedTripId: 6,
|
||||
reason: 'no_trip_in_range',
|
||||
});
|
||||
expect(chooseTrip(FLIGHT, 9, context({ trips: [trip(7, null, null)] })).reason).toBe('trip_without_dates');
|
||||
});
|
||||
|
||||
it('keeps a trip address’s booking to its trip, whatever the AI says', () => {
|
||||
expect(chooseTrip(FLIGHT, 2, context({ addressTripId: 1, trips: [JAPAN, second] }))).toEqual({
|
||||
tripId: 1,
|
||||
suggestedTripId: 1,
|
||||
reason: null,
|
||||
});
|
||||
const undated = trip(8, null, null);
|
||||
expect(chooseTrip(FLIGHT, null, context({ addressTripId: 8, trips: [undated] })).tripId).toBe(8);
|
||||
const elsewhen = trip(9, '2030-06-01', '2030-06-10');
|
||||
const outside = chooseTrip(FLIGHT, null, context({ addressTripId: 9, trips: [elsewhen] }));
|
||||
expect(outside.reason).toBe('outside_trip_dates');
|
||||
expect(chooseTrip(FLIGHT, null, context({ addressTripId: 10 })).reason).toBe('no_edit_permission');
|
||||
});
|
||||
});
|
||||
|
||||
describe('judgeBooking', () => {
|
||||
it('passes a machine-read booking that fits one trip', () => {
|
||||
const verdict = judgeBooking(draft(), context());
|
||||
expect(verdict.reason).toBeNull();
|
||||
expect(verdict.checks).toMatchObject({ schema: 'pass', dates: 'pass', agreement: 'skipped', in_text: 'skipped' });
|
||||
expect(verdict.choice.tripId).toBe(1);
|
||||
});
|
||||
|
||||
it('names the first reason in the order the inbox reads them', () => {
|
||||
expect(judgeBooking(draft({ kind: 'change' }), context()).reason).toBe('change_or_cancellation');
|
||||
expect(judgeBooking(draft({ kind: 'cancellation' }), context()).reason).toBe('change_or_cancellation');
|
||||
expect(judgeBooking(draft(), context({ subject: 'Cancelled: your flight' })).reason).toBe('cancelled');
|
||||
expect(judgeBooking(draft({ item: { ...FLIGHT, endpoints: [] } }), context()).reason).toBe(
|
||||
'missing_critical_field',
|
||||
);
|
||||
expect(judgeBooking(draft({ item: { ...FLIGHT, title: '' } }), context()).reason).toBe('missing_critical_field');
|
||||
expect(judgeBooking(draft(), context({ knownAirport: () => false })).reason).toBe('implausible');
|
||||
expect(judgeBooking(draft({ source: 'ai', disagrees: true }), context()).reason).toBe('source_mismatch');
|
||||
expect(judgeBooking(draft({ source: 'ai', imageOnly: true }), context()).reason).toBe('image_only');
|
||||
expect(judgeBooking(draft({ source: 'ai' }), context({ documentText: 'nothing' })).reason).toBe('not_in_text');
|
||||
const noReference = { ...FLIGHT, confirmation_number: null };
|
||||
expect(judgeBooking(draft({ item: noReference }), context()).reason).toBe('missing_confirmation');
|
||||
expect(judgeBooking(draft(), context({ trips: [] })).reason).toBe('no_trip_in_range');
|
||||
});
|
||||
|
||||
it('adds the fields it could not find to the AI’s own uncertain ones', () => {
|
||||
const verdict = judgeBooking(
|
||||
draft({ source: 'ai', uncertainFields: ['seat'], item: { ...FLIGHT, location: 'Terminal 9' } }),
|
||||
context(),
|
||||
);
|
||||
expect(verdict.uncertain).toEqual(['seat', 'location']);
|
||||
expect(verdict.checks.in_text).toBe('pass');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* The mail import's pure pieces: what a mail is before anything reads it, a
|
||||
* signed sender as a hint, a price read to the cent or not at all, the
|
||||
* address token, and the sealed mail.
|
||||
*/
|
||||
import { walkMail } from '../../../../src/nest/llm-parse/mime-walker';
|
||||
import { addressOf, classifyMail, senderVerified } from '../../../../src/nest/mail-ingest/mail-classify';
|
||||
import { hashToken, newToken, tokenOf } from '../../../../src/nest/mail-ingest/mail-ingest-address.service';
|
||||
import { parsePriceMinor } from '../../../../src/nest/mail-ingest/mail-money';
|
||||
import { openMail, sealMail } from '../../../../src/nest/mail-ingest/mail-seal';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
/** A mail with the given headers and body, as the walker reads it. */
|
||||
function mail(headers: Record<string, string>, body: string) {
|
||||
const head = Object.entries({ From: 'Mia <mia@example.com>', Subject: 'Hello', ...headers })
|
||||
.map(([name, value]) => `${name}: ${value}`)
|
||||
.join('\r\n');
|
||||
const walked = walkMail(Buffer.from(`${head}\r\nContent-Type: text/plain\r\n\r\n${body}`));
|
||||
if (!walked) throw new Error('the fixture is no mail');
|
||||
return walked;
|
||||
}
|
||||
|
||||
const BOOKING_BODY = 'Your booking ABC123 is confirmed. Flight LH 716 on 2030-03-02.';
|
||||
|
||||
describe('classifyMail', () => {
|
||||
it('reads a booking mail', () => {
|
||||
expect(classifyMail(mail({ Subject: 'Your flight' }, BOOKING_BODY))).toEqual({ kind: 'read' });
|
||||
});
|
||||
|
||||
it('shows the code of a forwarding confirmation and clicks nothing', () => {
|
||||
const gmail = mail(
|
||||
{ From: 'forwarding-noreply@google.com', Subject: '(#123) Gmail Forwarding Confirmation' },
|
||||
'Confirmation code: 612345678. To allow forwarding, click the link.',
|
||||
);
|
||||
expect(classifyMail(gmail)).toEqual({ kind: 'forwarding_confirmation', code: '612345678' });
|
||||
});
|
||||
|
||||
it('drops an automatic reply, but not a confirmation a booking system generated', () => {
|
||||
expect(classifyMail(mail({ 'Auto-Submitted': 'auto-replied' }, BOOKING_BODY))).toEqual({
|
||||
kind: 'drop',
|
||||
reason: 'auto_reply',
|
||||
});
|
||||
expect(classifyMail(mail({ Subject: 'Out of office' }, BOOKING_BODY)).kind).toBe('drop');
|
||||
expect(classifyMail(mail({ 'Auto-Submitted': 'auto-generated' }, BOOKING_BODY))).toEqual({ kind: 'read' });
|
||||
});
|
||||
|
||||
it('drops an empty mail', () => {
|
||||
expect(classifyMail(mail({}, 'hi'))).toEqual({ kind: 'drop', reason: 'empty_mail' });
|
||||
});
|
||||
|
||||
it('drops a list mail without a booking in it, and keeps one that has one', () => {
|
||||
const list = { 'List-Unsubscribe': '<mailto:off@example.com>' };
|
||||
expect(classifyMail(mail(list, 'Our autumn deals are here, have a look at them all.'))).toEqual({
|
||||
kind: 'drop',
|
||||
reason: 'newsletter',
|
||||
});
|
||||
expect(classifyMail(mail(list, BOOKING_BODY))).toEqual({ kind: 'read' });
|
||||
const bulk = mail({ Precedence: 'bulk' }, 'Weekly news for travellers who love to read.');
|
||||
expect(classifyMail(bulk).kind).toBe('drop');
|
||||
});
|
||||
});
|
||||
|
||||
describe('senderVerified', () => {
|
||||
const signed = { 'Authentication-Results': 'mx.example.net; dkim=pass header.d=example.com; spf=pass' };
|
||||
|
||||
it('holds for the person’s own address, signed by its domain', () => {
|
||||
expect(senderVerified(mail(signed, BOOKING_BODY), 'Mia@Example.com')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not hold for another sender or without a signature', () => {
|
||||
expect(senderVerified(mail(signed, BOOKING_BODY), 'other@example.com')).toBe(false);
|
||||
expect(senderVerified(mail({}, BOOKING_BODY), 'mia@example.com')).toBe(false);
|
||||
const forged = { 'Authentication-Results': 'mx.example.net; dkim=pass header.d=evil.test' };
|
||||
expect(senderVerified(mail(forged, BOOKING_BODY), 'mia@example.com')).toBe(false);
|
||||
});
|
||||
|
||||
it('reads a bare address out of a From header', () => {
|
||||
expect(addressOf('Mia <MIA@example.com>')).toBe('mia@example.com');
|
||||
expect(addressOf('mia@example.com')).toBe('mia@example.com');
|
||||
expect(addressOf(null)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('parsePriceMinor', () => {
|
||||
it('takes a number as it is and reads unambiguous text', () => {
|
||||
expect(parsePriceMinor(120.5, 'EUR')).toBe(12050);
|
||||
expect(parsePriceMinor('1.234,56', 'EUR')).toBe(123456);
|
||||
expect(parsePriceMinor('1,234.56', 'USD')).toBe(123456);
|
||||
expect(parsePriceMinor('€ 89,35', 'EUR')).toBe(8935);
|
||||
expect(parsePriceMinor('1.234.567', 'EUR')).toBe(123456700);
|
||||
expect(parsePriceMinor('12000', 'JPY')).toBe(1200000);
|
||||
expect(parsePriceMinor('0,5', null)).toBe(50);
|
||||
});
|
||||
|
||||
it('refuses what it would have to guess', () => {
|
||||
expect(parsePriceMinor('1.234', 'EUR')).toBeNull();
|
||||
expect(parsePriceMinor('12.50', 'JPY')).toBeNull();
|
||||
expect(parsePriceMinor('1,2,3', 'EUR')).toBeNull();
|
||||
expect(parsePriceMinor('12.34,5.6', 'EUR')).toBeNull();
|
||||
expect(parsePriceMinor(Number.NaN, 'EUR')).toBeNull();
|
||||
expect(parsePriceMinor({ amount: 1 }, 'EUR')).toBeNull();
|
||||
expect(parsePriceMinor('', 'EUR')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the address token', () => {
|
||||
it('is 24 base32 characters and looked up by its hash, whatever its case', () => {
|
||||
const token = newToken();
|
||||
expect(token).toMatch(/^[a-z2-7]{24}$/);
|
||||
expect(newToken()).not.toBe(token);
|
||||
expect(hashToken(token.toUpperCase())).toBe(hashToken(token));
|
||||
});
|
||||
|
||||
it('is read from where the template puts it, or after a plus', () => {
|
||||
const token = 'abcdefghijklmnopqrstuvwx';
|
||||
expect(tokenOf(`Trips <trek+${token}@mail.example.com>`, 'trek+{token}@mail.example.com')).toBe(token);
|
||||
expect(tokenOf(`ab12${token}@in.example.com`, 'ab12{token}@in.example.com')).toBe(token);
|
||||
expect(tokenOf(`box+${token}@example.com`, null)).toBe(token);
|
||||
expect(tokenOf('box+short@example.com', null)).toBeNull();
|
||||
expect(tokenOf('someone@example.com', 'trek+{token}@mail.example.com')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the sealed mail', () => {
|
||||
it('opens to the same bytes, and a broken seal to nothing', () => {
|
||||
const raw = Buffer.from([0x46, 0x72, 0x6f, 0x6d, 0x3a, 0x20, 0xe9, 0x00, 0xff]);
|
||||
expect(openMail(sealMail(raw))?.equals(raw)).toBe(true);
|
||||
expect(openMail(null)).toBeNull();
|
||||
expect(openMail('enc:v1:broken')).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* What the mail import tells people: the forwarder gets keep or undo, the
|
||||
* other members of the trip a notice, a waiting mail one note; a failed
|
||||
* notification never fails the write. The notification's undo button reaches
|
||||
* the inbox's undo with the person who tapped it.
|
||||
*/
|
||||
import { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service';
|
||||
import { MailIngestNotificationActions } from '../../../../src/nest/mail-ingest/mail-ingest-notification-actions';
|
||||
import { MailIngestNotifyService } from '../../../../src/nest/mail-ingest/mail-ingest-notify.service';
|
||||
import { NotificationsService } from '../../../../src/nest/notifications/notifications.service';
|
||||
import { Test } from '@nestjs/testing';
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../../../src/nest/audit/audit-log.logger', () => ({
|
||||
LOG_LEVEL: 'error',
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
const notifications = { send: vi.fn(), registerResponseAction: vi.fn() };
|
||||
const actions = { undo: vi.fn(async () => ({})) };
|
||||
let notify: MailIngestNotifyService;
|
||||
let buttons: MailIngestNotificationActions;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
notifications.send.mockResolvedValue({ attempted: 1, delivered: 1 });
|
||||
const moduleRef = await Test.createTestingModule({
|
||||
providers: [
|
||||
MailIngestNotifyService,
|
||||
MailIngestNotificationActions,
|
||||
{ provide: NotificationsService, useValue: notifications },
|
||||
{ provide: MailIngestActionsService, useValue: actions },
|
||||
],
|
||||
}).compile();
|
||||
notify = moduleRef.get(MailIngestNotifyService);
|
||||
buttons = moduleRef.get(MailIngestNotificationActions);
|
||||
});
|
||||
|
||||
const MIA = { id: 3, username: 'mia', email: 'mia@example.com', role: 'user' };
|
||||
const BOOKING = { candidateId: 11, tripId: 7, tripTitle: 'Japan', title: 'LH 716', type: 'flight' };
|
||||
|
||||
describe('MailIngestNotifyService', () => {
|
||||
it('asks the forwarder to keep or undo, and tells the trip’s other members', async () => {
|
||||
await notify.bookingAdded(MIA, BOOKING);
|
||||
const [forwarder, trip] = notifications.send.mock.calls.map(([payload]) => payload);
|
||||
expect(forwarder).toMatchObject({
|
||||
event: 'mail_booking_added',
|
||||
scope: 'user',
|
||||
targetId: 3,
|
||||
params: { booking: 'LH 716', trip: 'Japan', tripId: '7', role: 'forwarder' },
|
||||
inApp: {
|
||||
type: 'boolean',
|
||||
negativeTextKey: 'notif.action.undo',
|
||||
negativeCallback: { action: 'mail_ingest_undo', payload: { candidateId: 11 } },
|
||||
},
|
||||
});
|
||||
expect(trip).toMatchObject({ scope: 'trip', targetId: 7, actorId: 3, params: { role: 'traveller', actor: 'mia' } });
|
||||
});
|
||||
|
||||
it('sends one note for waiting bookings, and survives a failed send', async () => {
|
||||
notifications.send.mockRejectedValue(new Error('smtp down'));
|
||||
await expect(notify.bookingsPending(3, 2)).resolves.toBeUndefined();
|
||||
expect(notifications.send).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ event: 'mail_booking_pending', params: { count: '2' } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MailIngestNotificationActions', () => {
|
||||
it('registers undo and keep, and undoes the named booking for the person who tapped', async () => {
|
||||
buttons.onModuleInit();
|
||||
expect(notifications.registerResponseAction.mock.calls.map(([action]) => action)).toEqual([
|
||||
'mail_ingest_undo',
|
||||
'mail_ingest_keep',
|
||||
]);
|
||||
await buttons.undo({ candidateId: 11 }, 3);
|
||||
expect(actions.undo).toHaveBeenCalledWith(11, 3, false);
|
||||
await expect(buttons.undo({}, 3)).rejects.toThrow('This notification names no booking');
|
||||
const keep = notifications.registerResponseAction.mock.calls[1][1] as () => Promise<void>;
|
||||
await expect(keep()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Whether the mail import runs and how: off until switched on, the TREK Cloud
|
||||
* relay as the only transport of a managed install, the address template only
|
||||
* with its placeholder, and a person's switches all on until they change one.
|
||||
*/
|
||||
import { MailIngestPrefs } from '../../../../src/db/entities/MailIngestPrefs.entity';
|
||||
import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service';
|
||||
import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service';
|
||||
import { SettingsService } from '../../../../src/nest/settings/settings.service';
|
||||
import { getRepositoryToken } from '@mikro-orm/nestjs';
|
||||
import { Test } from '@nestjs/testing';
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const stored = new Map<string, string>();
|
||||
const settings = { getAppSetting: vi.fn(async (key: string) => stored.get(key) ?? null) };
|
||||
const env = { isManaged: vi.fn(() => false) };
|
||||
const prefs = { findForUser: vi.fn(), upsertForUser: vi.fn(async () => undefined) };
|
||||
let service: MailIngestSettingsService;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
stored.clear();
|
||||
env.isManaged.mockReturnValue(false);
|
||||
prefs.findForUser.mockResolvedValue(null);
|
||||
const moduleRef = await Test.createTestingModule({
|
||||
providers: [
|
||||
MailIngestSettingsService,
|
||||
{ provide: SettingsService, useValue: settings },
|
||||
{ provide: RuntimeEnvService, useValue: env },
|
||||
{ provide: getRepositoryToken(MailIngestPrefs), useValue: prefs },
|
||||
],
|
||||
}).compile();
|
||||
service = moduleRef.get(MailIngestSettingsService);
|
||||
});
|
||||
|
||||
describe('MailIngestSettingsService', () => {
|
||||
it('is off until switched on', async () => {
|
||||
expect(await service.isAvailable()).toBe(false);
|
||||
stored.set('mail_ingest_enabled', 'true');
|
||||
expect(await service.isAvailable()).toBe(true);
|
||||
});
|
||||
|
||||
it('reads the transport, off for anything it does not know, the relay on a managed install', async () => {
|
||||
expect(await service.transport()).toBe('off');
|
||||
stored.set('mail_ingest_transport', 'imap');
|
||||
expect(await service.transport()).toBe('imap');
|
||||
stored.set('mail_ingest_transport', 'carrier-pigeon');
|
||||
expect(await service.transport()).toBe('off');
|
||||
env.isManaged.mockReturnValue(true);
|
||||
expect(await service.transport()).toBe('cloud');
|
||||
});
|
||||
|
||||
it('takes an address template only with its placeholder', async () => {
|
||||
expect(await service.addressTemplate()).toBeNull();
|
||||
stored.set('mail_ingest_address_template', 'trips@example.com');
|
||||
expect(await service.addressTemplate()).toBeNull();
|
||||
stored.set('mail_ingest_address_template', ' trek+{token}@example.com ');
|
||||
expect(await service.addressTemplate()).toBe('trek+{token}@example.com');
|
||||
});
|
||||
|
||||
it('reads the daily AI limit, ten when unset or broken', async () => {
|
||||
expect(await service.aiDailyCap()).toBe(10);
|
||||
stored.set('mail_ingest_ai_per_user_daily', '3');
|
||||
expect(await service.aiDailyCap()).toBe(3);
|
||||
stored.set('mail_ingest_ai_per_user_daily', '-1');
|
||||
expect(await service.aiDailyCap()).toBe(10);
|
||||
});
|
||||
|
||||
it('takes larger mails on the TREK Cloud, and the admin’s file types', async () => {
|
||||
expect(service.maxMailBytes()).toBe(10 * 1024 * 1024);
|
||||
env.isManaged.mockReturnValue(true);
|
||||
expect(service.maxMailBytes()).toBe(25 * 1024 * 1024);
|
||||
expect(await service.allowedFileTypes()).toContain('pdf');
|
||||
stored.set('allowed_file_types', 'pdf');
|
||||
expect(await service.allowedFileTypes()).toBe('pdf');
|
||||
});
|
||||
|
||||
it('has every switch on for a person who never changed one, and writes only what changes', async () => {
|
||||
expect(await service.prefsOf(4)).toEqual({ enabled: true, ai_enabled: true, auto_add: true });
|
||||
prefs.findForUser.mockResolvedValue({ id: 1, user_id: 4, enabled: 1, ai_enabled: 0, auto_add: 1 });
|
||||
expect(await service.updatePrefs(4, { ai_enabled: false })).toEqual({
|
||||
enabled: true,
|
||||
ai_enabled: false,
|
||||
auto_add: true,
|
||||
});
|
||||
expect(prefs.upsertForUser).toHaveBeenCalledWith(4, { ai_enabled: 0 });
|
||||
await service.updatePrefs(4, { enabled: true, auto_add: false });
|
||||
expect(prefs.upsertForUser).toHaveBeenLastCalledWith(4, { enabled: 1, auto_add: 0 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* The forwarding addresses inside MailIngestModule over a snapshot test db: a
|
||||
* person's own address and a trip's, renewing that retires the old token at
|
||||
* once, and a trip address that stops working when its owner can no longer
|
||||
* add bookings to the trip.
|
||||
*/
|
||||
import { MailIngestAddresses } from '../../../../src/db/entities/MailIngestAddresses.entity';
|
||||
import { TripMembers } from '../../../../src/db/entities/TripMembers.entity';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import { MailIngestAddressService } from '../../../../src/nest/mail-ingest/mail-ingest-address.service';
|
||||
import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service';
|
||||
import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service';
|
||||
import { MailIngestModule } from '../../../../src/nest/mail-ingest/mail-ingest.module';
|
||||
import { PermissionsService } from '../../../../src/nest/permissions/permissions.service';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { deleteRows, findRow } from '../../../helpers/factories/rows';
|
||||
import { setAppSetting } from '../../../helpers/factories/settings';
|
||||
import { addTripMember, makeTrip } from '../../../helpers/factories/trips';
|
||||
import { makeUser } from '../../../helpers/factories/users';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestModule, type TestModule } from '../../../helpers/test-module';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const db = createSnapshotTestDb();
|
||||
let t: TestModule;
|
||||
const orm = () => t.get(MikroORM);
|
||||
const addresses = () => t.get(MailIngestAddressService);
|
||||
const TEMPLATE = 'trek+{token}@mail.example.com';
|
||||
const permissions = { checkPermission: vi.fn(async () => true) };
|
||||
|
||||
beforeAll(async () => {
|
||||
t = await createTestModule({
|
||||
db,
|
||||
imports: [MailIngestModule],
|
||||
overrides: [{ provide: PermissionsService, useValue: permissions }],
|
||||
});
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetTestDb(db);
|
||||
vi.clearAllMocks();
|
||||
permissions.checkPermission.mockResolvedValue(true);
|
||||
await setAppSetting(orm(), 'mail_ingest_enabled', 'true');
|
||||
await setAppSetting(orm(), 'mail_ingest_address_template', TEMPLATE);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
db.close();
|
||||
});
|
||||
|
||||
const tokenIn = (address: string) => /^trek\+([a-z2-7]+)@/.exec(address)?.[1] ?? '';
|
||||
function mailTo(address: string, id: string): Buffer {
|
||||
const lines = ['From: mia@example.com', `To: ${address}`, 'Subject: Booking', `Message-ID: <${id}@x>`];
|
||||
return Buffer.from(`${lines.join('\r\n')}\r\n\r\nBooking ABC123`);
|
||||
}
|
||||
|
||||
describe('a person’s own address', () => {
|
||||
it('exists once asked for, and a renewal retires the old token', async () => {
|
||||
const { user } = await makeUser(orm());
|
||||
expect(await addresses().ownAddress(user.id)).toBeNull();
|
||||
|
||||
const first = await addresses().renewOwn(user.id);
|
||||
expect(first?.address).toMatch(/^trek\+[a-z2-7]{24}@mail\.example\.com$/);
|
||||
expect(first?.trip_id).toBeNull();
|
||||
expect((await addresses().ownAddress(user.id))?.address).toBe(first?.address);
|
||||
|
||||
const second = await addresses().renewOwn(user.id);
|
||||
expect(second?.address).not.toBe(first?.address);
|
||||
const intake = t.get(MailIngestIntakeService);
|
||||
expect(await intake.receive(mailTo(first?.address ?? '', 'old'), 'webhook')).toEqual({
|
||||
status: 'refused',
|
||||
reason: 'unknown_address',
|
||||
});
|
||||
expect(await intake.receive(mailTo(second?.address ?? '', 'new'), 'webhook')).toMatchObject({ status: 'accepted' });
|
||||
const row = await findRow(orm(), MailIngestAddresses, { id: second?.id });
|
||||
expect(row?.last_used_at).not.toBeNull();
|
||||
});
|
||||
|
||||
it('cannot be made without an address template, and is not shown without one', async () => {
|
||||
const { user } = await makeUser(orm());
|
||||
await addresses().ensureOwn(user.id);
|
||||
await setAppSetting(orm(), 'mail_ingest_address_template', '');
|
||||
await expect(addresses().renewOwn(user.id)).rejects.toThrow(DomainError);
|
||||
expect(await addresses().ownAddress(user.id)).toBeNull();
|
||||
});
|
||||
|
||||
it('takes no mail while its person has the import off', async () => {
|
||||
const { user } = await makeUser(orm());
|
||||
const own = await addresses().renewOwn(user.id);
|
||||
await t.get(MailIngestSettingsService).updatePrefs(user.id, { enabled: false });
|
||||
expect(await addresses().resolve([own?.address ?? ''])).toBeNull();
|
||||
await t.get(MailIngestSettingsService).updatePrefs(user.id, { enabled: true });
|
||||
expect((await addresses().resolve([`Mia <${own?.address}>`]))?.person.id).toBe(user.id);
|
||||
expect(tokenIn(own?.address ?? '')).toHaveLength(24);
|
||||
});
|
||||
});
|
||||
|
||||
describe('a trip address', () => {
|
||||
it('belongs to the member who made it and stops working when they leave the trip', async () => {
|
||||
const { user: owner } = await makeUser(orm());
|
||||
const { user: member } = await makeUser(orm());
|
||||
const trip = await makeTrip(orm(), owner.id);
|
||||
await addTripMember(orm(), trip.id, member.id);
|
||||
|
||||
const made = await addresses().renewTripAddress(member.id, trip.id);
|
||||
expect(made?.trip_id).toBe(trip.id);
|
||||
expect((await addresses().tripAddress(member.id, trip.id))?.address).toBe(made?.address);
|
||||
expect(await addresses().tripAddress(owner.id, trip.id)).toBeNull();
|
||||
expect((await addresses().resolve([made?.address ?? '']))?.address.trip_id).toBe(trip.id);
|
||||
|
||||
await deleteRows(orm(), TripMembers, { trip: trip.id, user: member.id });
|
||||
expect(await addresses().resolve([made?.address ?? ''])).toBeNull();
|
||||
await expect(addresses().tripAddress(member.id, trip.id)).rejects.toThrow('Trip not found');
|
||||
});
|
||||
|
||||
it('is refused to a member who may not add bookings, and revoked on request', async () => {
|
||||
const { user: owner } = await makeUser(orm());
|
||||
const trip = await makeTrip(orm(), owner.id);
|
||||
const made = await addresses().renewTripAddress(owner.id, trip.id);
|
||||
await addresses().revokeTripAddress(owner.id, trip.id);
|
||||
expect(await addresses().tripAddress(owner.id, trip.id)).toBeNull();
|
||||
expect(await addresses().resolve([made?.address ?? ''])).toBeNull();
|
||||
|
||||
permissions.checkPermission.mockResolvedValue(false);
|
||||
await expect(addresses().renewTripAddress(owner.id, trip.id)).rejects.toThrow('Trip not found');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,418 @@
|
||||
/**
|
||||
* The mail import end to end inside MailIngestModule, over a snapshot test db
|
||||
* with its real repositories and the real reservation, place and cost writes.
|
||||
* Only what reaches outside is replaced: the AI reading (the walker stays
|
||||
* real), kitinerary, file storage, geocoding and the notifications.
|
||||
*/
|
||||
import { BudgetItems } from '../../../../src/db/entities/BudgetItems.entity';
|
||||
import { MailIngestCandidates } from '../../../../src/db/entities/MailIngestCandidates.entity';
|
||||
import { MailIngestMessages } from '../../../../src/db/entities/MailIngestMessages.entity';
|
||||
import { Reservations } from '../../../../src/db/entities/Reservations.entity';
|
||||
import { importExternalId } from '../../../../src/nest/booking-import/booking-origin';
|
||||
import { BookingImportService } from '../../../../src/nest/booking-import/booking-import.service';
|
||||
import { DomainError } from '../../../../src/nest/common/domain-error';
|
||||
import { FilesService } from '../../../../src/nest/files/files.service';
|
||||
import { MailReadService } from '../../../../src/nest/llm-parse/mail-read.service';
|
||||
import type { MailReading as AiReading } from '../../../../src/nest/llm-parse/mail-read.types';
|
||||
import { walkMail } from '../../../../src/nest/llm-parse/mime-walker';
|
||||
import { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service';
|
||||
import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service';
|
||||
import { MailIngestProcessJob } from '../../../../src/nest/mail-ingest/mail-ingest-process.job';
|
||||
import { MailIngestRetentionJob } from '../../../../src/nest/mail-ingest/mail-ingest-retention.job';
|
||||
import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service';
|
||||
import { MailIngestModule } from '../../../../src/nest/mail-ingest/mail-ingest.module';
|
||||
import { MapsService } from '../../../../src/nest/maps/maps.service';
|
||||
import { NotificationsService } from '../../../../src/nest/notifications/notifications.service';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { countRows, findRow, findRows, updateRows } from '../../../helpers/factories/rows';
|
||||
import { setAddonEnabled, setAppSetting } from '../../../helpers/factories/settings';
|
||||
import { addTripMember, makeTrip } from '../../../helpers/factories/trips';
|
||||
import { makeUser } from '../../../helpers/factories/users';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestModule, type TestModule } from '../../../helpers/test-module';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import type { BookingImportPreviewItem } from '@trek/shared';
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('../../../../src/nest/audit/audit-log.logger', () => ({
|
||||
LOG_LEVEL: 'error',
|
||||
logInfo: vi.fn(),
|
||||
logDebug: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
}));
|
||||
|
||||
const NOW = new Date('2030-01-15T10:00:00Z');
|
||||
const endpoint = (role: 'from' | 'to', name: string, code: string, lat: number, lng: number) => ({
|
||||
role,
|
||||
sequence: role === 'from' ? 0 : 1,
|
||||
name,
|
||||
code,
|
||||
lat,
|
||||
lng,
|
||||
timezone: null,
|
||||
local_time: null,
|
||||
local_date: null,
|
||||
});
|
||||
const FLIGHT: BookingImportPreviewItem = {
|
||||
type: 'flight',
|
||||
title: 'LH 716 Tokyo to Osaka',
|
||||
reservation_time: '2030-03-02T10:00',
|
||||
reservation_end_time: '2030-03-02T11:15',
|
||||
confirmation_number: 'ABC123',
|
||||
metadata: { flight_number: 'LH716', price: '120,50', priceCurrency: 'EUR' },
|
||||
endpoints: [
|
||||
endpoint('from', 'Tokyo Haneda', 'HND', 35.55, 139.78),
|
||||
endpoint('to', 'Osaka Itami', 'ITM', 34.78, 135.44),
|
||||
],
|
||||
source: { fileName: 'mail.eml', index: 0 },
|
||||
};
|
||||
const BODY = 'Booking reference ABC123. Flight LH 716 from Tokyo to Osaka on 2030-03-02.';
|
||||
|
||||
const machine = vi.fn(async (): Promise<BookingImportPreviewItem[]> => [FLIGHT]);
|
||||
const bookingImport = {
|
||||
readMailBookings: machine,
|
||||
mapAiReservations: vi.fn(async (): Promise<BookingImportPreviewItem[]> => [FLIGHT]),
|
||||
externalKey: (item: BookingImportPreviewItem) => importExternalId(item),
|
||||
};
|
||||
const aiRead = vi.fn();
|
||||
const reader = { walk: (raw: Buffer) => walkMail(raw), isAvailable: vi.fn(async () => false), read: aiRead };
|
||||
const files = {
|
||||
createFileFromBytes: vi.fn(async () => ({ id: 77 })),
|
||||
softDeleteFile: vi.fn(async () => undefined),
|
||||
};
|
||||
const notifications = { send: vi.fn(async () => ({ attempted: 1, delivered: 1 })), registerResponseAction: vi.fn() };
|
||||
|
||||
const db = createSnapshotTestDb();
|
||||
let t: TestModule;
|
||||
const orm = () => t.get(MikroORM);
|
||||
|
||||
beforeAll(async () => {
|
||||
t = await createTestModule({
|
||||
db,
|
||||
imports: [MailIngestModule],
|
||||
overrides: [
|
||||
{ provide: BookingImportService, useValue: bookingImport },
|
||||
{ provide: MailReadService, useValue: reader },
|
||||
{ provide: FilesService, useValue: files },
|
||||
{ provide: MapsService, useValue: { geocodeQuery: vi.fn(async () => null) } },
|
||||
{ provide: NotificationsService, useValue: notifications },
|
||||
],
|
||||
});
|
||||
});
|
||||
beforeEach(async () => {
|
||||
resetTestDb(db);
|
||||
vi.clearAllMocks();
|
||||
machine.mockResolvedValue([FLIGHT]);
|
||||
reader.isAvailable.mockResolvedValue(false);
|
||||
await setAppSetting(orm(), 'mail_ingest_enabled', 'true');
|
||||
await setAddonEnabled(orm(), 'budget', true);
|
||||
await setAddonEnabled(orm(), 'documents', true);
|
||||
});
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
db.close();
|
||||
});
|
||||
|
||||
const intake = () => t.get(MailIngestIntakeService);
|
||||
const job = () => t.get(MailIngestProcessJob);
|
||||
const actions = () => t.get(MailIngestActionsService);
|
||||
|
||||
function eml(id: string, subject = 'Fwd: Your flight LH 716', body = BODY): Buffer {
|
||||
return Buffer.from(
|
||||
[
|
||||
'From: Mia <mia@example.com>',
|
||||
'To: trips@example.com',
|
||||
`Subject: ${subject}`,
|
||||
`Message-ID: <${id}@example.com>`,
|
||||
'Date: Mon, 1 Jan 2030 10:00:00 +0000',
|
||||
'Content-Type: text/plain; charset=utf-8',
|
||||
'',
|
||||
body,
|
||||
].join('\r\n'),
|
||||
);
|
||||
}
|
||||
|
||||
async function person() {
|
||||
const { user } = await makeUser(orm(), { email: 'mia@example.com', username: 'mia' });
|
||||
return user;
|
||||
}
|
||||
|
||||
/** Uploads a mail for the person and runs the pipeline over it; answers the mail's id. */
|
||||
async function forward(userId: number, raw: Buffer): Promise<number> {
|
||||
const taken = await intake().upload(raw, userId);
|
||||
if (taken.status !== 'accepted') throw new Error(`the mail was not taken in: ${JSON.stringify(taken)}`);
|
||||
expect(await job().tick(NOW)).toBe(1);
|
||||
return taken.messageId;
|
||||
}
|
||||
|
||||
const entriesOf = (messageId: number) =>
|
||||
findRows(orm(), MailIngestCandidates, { message: messageId }, { position: 'asc' });
|
||||
|
||||
describe('taking a mail in', () => {
|
||||
it('refuses everything while the import is off', async () => {
|
||||
const user = await person();
|
||||
await setAppSetting(orm(), 'mail_ingest_enabled', 'false');
|
||||
expect(await intake().upload(eml('a'), user.id)).toEqual({ status: 'refused', reason: 'unavailable' });
|
||||
expect(await job().tick(NOW)).toBe(0);
|
||||
});
|
||||
|
||||
it('stores a mail once, and refuses what is no mail', async () => {
|
||||
const user = await person();
|
||||
const first = await intake().upload(eml('a'), user.id);
|
||||
expect(first.status).toBe('accepted');
|
||||
expect(await intake().upload(eml('a'), user.id)).toMatchObject({ status: 'duplicate' });
|
||||
expect(await intake().upload(Buffer.from('nothing to see'), user.id)).toEqual({
|
||||
status: 'refused',
|
||||
reason: 'not_a_mail',
|
||||
});
|
||||
const stored = await findRow(orm(), MailIngestMessages, { user: user.id });
|
||||
expect(stored?.raw_eml_enc).toMatch(/^enc:v1:/);
|
||||
});
|
||||
|
||||
it('refuses a mail to an address nobody holds', async () => {
|
||||
expect(await intake().receive(eml('b'), 'webhook', ['trek+aaaaaaaaaaaaaaaaaaaaaaaa@example.com'])).toEqual({
|
||||
status: 'refused',
|
||||
reason: 'unknown_address',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('a booking that passes every check', () => {
|
||||
it('lands in its trip with its cost and documents, and tells the forwarder and the trip', async () => {
|
||||
const user = await person();
|
||||
const { user: friend } = await makeUser(orm());
|
||||
const trip = await makeTrip(orm(), user.id, { title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
await addTripMember(orm(), trip.id, friend.id);
|
||||
|
||||
const messageId = await forward(user.id, eml('flight'));
|
||||
|
||||
const [entry] = await entriesOf(messageId);
|
||||
expect(entry).toMatchObject({
|
||||
resolution: 'auto_applied',
|
||||
reason: null,
|
||||
resolved_trip_id: trip.id,
|
||||
source: 'kitinerary',
|
||||
});
|
||||
const booking = await findRow(orm(), Reservations, { trip: trip.id });
|
||||
expect(booking).toMatchObject({
|
||||
title: 'LH 716 Tokyo to Osaka',
|
||||
external_source: 'mail',
|
||||
external_id: importExternalId(FLIGHT),
|
||||
sync_enabled: 0,
|
||||
needs_review: 0,
|
||||
});
|
||||
expect(await findRow(orm(), BudgetItems, { trip: trip.id })).toMatchObject({ total_price: 120.5 });
|
||||
expect(files.createFileFromBytes).toHaveBeenCalledWith(
|
||||
trip.id,
|
||||
expect.objectContaining({ originalname: 'mail.txt' }),
|
||||
user.id,
|
||||
expect.objectContaining({ reservation_id: booking?.id }),
|
||||
);
|
||||
expect(JSON.parse(String((await entriesOf(messageId))[0].created_ids_json)).file_ids).toEqual([77]);
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({ status: 'processed' });
|
||||
const events = notifications.send.mock.calls.map(([payload]) => [payload.event, payload.scope]);
|
||||
expect(events).toEqual([
|
||||
['mail_booking_added', 'user'],
|
||||
['mail_booking_added', 'trip'],
|
||||
]);
|
||||
expect(t.realtime.broadcastMock).toHaveBeenCalledWith(trip.id, 'reservation:created', expect.anything());
|
||||
});
|
||||
|
||||
it('is undone by its forwarder: the booking and its cost go, the same mail again waits in the inbox', async () => {
|
||||
const user = await person();
|
||||
const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
|
||||
await actions().undo(entry.id, user.id, false, NOW);
|
||||
expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(0);
|
||||
expect(await countRows(orm(), BudgetItems, { trip: trip.id })).toBe(0);
|
||||
expect(files.softDeleteFile).toHaveBeenCalledWith(77);
|
||||
await expect(actions().undo(entry.id, user.id, false, NOW)).rejects.toThrow(DomainError);
|
||||
|
||||
const [again] = await entriesOf(await forward(user.id, eml('flight-again')));
|
||||
expect(again).toMatchObject({ resolution: 'pending', reason: 'possible_duplicate' });
|
||||
});
|
||||
|
||||
it('asks before undoing a booking that was edited since', async () => {
|
||||
const user = await person();
|
||||
const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
await updateRows(orm(), Reservations, { trip: trip.id }, { notes: 'Window seat please' });
|
||||
|
||||
const refused = await actions()
|
||||
.undo(entry.id, user.id, false, NOW)
|
||||
.catch((err: unknown) => err);
|
||||
expect(refused).toBeInstanceOf(DomainError);
|
||||
expect((refused as DomainError).getResponse()).toMatchObject({ code: 'needs_confirmation', affects: ['edited'] });
|
||||
await actions().undo(entry.id, user.id, true, NOW);
|
||||
expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(0);
|
||||
});
|
||||
|
||||
it('is not written twice when the trip holds it already', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
await forward(user.id, eml('flight'));
|
||||
const [second] = await entriesOf(await forward(user.id, eml('flight-forwarded-again')));
|
||||
expect(second).toMatchObject({ resolution: 'duplicate' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('a booking that waits in the inbox', () => {
|
||||
it('names why: no trip in its dates, and the person hears of it once', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-06-01', end_date: '2030-06-10' });
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
expect(entry).toMatchObject({ resolution: 'pending', reason: 'no_trip_in_range' });
|
||||
expect(notifications.send).toHaveBeenCalledWith(expect.objectContaining({ event: 'mail_booking_pending' }));
|
||||
});
|
||||
|
||||
it('asks first when the person said so', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false });
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
expect(entry).toMatchObject({ resolution: 'pending', reason: 'ask_first', suggested_trip_id: expect.any(Number) });
|
||||
});
|
||||
|
||||
it('keeps a mail without a booking for its ways out, with why the AI did not read it', async () => {
|
||||
const user = await person();
|
||||
machine.mockResolvedValue([]);
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('nothing')));
|
||||
expect(entry).toMatchObject({ resolution: 'pending', reason: 'ai_disabled', item_json: null });
|
||||
|
||||
const prefill = await actions().prefill(entry.id, user.id);
|
||||
expect(prefill.item).toMatchObject({
|
||||
type: 'flight',
|
||||
confirmation_number: 'ABC123',
|
||||
reservation_time: '2030-03-02',
|
||||
});
|
||||
|
||||
const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
const filed = await actions().fileToTrip(entry.id, user.id, trip.id, NOW);
|
||||
expect(filed).toMatchObject({ resolution: 'filed', resolved_trip_id: trip.id });
|
||||
});
|
||||
|
||||
it('is added by hand to the trip the person picks, and can be discarded instead', async () => {
|
||||
const user = await person();
|
||||
const trip = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false });
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
const applied = await actions().applyToTrip(entry.id, user.id, trip.id, NOW);
|
||||
expect(applied).toMatchObject({ resolution: 'applied', resolved_trip_id: trip.id });
|
||||
expect(await countRows(orm(), Reservations, { trip: trip.id })).toBe(1);
|
||||
await expect(actions().discard(entry.id, user.id, NOW)).rejects.toThrow('This entry was decided already');
|
||||
});
|
||||
});
|
||||
|
||||
describe('mails that are no booking', () => {
|
||||
it('drops a newsletter without reading it', async () => {
|
||||
const user = await person();
|
||||
const raw = Buffer.from(
|
||||
[
|
||||
'From: news@shop.example',
|
||||
'Subject: Our autumn deals',
|
||||
'List-Unsubscribe: <mailto:off@shop.example>',
|
||||
'Message-ID: <news@shop.example>',
|
||||
'',
|
||||
'Fresh deals for everyone who loves to shop with us.',
|
||||
].join('\r\n'),
|
||||
);
|
||||
const messageId = await forward(user.id, raw);
|
||||
expect(machine).not.toHaveBeenCalled();
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({
|
||||
status: 'discarded',
|
||||
reason: 'newsletter',
|
||||
});
|
||||
});
|
||||
|
||||
it('shows a forwarding confirmation’s code in the inbox', async () => {
|
||||
const user = await person();
|
||||
const messageId = await forward(
|
||||
user.id,
|
||||
eml('confirm', 'Gmail Forwarding Confirmation', 'Confirmation code: 612345678 for forwarding.'),
|
||||
);
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({
|
||||
forwarding_code: '612345678',
|
||||
reason: 'forwarding_confirmation',
|
||||
});
|
||||
expect((await entriesOf(messageId))[0]).toMatchObject({ reason: 'forwarding_confirmation' });
|
||||
});
|
||||
|
||||
it('discards a mail the AI read and found no booking in', async () => {
|
||||
const user = await person();
|
||||
reader.isAvailable.mockResolvedValue(true);
|
||||
machine.mockResolvedValue([]);
|
||||
aiRead.mockResolvedValue({ outcome: 'empty', bookings: [], documentText: BODY, imageOnly: false } as AiReading);
|
||||
const messageId = await forward(user.id, eml('chat'));
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: messageId })).toMatchObject({
|
||||
status: 'discarded',
|
||||
reason: 'not_a_booking',
|
||||
ai_used: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('the AI’s trip pick', () => {
|
||||
it('decides between two trips that fit only where the booking lies near the pick', async () => {
|
||||
const user = await person();
|
||||
reader.isAvailable.mockResolvedValue(true);
|
||||
const japan = await makeTrip(orm(), user.id, { title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
await makeTrip(orm(), user.id, { title: 'Work', start_date: '2030-02-25', end_date: '2030-03-05' });
|
||||
aiRead.mockImplementation(async (input: { trips: Array<{ ref: string; title: string }> }) => ({
|
||||
outcome: 'ok',
|
||||
documentText: BODY,
|
||||
imageOnly: false,
|
||||
bookings: [
|
||||
{
|
||||
kind: 'new_booking',
|
||||
reservations: [{ '@type': 'FlightReservation' }],
|
||||
tripRef: input.trips.find((trip) => trip.title === 'Japan')?.ref ?? null,
|
||||
tripConfidence: 0.9,
|
||||
uncertainFields: [],
|
||||
suggestedNewTrip: null,
|
||||
},
|
||||
],
|
||||
}));
|
||||
const [entry] = await entriesOf(await forward(user.id, eml('flight')));
|
||||
// Japan has no place near Osaka the server could confirm: the inbox suggests it.
|
||||
expect(entry).toMatchObject({ resolution: 'pending', reason: 'trip_ambiguous', suggested_trip_id: japan.id });
|
||||
expect(entry.source).toBe('ai');
|
||||
});
|
||||
});
|
||||
|
||||
describe('a mail that keeps failing', () => {
|
||||
it('is tried again later and given up after its last try', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
machine.mockRejectedValue(new Error('kitinerary crashed'));
|
||||
const taken = await intake().upload(eml('poison'), user.id);
|
||||
if (taken.status !== 'accepted') throw new Error('not taken in');
|
||||
expect(await job().tick(NOW)).toBe(1);
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: taken.messageId })).toMatchObject({ status: 'queued' });
|
||||
expect(await job().tick(new Date(NOW.getTime() + 60 * 60 * 1000))).toBe(1);
|
||||
expect(await job().tick(new Date(NOW.getTime() + 2 * 60 * 60 * 1000))).toBe(1);
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: taken.messageId })).toMatchObject({
|
||||
status: 'failed',
|
||||
reason: 'processing_failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('the retention', () => {
|
||||
it('purges a settled mail and keeps one an entry waits on', async () => {
|
||||
const user = await person();
|
||||
await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' });
|
||||
const added = await forward(user.id, eml('flight'));
|
||||
machine.mockResolvedValue([]);
|
||||
const waiting = await forward(user.id, eml('nothing'));
|
||||
|
||||
// Settled long ago: the mail with nothing waiting on it goes, the one an entry waits on stays.
|
||||
const longAgo = { processed_at: '2000-01-01 00:00:00' };
|
||||
await updateRows(orm(), MailIngestMessages, { id: { $in: [added, waiting] } }, longAgo);
|
||||
await t.get(MailIngestRetentionJob).tick(new Date());
|
||||
expect(await findRow(orm(), MailIngestMessages, { id: added })).toMatchObject({ raw_eml_enc: null });
|
||||
expect((await findRow(orm(), MailIngestMessages, { id: waiting }))?.raw_eml_enc).toMatch(/^enc:v1:/);
|
||||
});
|
||||
});
|
||||
@@ -136,6 +136,9 @@ export default defineConfig({
|
||||
// like ferratas until the first full run regenerates it.
|
||||
'src/nest/journey-recordings/**/*.ts': { statements: 80, branches: 80, functions: 80, lines: 80 },
|
||||
'src/nest/llm-parse/**/*.ts': { statements: 91, branches: 85, functions: 85, lines: 94 },
|
||||
// New domain in this change (the booking mail import), unmeasured: pinned at the 80 floor
|
||||
// until the first full run regenerates it.
|
||||
'src/nest/mail-ingest/**/*.ts': { statements: 80, branches: 80, functions: 80, lines: 80 },
|
||||
'src/nest/managed/**/*.ts': { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
'src/nest/maps/**/*.ts': { statements: 93, branches: 86, functions: 97, lines: 96 },
|
||||
'src/nest/mcp-shared/**/*.ts': { statements: 84, branches: 59, functions: 99, lines: 88 },
|
||||
|
||||
@@ -113,7 +113,10 @@ export const MAIL_INGEST_REASON_BY_LLM_OUTCOME: Readonly<Record<LlmParseOutcome,
|
||||
too_large: 'too_large',
|
||||
};
|
||||
|
||||
/** The server's checks on a booking read from a mail, as the inbox lists them (plan G1 to G8). */
|
||||
/**
|
||||
* The server's checks on a booking read from a mail, as the inbox lists them:
|
||||
* plan G1 to G8, and G9, the trip holding the booking already.
|
||||
*/
|
||||
export const mailIngestCheckSchema = z.enum([
|
||||
'schema',
|
||||
'trip',
|
||||
@@ -123,6 +126,7 @@ export const mailIngestCheckSchema = z.enum([
|
||||
'in_text',
|
||||
'assignment',
|
||||
'cancellation',
|
||||
'duplicate',
|
||||
]);
|
||||
export type MailIngestCheck = z.infer<typeof mailIngestCheckSchema>;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user