diff --git a/server/src/db/repositories/MailIngestCandidates.repository.ts b/server/src/db/repositories/MailIngestCandidates.repository.ts index 2e3a5c502..e3247580d 100644 --- a/server/src/db/repositories/MailIngestCandidates.repository.ts +++ b/server/src/db/repositories/MailIngestCandidates.repository.ts @@ -87,6 +87,14 @@ interface MailIngestTripPlaceRow { lng: number; } +/** A trip as the inbox names it to a person who can see it. */ +export interface MailIngestTripRefRow { + id: number; + title: string; + start_date: string | null; + end_date: string | null; +} + /** The person a mail is for, as the pipeline needs them. */ export interface MailIngestPersonRow { id: number; @@ -223,6 +231,47 @@ export class MailIngestCandidatesRepository extends TrekRepository { + let query = this.db() + .selectFrom('mail_ingest_candidates') + .selectAll() + .where('user_id', '=', userId) + .where('resolution', open ? '=' : '<>', 'pending'); + if (beforeId !== null) query = query.where('id', '<', beforeId); + return query.orderBy('id', 'desc').limit(limit).execute(); + } + + /** The entries made since `since`, counted by what became of them. */ + async tallyResolutionsSince(since: DbTimestamp): Promise> { + const rows = await this.db() + .selectFrom('mail_ingest_candidates') + .select((eb) => ['resolution', eb.fn.countAll().as('n')]) + .where('created_at', '>=', since) + .groupBy('resolution') + .execute(); + return rows.map((row) => ({ resolution: row.resolution, n: Number(row.n) })); + } + + /** How many entries wait for each person who has any, the most first; who they are is left out. */ + async pendingPerUser(): Promise { + const rows = await this.db() + .selectFrom('mail_ingest_candidates') + .select((eb) => ['user_id', eb.fn.countAll().as('n')]) + .where('resolution', '=', 'pending') + .groupBy('user_id') + .execute(); + return rows.map((row) => Number(row.n)).sort((a, b) => b - a); + } + /** Whether a booking of this content went into the trip from a mail and was undone: never added again on its own. */ async wasUndoneInTrip(tripId: number, contentHash: string): Promise { const row = await this.db() @@ -299,6 +348,31 @@ export class MailIngestCandidatesRepository extends TrekRepository { + if (tripIds.length === 0) return []; + return this.db() + .selectFrom('trips as t') + .select(['t.id', 't.title', 't.start_date', 't.end_date']) + .where('t.id', 'in', [...tripIds]) + .where((eb) => + eb.or([ + eb('t.user_id', '=', userId), + eb.exists( + eb + .selectFrom('trip_members as m') + .select('m.id') + .whereRef('m.trip_id', '=', 't.id') + .where('m.user_id', '=', userId), + ), + ]), + ) + .execute(); + } + /** The dated days of these trips, by date. */ async listDatedDays(tripIds: readonly number[]): Promise> { if (tripIds.length === 0) return []; diff --git a/server/src/db/repositories/MailIngestMessages.repository.ts b/server/src/db/repositories/MailIngestMessages.repository.ts index 08ea85d58..e1d466251 100644 --- a/server/src/db/repositories/MailIngestMessages.repository.ts +++ b/server/src/db/repositories/MailIngestMessages.repository.ts @@ -35,6 +35,9 @@ export interface MailIngestMessageRow { const _mailIngestMessageRowKeys: AssertRowKeys = true; +/** A mail as the inbox lists it: everything but the stored mail and its sender's signature data. */ +export type MailIngestMessageHead = Omit; + /** A mail as a transport hands it in, before anything has read it. */ interface MailIngestMessageInsert { transport: string; @@ -112,6 +115,62 @@ export class MailIngestMessagesRepository extends TrekRepository { + if (ids.length === 0) return []; + return this.db() + .selectFrom('mail_ingest_messages') + .select([ + 'id', + 'transport', + 'message_id_hdr', + 'dedupe_hash', + 'address_id', + 'user_id', + 'trip_id', + 'from_addr', + 'from_verified', + 'subject', + 'received_at', + 'size', + 'raw_purged_at', + 'status', + 'reason', + 'hints', + 'attachments', + 'forwarding_code', + 'attempts', + 'next_attempt_at', + 'lease_until', + 'ai_used', + 'processed_at', + ]) + .where('id', 'in', [...ids]) + .execute(); + } + + /** The mails that came in since `since`, counted by status and reason, for the admin's status card. */ + async tallySince(since: DbTimestamp): Promise> { + const rows = await this.db() + .selectFrom('mail_ingest_messages') + .select((eb) => ['status', 'reason', eb.fn.countAll().as('n')]) + .where('received_at', '>=', since) + .groupBy(['status', 'reason']) + .execute(); + return rows.map((row) => ({ status: row.status, reason: row.reason, n: Number(row.n) })); + } + + /** How many mails, of everyone, an AI read since `since`. */ + async countAllAiReadSince(since: DbTimestamp): Promise { + const row = await this.db() + .selectFrom('mail_ingest_messages') + .select((eb) => eb.fn.countAll().as('n')) + .where('ai_used', '=', 1) + .where('received_at', '>=', since) + .executeTakeFirstOrThrow(); + return Number(row.n); + } + /** How many mails came in on the address since `since`, for its daily limit. */ async countForAddressSince(addressId: number, since: DbTimestamp): Promise { const row = await this.db() diff --git a/server/src/nest/mail-ingest/mail-ingest-actions.service.ts b/server/src/nest/mail-ingest/mail-ingest-actions.service.ts index 87a3f9dd8..54aff5821 100644 --- a/server/src/nest/mail-ingest/mail-ingest-actions.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-actions.service.ts @@ -11,7 +11,7 @@ import type { MailIngestCandidatesRepository, } from '../../db/repositories/MailIngestCandidates.repository'; import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; -import { dbNow, parseDbTimestamp } from '../../db/types'; +import { dbNow, parseDbTimestamp, type DbTimestamp } from '../../db/types'; import { decodeJson, encodeJson } from '../../utils/json-column'; import { AirportsService } from '../airports/airports.service'; import { DemoService } from '../common/demo.service'; @@ -19,10 +19,11 @@ import { DomainError } from '../common/domain-error'; import { UnitOfWork } from '../database/unit-of-work'; import { MailReadService } from '../llm-parse/mail-read.service'; import type { WalkedMail } from '../llm-parse/mail-read.types'; -import { MailIngestApplyService } from './mail-ingest-apply.service'; +import { MailIngestApplyService, type WrittenBooking } from './mail-ingest-apply.service'; import { MailIngestNotifyService } from './mail-ingest-notify.service'; -import { implausible, type TripOption } from './mail-ingest-checks'; +import { fitsTrip, implausible, type TripOption } from './mail-ingest-checks'; import { MailIngestTripsService } from './mail-ingest-trips.service'; +import { ADDED_RESOLUTIONS } from './mail-ingest-views'; import { IMPORT_SOURCES, UNDO_WINDOW_DAYS } from './mail-ingest.constants'; import { openMail } from './mail-seal'; import { InjectRepository } from '@mikro-orm/nestjs'; @@ -30,8 +31,6 @@ import { Injectable } from '@nestjs/common'; import type { BookingImportPreviewItem } from '@trek/shared'; const DAY_MS = 24 * 60 * 60 * 1000; -/** The states an added booking is undone from. */ -const ADDED = ['auto_applied', 'applied'] as const; const ISO_DATE_RE = /\b(20\d\d-[01]\d-[0-3]\d)\b/; /** A booking reference as a mail writes it: after its label, letters and digits. */ const REFERENCE_RE = @@ -48,7 +47,7 @@ const TYPE_WORDS: ReadonlyArray<[RegExp, string]> = [ /** * What a person does with an entry of their mail inbox: add it to a trip, * file the mail as documents, fill the booking form in themselves, discard - * it, or undo a booking a mail added. Every action checks again that the + * it, or undo or move a booking a mail added. Every action checks again that the * person may add bookings to the trip it touches, and that every id belongs * to that trip. */ @@ -81,26 +80,10 @@ export class MailIngestActionsService { confirm: boolean, now: Date = new Date(), ): Promise { - const entry = await this.candidates.findById(candidateId); - const tripId = entry?.resolved_trip_id ?? null; - if (!entry || tripId === null || !(await this.editableBy(userId, tripId))) { - throw new DomainError(404, 'Booking not found'); - } - if (!(ADDED as readonly string[]).includes(entry.resolution)) { - throw new DomainError(409, 'This booking is not added from a mail any more'); - } - const until = parseDbTimestamp(entry.undo_until); - if (!until || until < now) throw new DomainError(409, 'The time to undo this booking has passed'); - - const created = decodeJson(MAIL_INGEST_CREATED, entry.created_ids_json); - const affects = await this.undoAffects(tripId, created); - if (affects.length > 0 && !confirm) { - throw new DomainError(409, 'Undoing this booking changes more than the booking', { - details: { code: 'needs_confirmation', affects }, - }); - } + const { entry, tripId, created } = await this.addedEntry(candidateId, userId, now); + await this.requireConfirmed(tripId, created, confirm, 'Undoing this booking changes more than the booking'); const removed = await this.uow.transactional(async () => { - const claimed = await this.candidates.resolve(entry.id, ADDED, { + const claimed = await this.candidates.resolve(entry.id, ADDED_RESOLUTIONS, { resolution: 'undone', reason: null, resolvedTripId: tripId, @@ -116,6 +99,68 @@ export class MailIngestActionsService { return this.reload(entry.id); } + /** + * Moves an added booking into another trip the person may add bookings to, + * within the undo window: the booking is written there anew from what the + * mail said (its venue located and its cost rate frozen for that trip) and + * what it created in the first trip goes, in one transaction. The checks + * run again for the new trip: its dates (G3), the booking's values (G5), + * and that it does not hold the booking already. A booking whose cost moved + * balances, or that was edited since, moves only with `confirm`, since the + * edit would be lost. + * + * @txIndependent the move is one transaction; the documents follow it, as + * storage I/O, and are noted on the entry. + */ + async move( + candidateId: number, + userId: number, + tripId: number, + confirm: boolean, + now: Date = new Date(), + ): Promise { + const { entry, tripId: fromTripId, created } = await this.addedEntry(candidateId, userId, now); + if (tripId === fromTripId) throw new DomainError(409, 'This booking is in that trip already'); + const item = decodeJson(MAIL_INGEST_ITEM, entry.item_json); + if (!item) throw new DomainError(409, 'This entry holds no booking to move'); + const target = await this.tripFor(userId, tripId); + if (target.start_date && target.end_date && !fitsTrip(item, target)) { + throw new DomainError(422, 'This booking lies outside the dates of that trip'); + } + if (implausible(item, now, (code) => this.airports.findByIata(code) !== null)) { + throw new DomainError(422, 'This booking has values that cannot be right; fill it in yourself instead'); + } + if (await this.candidates.tripHasExternal(target.id, IMPORT_SOURCES, this.apply.externalKey(item))) { + throw new DomainError(409, 'This booking is already in the trip.'); + } + await this.requireConfirmed(fromTripId, created, confirm, 'Moving this booking changes more than the booking'); + const uncertain = decodeJson(MAIL_INGEST_UNCERTAIN_FIELDS, entry.uncertain_fields); + const prepared = await this.apply.prepare(item, target, entry.user_id, uncertain); + const { removed, written } = await this.uow.transactional(async () => { + // Read again inside the transaction: of two moves at once, the second sees where the first one went. + const current = await this.candidates.findById(entry.id); + if (current?.resolved_trip_id !== fromTripId || !ADDED_RESOLUTIONS.includes(current.resolution)) { + throw new DomainError(409, 'This booking was moved or undone already'); + } + const gone = await this.apply.removeInTx(fromTripId, created); + const done = await this.apply.writeInTx(prepared); + await this.candidates.resolve(entry.id, ADDED_RESOLUTIONS, { + resolution: entry.resolution, + reason: null, + resolvedTripId: target.id, + createdIdsJson: encodeJson(MAIL_INGEST_CREATED, done.created), + resolvedBy: userId, + resolvedAt: dbNow(now), + undoUntil: entry.undo_until as DbTimestamp | null, + }); + return { removed: gone, written: done }; + }); + await this.apply.announceRemoved(removed, created.file_ids); + await this.apply.announce(written); + await this.attachMail(entry, target.id, written, userId); + return this.reload(entry.id); + } + /** * Adds a waiting booking to the trip the inbox suggested or the person * picked. The person decides the trip; the server still checks that they @@ -159,12 +204,7 @@ export class MailIngestActionsService { return done; }); await this.apply.announce(written); - const mail = await this.mailOf(entry); - if (mail) { - const { fileIds, error } = await this.apply.attachDocuments(trip.id, written.reservation.id, mail, userId); - const created = { ...written.created, file_ids: fileIds }; - await this.candidates.setAttachments(entry.id, encodeJson(MAIL_INGEST_CREATED, created), error); - } + await this.attachMail(entry, trip.id, written, userId); return this.reload(entry.id); } @@ -232,12 +272,63 @@ export class MailIngestActionsService { return this.reload(entry.id); } - /** What an undo would touch beyond the booking: costs someone settled with, and an edit since. */ - private async undoAffects(tripId: number, created: MailIngestCreated): Promise> { + /** + * An added booking within its undo window, with the trip it is in and what + * adding it created: the forwarder or any member who may add bookings to + * that trip may act on it. + */ + private async addedEntry( + candidateId: number, + userId: number, + now: Date, + ): Promise<{ entry: MailIngestCandidateRow; tripId: number; created: MailIngestCreated }> { + const entry = await this.candidates.findById(candidateId); + const tripId = entry?.resolved_trip_id ?? null; + if (!entry || tripId === null || !(await this.editableBy(userId, tripId))) { + throw new DomainError(404, 'Booking not found'); + } + if (!ADDED_RESOLUTIONS.includes(entry.resolution)) { + throw new DomainError(409, 'This booking is not added from a mail any more'); + } + const until = parseDbTimestamp(entry.undo_until); + if (!until || until < now) throw new DomainError(409, 'The time to undo this booking has passed'); + return { entry, tripId, created: decodeJson(MAIL_INGEST_CREATED, entry.created_ids_json) }; + } + + /** + * Refuses, unless confirmed, a change that touches more than the booking: + * costs someone settled with, or an edit made since it was added. + */ + private async requireConfirmed( + tripId: number, + created: MailIngestCreated, + confirm: boolean, + error: string, + ): Promise { + if (confirm) return; const affects: Array<'balances' | 'edited'> = []; if ((await this.candidates.countPayers(created.budget_item_ids)) > 0) affects.push('balances'); if (await this.apply.editedSince(tripId, created)) affects.push('edited'); - return affects; + if (affects.length > 0) throw new DomainError(409, error, { details: { code: 'needs_confirmation', affects } }); + } + + /** + * The mail's documents on the booking just written, noted on its entry; a + * mail no longer stored leaves none. + * + * @txIndependent each document is storage I/O; the entry notes what landed. + */ + private async attachMail( + entry: MailIngestCandidateRow, + tripId: number, + written: WrittenBooking, + userId: number, + ): Promise { + const mail = await this.mailOf(entry); + if (!mail) return; + const { fileIds, error } = await this.apply.attachDocuments(tripId, written.reservation.id, mail, userId); + const created = { ...written.created, file_ids: fileIds }; + await this.candidates.setAttachments(entry.id, encodeJson(MAIL_INGEST_CREATED, created), error); } /** The person's own waiting entry; anyone else's answers as missing. */ diff --git a/server/src/nest/mail-ingest/mail-ingest-inbox.service.ts b/server/src/nest/mail-ingest/mail-ingest-inbox.service.ts new file mode 100644 index 000000000..1b25e6399 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-inbox.service.ts @@ -0,0 +1,196 @@ +import { MailIngestCandidates } from '../../db/entities/MailIngestCandidates.entity'; +import { MailIngestMessages } from '../../db/entities/MailIngestMessages.entity'; +import { MAIL_INGEST_ATTACHMENTS } from '../../db/json-columns'; +import type { + MailIngestCandidateRow, + MailIngestCandidatesRepository, +} from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessagesRepository } from '../../db/repositories/MailIngestMessages.repository'; +import { utcSuffix } from '../../db/types'; +import { decodeJson } from '../../utils/json-column'; +import { logError } from '../audit/audit-log.logger'; +import { DomainError } from '../common/domain-error'; +import { withRequestContext } from '../database/request-context'; +import { MailReadService } from '../llm-parse/mail-read.service'; +import { MailIngestAddressService } from './mail-ingest-address.service'; +import { MailIngestIntakeService, type MailIntakeRefusal } from './mail-ingest-intake.service'; +import { MailIngestProcessJob } from './mail-ingest-process.job'; +import { MailIngestSettingsService } from './mail-ingest-settings.service'; +import { candidateView, messageView } from './mail-ingest-views'; +import { openMail } from './mail-seal'; +import { MikroORM } from '@mikro-orm/core'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import type { + MailIngestAddressResponse, + MailIngestCandidate, + MailIngestInboxCountResponse, + MailIngestInboxQuery, + MailIngestInboxResponse, + MailIngestMessagePreviewResponse, + MailIngestPrefsUpdateRequest, + MailIngestSettingsResponse, + MailIngestTripRef, + MailIngestUploadResponse, +} from '@trek/shared'; + +/** How an upload the intake refused answers, by why. */ +const UPLOAD_REFUSALS: Record = { + unavailable: { status: 404, error: 'Mail import is not available' }, + too_large: { status: 413, error: 'This mail is larger than the import takes' }, + not_a_mail: { status: 422, error: 'This file is not a mail' }, + unknown_address: { status: 422, error: 'This file is not a mail' }, + address_daily_limit: { status: 429, error: 'Your address took all the mails it takes today' }, + inbox_full: { status: 409, error: 'Your inbox is full; decide some of its entries first' }, +}; + +/** + * What a person sees of their mail import: the settings card (their address, + * their switches, whether AI reads for them), their inbox page by page, a + * mail as plain text, and an .eml they upload. Every read is the person's + * own: an entry or a mail of someone else answers as missing. + */ +@Injectable() +export class MailIngestInboxService { + constructor( + @InjectRepository(MailIngestCandidates) private readonly candidates: MailIngestCandidatesRepository, + @InjectRepository(MailIngestMessages) private readonly messages: MailIngestMessagesRepository, + private readonly settings: MailIngestSettingsService, + private readonly addresses: MailIngestAddressService, + private readonly intake: MailIngestIntakeService, + private readonly reader: MailReadService, + private readonly job: MailIngestProcessJob, + private readonly orm: MikroORM, + ) {} + + /** The settings card. Where the import is off or not part of the plan it says so, and nothing else. */ + async settingsOf(userId: number): Promise { + const prefs = await this.settings.prefsOf(userId); + if (!(await this.settings.isAvailable())) { + return { available: false, address: null, prefs, ai_available: false, pending: 0 }; + } + return { + available: true, + address: await this.addresses.ownAddress(userId), + prefs, + ai_available: prefs.ai_enabled && (await this.reader.isAvailable(userId)), + pending: await this.candidates.countPending(userId), + }; + } + + async updatePrefs(userId: number, update: MailIngestPrefsUpdateRequest): Promise { + await this.requireAvailable(); + await this.settings.updatePrefs(userId, update); + return this.settingsOf(userId); + } + + /** A new own address; the old one stops taking mail at once. */ + async renewOwnAddress(userId: number): Promise { + await this.requireAvailable(); + return { address: await this.addresses.renewOwn(userId) }; + } + + async tripAddress(userId: number, tripId: number): Promise { + await this.requireAvailable(); + return { address: await this.addresses.tripAddress(userId, tripId) }; + } + + /** The person's address for the trip, made now or renewed. */ + async renewTripAddress(userId: number, tripId: number): Promise { + await this.requireAvailable(); + return { address: await this.addresses.renewTripAddress(userId, tripId) }; + } + + async revokeTripAddress(userId: number, tripId: number): Promise { + await this.requireAvailable(); + await this.addresses.revokeTripAddress(userId, tripId); + } + + /** One page of the inbox, newest first, each entry with the mail it came from. */ + async inbox(userId: number, query: MailIngestInboxQuery, now: Date = new Date()): Promise { + await this.requireAvailable(); + const rows = await this.candidates.listForUser(userId, query.status === 'open', query.before ?? null, query.limit); + const mails = await this.messages.listHeads(uniq(rows.map((row) => row.message_id))); + const heads = new Map(mails.map((head) => [head.id, head])); + const trips = await this.tripsNamedBy(userId, rows); + const entries = rows.flatMap((row) => { + const head = heads.get(row.message_id); + return head ? [{ ...candidateView(row, trips, now), message: messageView(head) }] : []; + }); + return { + entries, + pending: await this.candidates.countPending(userId), + next_before: rows.length === query.limit ? (rows.at(-1)?.id ?? null) : null, + }; + } + + async count(userId: number): Promise { + if (!(await this.settings.isAvailable())) return { pending: 0 }; + return { pending: await this.candidates.countPending(userId) }; + } + + /** An entry as the inbox shows it, for the answer of an action on it. */ + async entry(row: MailIngestCandidateRow, now: Date = new Date()): Promise { + return candidateView(row, await this.tripsNamedBy(row.user_id, [row]), now); + } + + /** "Show mail": the mail as plain text, without its headers, as long as it is stored. */ + async preview(userId: number, messageId: number): Promise { + await this.requireAvailable(); + const message = await this.messages.findById(messageId); + if (!message || message.user_id !== userId) throw new DomainError(404, 'Mail not found'); + const raw = openMail(message.raw_eml_enc); + const mail = raw ? this.reader.walk(raw) : null; + if (!mail) throw new DomainError(410, 'The mail is no longer stored'); + return { + id: message.id, + subject: mail.subject ?? message.subject, + received_at: utcSuffix(message.received_at) ?? message.received_at, + text: mail.body, + attachments: decodeJson(MAIL_INGEST_ATTACHMENTS, message.attachments), + }; + } + + /** + * An .eml the person uploaded, read as if they had forwarded it to their + * own address. The pipeline is asked to run at once rather than at its next + * tick, so the inbox fills while the person still looks at it. + * + * @txIndependent the mail is stored by itself; its reading runs on its own afterwards. + */ + async upload(userId: number, raw: Buffer): Promise { + const result = await this.intake.upload(raw, userId); + if (result.status === 'refused') { + const { status, error } = UPLOAD_REFUSALS[result.reason]; + throw new DomainError(status, error); + } + const [head] = await this.messages.listHeads([result.messageId]); + if (!head) throw new Error(`mail ${result.messageId} vanished after its upload`); + if (result.status === 'accepted') this.processSoon(); + return { message: messageView(head), duplicate: result.status === 'duplicate' }; + } + + private processSoon(): void { + withRequestContext(this.orm, () => this.job.tick()).catch((err: unknown) => { + logError(`[mail-ingest] Reading an uploaded mail failed: ${err instanceof Error ? err.message : String(err)}`); + }); + } + + private async requireAvailable(): Promise { + if (!(await this.settings.isAvailable())) throw new DomainError(404, 'Mail import is not available'); + } + + /** The trips these entries suggest, as far as the reader can see them. */ + private async tripsNamedBy( + userId: number, + rows: readonly MailIngestCandidateRow[], + ): Promise> { + const ids = uniq(rows.flatMap((row) => (row.suggested_trip_id === null ? [] : [row.suggested_trip_id]))); + const refs = await this.candidates.listTripRefs(userId, ids); + return new Map(refs.map((ref) => [ref.id, ref])); + } +} + +function uniq(ids: readonly number[]): number[] { + return [...new Set(ids)]; +} diff --git a/server/src/nest/mail-ingest/mail-ingest-intake.service.ts b/server/src/nest/mail-ingest/mail-ingest-intake.service.ts index 10cc1e4e0..bc5010c52 100644 --- a/server/src/nest/mail-ingest/mail-ingest-intake.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-intake.service.ts @@ -16,7 +16,7 @@ import type { MailIngestTransport } from '@trek/shared'; import { createHash } from 'node:crypto'; /** Why a mail was not taken in. A transport answers each the way its protocol does (a 550, a 404, a 413). */ -type MailIntakeRefusal = +export type MailIntakeRefusal = | 'unavailable' | 'too_large' | 'not_a_mail' diff --git a/server/src/nest/mail-ingest/mail-ingest-views.ts b/server/src/nest/mail-ingest/mail-ingest-views.ts new file mode 100644 index 000000000..1c8ec7163 --- /dev/null +++ b/server/src/nest/mail-ingest/mail-ingest-views.ts @@ -0,0 +1,82 @@ +import { + MAIL_INGEST_ATTACHMENTS, + MAIL_INGEST_CHECKS, + MAIL_INGEST_CREATED, + MAIL_INGEST_HINTS, + MAIL_INGEST_ITEM, + MAIL_INGEST_MATCH, + MAIL_INGEST_UNCERTAIN_FIELDS, +} from '../../db/json-columns'; +import type { MailIngestCandidateRow } from '../../db/repositories/MailIngestCandidates.repository'; +import type { MailIngestMessageHead } from '../../db/repositories/MailIngestMessages.repository'; +import { parseDbTimestamp, utcSuffix } from '../../db/types'; +import { decodeJson } from '../../utils/json-column'; +import { + mailIngestMessageStatusSchema, + mailIngestReasonSchema, + mailIngestResolutionSchema, + mailIngestSourceSchema, + mailIngestTransportSchema, + type MailIngestCandidate, + type MailIngestMessage, + type MailIngestTripRef, +} from '@trek/shared'; + +/** The states an added booking can still be undone or moved from. */ +export const ADDED_RESOLUTIONS: readonly string[] = ['auto_applied', 'applied']; + +/** + * How the inbox shows an entry: its booking, why it waits, the trip it + * suggests (named only when the reader can see that trip), what adding it + * created, and until when that can be undone. A stored value the contract + * does not know reads as the closest safe one rather than failing the page. + */ +export function candidateView( + row: MailIngestCandidateRow, + trips: ReadonlyMap, + now: Date, +): MailIngestCandidate { + const created = decodeJson(MAIL_INGEST_CREATED, row.created_ids_json); + const undoUntil = parseDbTimestamp(row.undo_until); + const undoable = ADDED_RESOLUTIONS.includes(row.resolution) && undoUntil !== null && undoUntil > now; + return { + id: row.id, + message_id: row.message_id, + item: decodeJson(MAIL_INGEST_ITEM, row.item_json), + source: mailIngestSourceSchema.nullable().catch(null).parse(row.source), + uncertain_fields: decodeJson(MAIL_INGEST_UNCERTAIN_FIELDS, row.uncertain_fields), + checks: decodeJson(MAIL_INGEST_CHECKS, row.checks_json), + suggested_trip: row.suggested_trip_id === null ? null : (trips.get(row.suggested_trip_id) ?? null), + suggested_new_trip: decodeJson(MAIL_INGEST_MATCH, row.match_json).suggested_new_trip, + resolution: mailIngestResolutionSchema.catch('pending').parse(row.resolution), + reason: mailIngestReasonSchema.nullable().catch(null).parse(row.reason), + resolved_trip_id: row.resolved_trip_id, + reservation_ids: created.reservation_ids, + file_ids: created.file_ids, + created_at: isoOf(row.created_at), + resolved_at: utcSuffix(row.resolved_at), + undo_until: undoable ? undoUntil.toISOString() : null, + }; +} + +/** How the inbox shows the mail an entry came from: its header and what it carried, never its content. */ +export function messageView(head: MailIngestMessageHead): MailIngestMessage { + return { + id: head.id, + transport: mailIngestTransportSchema.catch('upload').parse(head.transport), + subject: head.subject, + received_at: isoOf(head.received_at), + status: mailIngestMessageStatusSchema.catch('failed').parse(head.status), + reason: mailIngestReasonSchema.nullable().catch(null).parse(head.reason), + hints: decodeJson(MAIL_INGEST_HINTS, head.hints), + ai_used: head.ai_used === 1, + trip_id: head.trip_id, + attachments: decodeJson(MAIL_INGEST_ATTACHMENTS, head.attachments), + forwarding_code: head.forwarding_code, + }; +} + +/** A stored timestamp in the ISO spelling the contract carries. */ +function isoOf(stored: string): string { + return utcSuffix(stored) ?? stored; +} diff --git a/server/src/nest/mail-ingest/mail-ingest.module.ts b/server/src/nest/mail-ingest/mail-ingest.module.ts index f8c162c73..4e2efb5d3 100644 --- a/server/src/nest/mail-ingest/mail-ingest.module.ts +++ b/server/src/nest/mail-ingest/mail-ingest.module.ts @@ -21,6 +21,7 @@ import { SettingsModule } from '../settings/settings.module'; import { MailIngestActionsService } from './mail-ingest-actions.service'; import { MailIngestAddressService } from './mail-ingest-address.service'; import { MailIngestApplyService } from './mail-ingest-apply.service'; +import { MailIngestInboxService } from './mail-ingest-inbox.service'; import { MailIngestIntakeService } from './mail-ingest-intake.service'; import { MailIngestNotificationActions } from './mail-ingest-notification-actions'; import { MailIngestNotifyService } from './mail-ingest-notify.service'; @@ -76,6 +77,7 @@ import { Module } from '@nestjs/common'; MailIngestNotifyService, MailIngestProcessService, MailIngestActionsService, + MailIngestInboxService, MailIngestNotificationActions, MailIngestProcessJob, MailIngestRetentionJob, diff --git a/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts b/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts index 0b1c523df..7ac9d90fe 100644 --- a/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts +++ b/server/tests/unit/db/repositories/MailIngestCandidates.repository.test.ts @@ -305,3 +305,55 @@ describe('MailIngestCandidatesRepository: what it reads of the trips', () => { expect(await candidates.countPayers([])).toBe(0); }); }); + +describe('MailIngestCandidatesRepository: the inbox and its counts', () => { + it('pages a person’s open or decided entries, newest first', async () => { + const { user } = await makeUser(t); + const { user: other } = await makeUser(t); + const mail = await mailOf(user.id); + const oldest = await entry(mail, user.id); + const middle = await entry(mail, user.id, { position: 1 }); + const newest = await entry(mail, user.id, { position: 2 }); + const decided = await entry(mail, user.id, { position: 3, resolution: 'discarded', reason: null }); + await entry(await mailOf(other.id), other.id); + + const open = await candidates.listForUser(user.id, true, null, 2); + expect(open.map((row) => row.id)).toEqual([newest, middle]); + expect((await candidates.listForUser(user.id, true, middle, 10)).map((row) => row.id)).toEqual([oldest]); + expect((await candidates.listForUser(user.id, false, null, 10)).map((row) => row.id)).toEqual([decided]); + }); + + it('names only the trips the reader owns or joined', async () => { + const { user } = await makeUser(t); + const { user: other } = await makeUser(t); + const owned = await makeTrip(t, user.id, { title: 'Japan', start_date: '2027-03-01', end_date: '2027-03-03' }); + const joined = await makeTrip(t, other.id, { title: 'Lisbon' }); + await addTripMember(t, joined.id, user.id); + const foreign = await makeTrip(t, other.id, { title: 'Secret' }); + + const refs = await candidates.listTripRefs(user.id, [owned.id, joined.id, foreign.id]); + expect(refs.sort((a, b) => a.id - b.id)).toEqual([ + { id: owned.id, title: 'Japan', start_date: '2027-03-01', end_date: '2027-03-03' }, + { id: joined.id, title: 'Lisbon', start_date: null, end_date: null }, + ]); + expect(await candidates.listTripRefs(user.id, [])).toEqual([]); + }); + + it('counts the entries made since a time by outcome, and the waiting ones per person', async () => { + const { user } = await makeUser(t); + const { user: other } = await makeUser(t); + const mail = await mailOf(user.id); + await entry(mail, user.id); + await entry(mail, user.id, { position: 1 }); + await entry(mail, user.id, { position: 2, resolution: 'auto_applied', reason: null }); + await entry(mail, user.id, { position: 3, resolution: 'undone', createdAt: at('2026-10-01T10:00:00Z') }); + await entry(await mailOf(other.id), other.id); + + const tally = await candidates.tallyResolutionsSince(at('2026-10-15T00:00:00Z')); + expect(tally.sort((a, b) => a.resolution.localeCompare(b.resolution))).toEqual([ + { resolution: 'auto_applied', n: 1 }, + { resolution: 'pending', n: 3 }, + ]); + expect(await candidates.pendingPerUser()).toEqual([2, 1]); + }); +}); diff --git a/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts b/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts index f7faf44ec..bc9ec6916 100644 --- a/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts +++ b/server/tests/unit/db/repositories/MailIngestMessages.repository.test.ts @@ -112,6 +112,28 @@ describe('MailIngestMessagesRepository', () => { expect(await messages.countAiReadSince(box.userId, at('2026-11-01T00:00:00Z'))).toBe(0); }); + it('lists mails without their content, and counts everyone’s by status, reason and AI', async () => { + const box = await inbox(); + const read = await store(box, 'h1'); + const dropped = await store(box, 'h2'); + await store(box, 'h3', at('2026-10-01T10:00:00Z')); + await messages.finish(read, DONE); + await messages.finish(dropped, { ...DONE, status: 'discarded', reason: 'newsletter', aiUsed: false }); + + const heads = await messages.listHeads([read, dropped]); + expect(heads.map((head) => head.id).sort()).toEqual([read, dropped].sort()); + expect(heads[0]).not.toHaveProperty('raw_eml_enc'); + expect(heads[0]).not.toHaveProperty('auth_results'); + expect(await messages.listHeads([])).toEqual([]); + + const tally = await messages.tallySince(at('2026-10-15T00:00:00Z')); + expect(tally.sort((a, b) => a.status.localeCompare(b.status))).toEqual([ + { status: 'discarded', reason: 'newsletter', n: 1 }, + { status: 'processed', reason: null, n: 1 }, + ]); + expect(await messages.countAllAiReadSince(at('2026-10-15T00:00:00Z'))).toBe(1); + }); + it('claims a due mail for one run and lets a claim that ran out go to the next', async () => { const box = await inbox(); const id = await store(box, 'd1'); diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts index 6a37c53d9..0837abd8e 100644 --- a/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts +++ b/server/tests/unit/nest/mail-ingest/mail-ingest.pipeline.test.ts @@ -16,6 +16,7 @@ import { MailReadService } from '../../../../src/nest/llm-parse/mail-read.servic import type { MailReading as AiReading } from '../../../../src/nest/llm-parse/mail-read.types'; import { walkMail } from '../../../../src/nest/llm-parse/mime-walker'; import { MailIngestActionsService } from '../../../../src/nest/mail-ingest/mail-ingest-actions.service'; +import { MailIngestInboxService } from '../../../../src/nest/mail-ingest/mail-ingest-inbox.service'; import { MailIngestIntakeService } from '../../../../src/nest/mail-ingest/mail-ingest-intake.service'; import { MailIngestProcessJob } from '../../../../src/nest/mail-ingest/mail-ingest-process.job'; import { MailIngestRetentionJob } from '../../../../src/nest/mail-ingest/mail-ingest-retention.job'; @@ -257,6 +258,49 @@ describe('a booking that passes every check', () => { const [second] = await entriesOf(await forward(user.id, eml('flight-forwarded-again'))); expect(second).toMatchObject({ resolution: 'duplicate' }); }); + + it('moves into another trip in its dates: written there anew, gone from the first, still undoable', async () => { + const user = await person(); + const first = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const second = await makeTrip(orm(), user.id, { start_date: '2030-02-25', end_date: '2030-03-05' }); + const elsewhere = await makeTrip(orm(), user.id, { start_date: '2030-08-01', end_date: '2030-08-05' }); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false }); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + await actions().applyToTrip(entry.id, user.id, first.id, NOW); + + await expect(actions().move(entry.id, user.id, elsewhere.id, false, NOW)).rejects.toThrow( + 'This booking lies outside the dates of that trip', + ); + await expect(actions().move(entry.id, user.id, first.id, false, NOW)).rejects.toThrow( + 'This booking is in that trip already', + ); + const moved = await actions().move(entry.id, user.id, second.id, false, NOW); + expect(moved).toMatchObject({ resolution: 'applied', resolved_trip_id: second.id }); + expect(await countRows(orm(), Reservations, { trip: first.id })).toBe(0); + expect(await countRows(orm(), Reservations, { trip: second.id })).toBe(1); + expect(t.realtime.broadcastMock).toHaveBeenCalledWith(first.id, 'reservation:deleted', expect.anything()); + expect(t.realtime.broadcastMock).toHaveBeenCalledWith(second.id, 'reservation:created', expect.anything()); + + await actions().undo(entry.id, user.id, false, NOW); + expect(await countRows(orm(), Reservations, { trip: second.id })).toBe(0); + }); + + it('asks before moving a booking that was edited since, and refuses a stranger', async () => { + const user = await person(); + const { user: stranger } = await makeUser(orm()); + const first = await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + const second = await makeTrip(orm(), user.id); + const [entry] = await entriesOf(await forward(user.id, eml('flight'))); + await updateRows(orm(), Reservations, { trip: first.id }, { notes: 'Aisle seat' }); + + const refused = await actions() + .move(entry.id, user.id, second.id, false, NOW) + .catch((err: unknown) => err); + expect((refused as DomainError).getResponse()).toMatchObject({ code: 'needs_confirmation', affects: ['edited'] }); + await expect(actions().move(entry.id, stranger.id, second.id, true, NOW)).rejects.toThrow('Booking not found'); + await actions().move(entry.id, user.id, second.id, true, NOW); + expect(await countRows(orm(), Reservations, { trip: second.id })).toBe(1); + }); }); describe('a booking that waits in the inbox', () => { @@ -400,6 +444,95 @@ describe('a mail that keeps failing', () => { }); }); +describe('what the person sees', () => { + const inbox = () => t.get(MailIngestInboxService); + + it('lists the waiting entries with their mail and the trip they suggest, and no one else’s', async () => { + const user = await person(); + const { user: other } = await makeUser(orm()); + const trip = await makeTrip(orm(), user.id, { title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }); + await t.get(MailIngestSettingsService).updatePrefs(user.id, { auto_add: false }); + const messageId = await forward(user.id, eml('flight')); + + const page = await inbox().inbox(user.id, { status: 'open', limit: 50 }, NOW); + expect(page.pending).toBe(1); + expect(page.next_before).toBeNull(); + expect(page.entries).toHaveLength(1); + expect(page.entries[0]).toMatchObject({ + message_id: messageId, + resolution: 'pending', + reason: 'ask_first', + suggested_trip: { id: trip.id, title: 'Japan', start_date: '2030-03-01', end_date: '2030-03-10' }, + undo_until: null, + message: { id: messageId, transport: 'upload', status: 'processed', subject: 'Fwd: Your flight LH 716' }, + }); + expect(page.entries[0].message).not.toHaveProperty('raw_eml_enc'); + expect((await inbox().inbox(other.id, { status: 'open', limit: 50 }, NOW)).entries).toEqual([]); + expect(await inbox().count(user.id)).toEqual({ pending: 1 }); + }); + + it('pages decided entries, and an added one carries until when it can be undone', async () => { + const user = await person(); + await makeTrip(orm(), user.id, { start_date: '2030-03-01', end_date: '2030-03-10' }); + await forward(user.id, eml('first')); + await forward(user.id, eml('again')); + + const page = await inbox().inbox(user.id, { status: 'resolved', limit: 1 }, NOW); + expect(page.entries).toHaveLength(1); + expect(page.entries[0]).toMatchObject({ resolution: 'duplicate', undo_until: null }); + expect(page.next_before).toBe(page.entries[0].id); + const next = await inbox().inbox(user.id, { status: 'resolved', before: page.entries[0].id, limit: 1 }, NOW); + expect(next.entries[0]).toMatchObject({ resolution: 'auto_applied', undo_until: '2030-01-22T10:00:00.000Z' }); + expect(next.entries[0].reservation_ids).toHaveLength(1); + }); + + it('shows a mail as text without its headers, to its person only, until it is purged', async () => { + const user = await person(); + const { user: other } = await makeUser(orm()); + machine.mockResolvedValue([]); + const messageId = await forward(user.id, eml('nothing')); + + const preview = await inbox().preview(user.id, messageId); + expect(preview).toMatchObject({ id: messageId, subject: 'Fwd: Your flight LH 716', attachments: [] }); + expect(preview.text).toContain('Booking reference ABC123'); + expect(preview.text).not.toContain('mia@example.com'); + await expect(inbox().preview(other.id, messageId)).rejects.toThrow('Mail not found'); + await updateRows(orm(), MailIngestMessages, { id: messageId }, { raw_eml_enc: null }); + await expect(inbox().preview(user.id, messageId)).rejects.toThrow('The mail is no longer stored'); + }); + + it('answers the settings card, and only that it is unavailable while the import is off', async () => { + const user = await person(); + expect(await inbox().settingsOf(user.id)).toEqual({ + available: true, + address: null, + prefs: { enabled: true, ai_enabled: true, auto_add: true }, + ai_available: false, + pending: 0, + }); + await setAppSetting(orm(), 'mail_ingest_enabled', 'false'); + expect(await inbox().settingsOf(user.id)).toMatchObject({ available: false, address: null, pending: 0 }); + await expect(inbox().updatePrefs(user.id, { auto_add: false })).rejects.toThrow('Mail import is not available'); + expect(await inbox().count(user.id)).toEqual({ pending: 0 }); + }); + + it('reads an uploaded mail at once, knows one it has, and refuses what it cannot take', async () => { + const user = await person(); + // The reading the upload starts runs on its own; held here, so it cannot outlive the case. + const tick = vi.spyOn(job(), 'tick').mockResolvedValue(0); + try { + const first = await inbox().upload(user.id, eml('upload')); + expect(first).toMatchObject({ duplicate: false, message: { transport: 'upload', status: 'queued' } }); + const again = await inbox().upload(user.id, eml('upload')); + expect(again).toMatchObject({ duplicate: true, message: { id: first.message.id } }); + expect(tick).toHaveBeenCalledTimes(1); + await expect(inbox().upload(user.id, Buffer.from('nothing to see'))).rejects.toThrow('This file is not a mail'); + } finally { + tick.mockRestore(); + } + }); +}); + describe('the retention', () => { it('purges a settled mail and keeps one an entry waits on', async () => { const user = await person();