From 7d3ed21a0389671577805fb3c096f9beafa47e50 Mon Sep 17 00:00:00 2001 From: Maurice Date: Fri, 13 Nov 2026 16:22:31 +0100 Subject: [PATCH] feat(mail-ingest): the transport settings in the admin form, each overridable from the environment The IMAP box, the webhook secret and the address template join the admin settings form (masked, encrypted, held by the operator on a managed install), and MAIL_INGEST_* variables override them through the mailIngestConfig token. The audit row masks every masked key, and key rotation covers the new secrets. --- server/.env.example | 10 +++ server/scripts/migrate-encryption.ts | 27 ++++++++ server/src/app-config/README.md | 6 +- server/src/app-config/boot-derive.ts | 22 +++++++ server/src/app-config/env.schema.ts | 14 ++++ server/src/nest/app-config/tokens.ts | 12 +++- server/src/nest/auth/auth.service.ts | 6 +- .../src/nest/common/app-settings.registry.ts | 45 +++++++++++-- server/src/nest/common/managed.ts | 13 ++++ .../mail-ingest-settings.service.ts | 64 +++++++++++++++++-- server/tests/unit/app-config/derive.test.ts | 18 ++++++ .../unit/db/migrate-encryption-parity.test.ts | 7 ++ .../globalMiddleware.redaction.test.ts | 15 +++++ .../unit/nest/app-settings.registry.test.ts | 21 +++++- server/tests/unit/nest/auth.service.test.ts | 23 +++++++ .../mail-ingest-settings.service.test.ts | 59 ++++++++++++++++- server/tests/unit/nest/managed-keys.test.ts | 11 ++++ .../mail-ingest/mail-ingest.schema.spec.ts | 9 +++ shared/src/mail-ingest/mail-ingest.schema.ts | 25 +++++++- wiki/Environment-Variables.md | 19 ++++++ 20 files changed, 406 insertions(+), 20 deletions(-) diff --git a/server/.env.example b/server/.env.example index 5c476f694..e0bf2fcbe 100644 --- a/server/.env.example +++ b/server/.env.example @@ -106,6 +106,16 @@ DEMO_MODE=false # Demo mode: resets data hourly # SMTP_FROM=TREK # Sender address of every email TREK sends. # SMTP_SKIP_TLS_VERIFY=false # true accepts a self-signed certificate on an internal relay. Not for production. +# MAIL_INGEST_TRANSPORT=off # How forwarded booking mails reach TREK: off (uploads only), imap or webhook. Each MAIL_INGEST_* variable overrides its field in Admin > Integrations > Booking mail import. +# MAIL_INGEST_IMAP_HOST=imap.example.com # The mailbox TREK reads every two minutes. A host on your own network needs ALLOW_INTERNAL_NETWORK=true. +# MAIL_INGEST_IMAP_PORT=993 # 993 for TLS, 143 for STARTTLS. +# MAIL_INGEST_IMAP_TLS=tls # tls (from the first byte) or starttls. +# MAIL_INGEST_IMAP_USER= # IMAP login. +# MAIL_INGEST_IMAP_PASS= # IMAP password. +# MAIL_INGEST_IMAP_MAILBOX=INBOX # The folder the mails arrive in. Read mails move on to TREK/processed and are never deleted. +# MAIL_INGEST_ADDRESS_TEMPLATE=trek+{token}@example.com # The forwarding address; each person's token stands where {token} stands. +# MAIL_INGEST_WEBHOOK_SECRET= # The key the webhook sender signs each mail with (HMAC-SHA256), at least 32 characters. + # WEBAUTHN_RP_ID=trek.example.com # The domain passkeys are bound to. Overrides the setting in the admin panel; unset, it comes from the hostname of APP_URL. An IP address is refused. # WEBAUTHN_ORIGINS=https://trek.example.com # Comma-separated origins allowed for passkey sign-in. Overrides the setting in the admin panel; unset, the origin of APP_URL is used. diff --git a/server/scripts/migrate-encryption.ts b/server/scripts/migrate-encryption.ts index d5bc0b842..8838f65d0 100644 --- a/server/scripts/migrate-encryption.ts +++ b/server/scripts/migrate-encryption.ts @@ -317,6 +317,9 @@ async function main() { // service_key is the installation's key towards the liketrek services. // Missed here it reads back as unusable, the installation registers anew and // loses its id, and a revoked key would come back as a fresh one. + // + // The mail import's IMAP password and webhook secret: missed, the box stops + // being read and every signed mail is refused until an admin types them again. for (const key of [ 'oidc_client_secret', 'smtp_pass', @@ -327,6 +330,8 @@ async function main() { 'amap_api_key', 'web_push_vapid_private_key', 'service_key', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', ]) { const row = db.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as { value: string } | undefined; if (!row?.value) continue; @@ -567,6 +572,28 @@ async function main() { } } + // --- mail_ingest: forwarding address tokens and the stored mails --- + // A person sees their address again through its token_enc; a forwarded mail + // waits for its reading, and for the inbox's ways out, as raw_eml_enc. Missed, + // the addresses still take mail (they are found by hash) but show as unset, + // and every stored mail reads as unreadable. + for (const [table, column] of [ + ['mail_ingest_addresses', 'token_enc'], + ['mail_ingest_messages', 'raw_eml_enc'], + ] as const) { + if (!tableExists(table)) continue; + const rows = db.prepare(`SELECT id, ${column} AS value FROM ${table} WHERE ${column} IS NOT NULL`).all() as { + id: number; + value: string; + }[]; + for (const row of rows) { + const newVal = migrateApiKeyValue(row.value, `${table}[${row.id}].${column}`); + if (newVal !== null) { + db.prepare(`UPDATE ${table} SET ${column} = ? WHERE id = ?`).run(newVal, row.id); + } + } + } + // --- addons: the AI Parsing addon's instance-wide API key --- // Stored encrypted as `apiKey` inside the addon's JSON config // (llm-parse/llm-config.ts). Left out, LLM parsing quietly runs without a diff --git a/server/src/app-config/README.md b/server/src/app-config/README.md index 8c06cd676..e90a95aa6 100644 --- a/server/src/app-config/README.md +++ b/server/src/app-config/README.md @@ -105,8 +105,10 @@ Every variable has exactly one owner, and the owner decides how it is read: `app.get(httpConfig.KEY)` for the pre-init Express layer). Today: `httpConfig` (TRUST_PROXY, HSTS_INCLUDE_SUBDOMAINS, HTTP_KEEP_ALIVE_TIMEOUT_MS), `storageConfig` (TREK_PLACE_PHOTO_DIR), `transitConfig` (TRANSIT_API_URL, - for TransitService) and `kitineraryConfig` (KITINERARY_EXTRACTOR_PATH and - PATH, for KitineraryExtractorService's binary probe). A module whose provider + for TransitService), `kitineraryConfig` (KITINERARY_EXTRACTOR_PATH and + PATH, for KitineraryExtractorService's binary probe), `servicesConfig` + (TREK_SERVICES and its hosts) and `mailIngestConfig` (MAIL_INGEST_*, for + MailIngestSettingsService, each over its stored setting). A module whose provider injects one imports `AppConfigModule`, so it also builds on its own in a test. - **`readEnv()` / `RuntimeEnvService`** (`derive.ts`): everything else, both the runtime-toggled values and the boot-stable ones that code outside a Nest diff --git a/server/src/app-config/boot-derive.ts b/server/src/app-config/boot-derive.ts index ade88329d..0e054e28b 100644 --- a/server/src/app-config/boot-derive.ts +++ b/server/src/app-config/boot-derive.ts @@ -90,6 +90,27 @@ export function deriveServices(raw: RawEnv) { }; } +/** + * How the booking mail import takes mail in, set in the environment rather + * than in the admin dialog: each value wins over its stored setting when it is + * non-empty (MailIngestSettingsService applies them). Unset leaves the dialog + * in charge, so an install that sets none of them behaves as before. + */ +export function deriveMailIngest(raw: RawEnv) { + const read = (value: string | undefined) => value?.trim() || null; + return { + transport: read(raw.MAIL_INGEST_TRANSPORT)?.toLowerCase() ?? null, + imapHost: read(raw.MAIL_INGEST_IMAP_HOST), + imapPort: read(raw.MAIL_INGEST_IMAP_PORT), + imapTls: read(raw.MAIL_INGEST_IMAP_TLS)?.toLowerCase() ?? null, + imapUser: read(raw.MAIL_INGEST_IMAP_USER), + imapPass: read(raw.MAIL_INGEST_IMAP_PASS), + imapMailbox: read(raw.MAIL_INGEST_IMAP_MAILBOX), + addressTemplate: read(raw.MAIL_INGEST_ADDRESS_TEMPLATE), + webhookSecret: read(raw.MAIL_INGEST_WEBHOOK_SECRET), + }; +} + /** One entry per token: the namespace it registers under and the function it derives with. */ export const BOOT_DERIVERS = { http: deriveHttpBoot, @@ -97,6 +118,7 @@ export const BOOT_DERIVERS = { transit: deriveTransit, kitinerary: deriveKitinerary, services: deriveServices, + mailIngest: deriveMailIngest, } as const; export type BootNamespace = keyof typeof BOOT_DERIVERS; diff --git a/server/src/app-config/env.schema.ts b/server/src/app-config/env.schema.ts index 9f1532ff3..e05a2d202 100644 --- a/server/src/app-config/env.schema.ts +++ b/server/src/app-config/env.schema.ts @@ -110,6 +110,20 @@ export const envSchema = z.object({ SMTP_FROM: anyString, SMTP_SKIP_TLS_VERIFY: boolStr, + // Booking mail import: each wins over its field in the admin dialog + MAIL_INGEST_TRANSPORT: oneOf(['off', 'imap', 'webhook', 'cloud']), + MAIL_INGEST_IMAP_HOST: anyString, + MAIL_INGEST_IMAP_PORT: integer(1, 65535, 'must be a port number (1-65535)'), + MAIL_INGEST_IMAP_TLS: oneOf(['tls', 'starttls']), + MAIL_INGEST_IMAP_USER: anyString, + MAIL_INGEST_IMAP_PASS: anyString, + MAIL_INGEST_IMAP_MAILBOX: anyString, + MAIL_INGEST_ADDRESS_TEMPLATE: optionalWith( + (v) => v.includes('{token}') && v.includes('@'), + 'must hold {token} and an @, like trek+{token}@example.com', + ), + MAIL_INGEST_WEBHOOK_SECRET: optionalWith((v) => v.length >= 32, 'must be at least 32 characters'), + // WebAuthn WEBAUTHN_RP_ID: anyString, WEBAUTHN_ORIGINS: anyString, diff --git a/server/src/nest/app-config/tokens.ts b/server/src/nest/app-config/tokens.ts index e8671a8f0..07cf9268e 100644 --- a/server/src/nest/app-config/tokens.ts +++ b/server/src/nest/app-config/tokens.ts @@ -38,4 +38,14 @@ export const kitineraryConfig = registerAs('kitinerary', () => BOOT_DERIVERS.kit */ export const servicesConfig = registerAs('services', () => BOOT_DERIVERS.services(process.env)); -export const BOOT_STABLE_TOKENS = [httpConfig, storageConfig, transitConfig, kitineraryConfig, servicesConfig]; +/** How the booking mail import takes mail in (MAIL_INGEST_*), each value over its stored setting. */ +export const mailIngestConfig = registerAs('mailIngest', () => BOOT_DERIVERS.mailIngest(process.env)); + +export const BOOT_STABLE_TOKENS = [ + httpConfig, + storageConfig, + transitConfig, + kitineraryConfig, + servicesConfig, + mailIngestConfig, +]; diff --git a/server/src/nest/auth/auth.service.ts b/server/src/nest/auth/auth.service.ts index b6aa46ac9..23a1d9096 100644 --- a/server/src/nest/auth/auth.service.ts +++ b/server/src/nest/auth/auth.service.ts @@ -778,7 +778,8 @@ export class AuthService { } }); - const changedKeys = ADMIN_FORM_SETTING_KEYS.filter(k => !blocked.includes(k) && body[k] !== undefined && !(k === 'smtp_pass' && String(body[k]) === '••••••••')); + // A masked key sent back as its mask was not changed, and a secret's value never reaches the audit row. + const changedKeys = ADMIN_FORM_SETTING_KEYS.filter(k => !blocked.includes(k) && body[k] !== undefined && !(MASKED_ADMIN_SETTING_KEYS.has(k) && String(body[k]) === '••••••••')); const summary: Record = {}; const smtpChanged = changedKeys.some(k => k.startsWith('smtp_')); @@ -789,10 +790,11 @@ export class AuthService { if (changedKeys.includes('allow_registration')) summary.allow_registration = body.allow_registration; if (changedKeys.includes('allowed_file_types')) summary.allowed_file_types_updated = true; if (changedKeys.includes('require_mfa')) summary.require_mfa = body.require_mfa; + if (changedKeys.some(k => k.startsWith('mail_ingest_'))) summary.mail_ingest_settings_updated = true; const debugDetails: Record = {}; for (const k of changedKeys) { - debugDetails[k] = k === 'smtp_pass' ? '***' : body[k]; + debugDetails[k] = MASKED_ADMIN_SETTING_KEYS.has(k) ? '***' : body[k]; } // The reminder crons read their enable gates and channels per tick, so a diff --git a/server/src/nest/common/app-settings.registry.ts b/server/src/nest/common/app-settings.registry.ts index a4dca77b0..7627c4760 100644 --- a/server/src/nest/common/app-settings.registry.ts +++ b/server/src/nest/common/app-settings.registry.ts @@ -1,6 +1,6 @@ import { readEnv } from '../../app-config'; import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; -import { aiPolicySchema, mailIngestTransportSettingSchema } from '@trek/shared'; +import { aiPolicySchema, mailIngestImapTlsSchema, mailIngestTransportSettingSchema } from '@trek/shared'; import { z } from 'zod'; @@ -161,15 +161,46 @@ export const APP_SETTINGS = { synology_capture_repair_3019: { schema: text, default: null }, /** How many starts have tried that so far, while some photo could not be confirmed. */ synology_capture_repair_3019_tries: { schema: intText, default: null }, - // The booking mail import (mail-ingest). Off until switched on: on a self-hosted install by its - // admin, on a managed one by the operator, for the plans that include it and never for Start. - mail_ingest_enabled: { schema: flag, default: 'false' }, + // The booking mail import (mail-ingest), set in its own admin dialog. Off until switched on: on a + // self-hosted install by its admin, on a managed one by the operator, for the plans that include + // it and never for Start, so every key is the operator's there. The MAIL_INGEST_* environment + // variables win over the stored transport keys; they belong to the mailIngestConfig token, so + // MailIngestSettingsService applies them rather than the `env` of an entry here. + mail_ingest_enabled: { schema: flag, default: 'false', adminForm: true, managedLocked: true }, /** How mail reaches the install: not at all (uploads only), IMAP, a signed webhook, or the TREK Cloud relay. */ - mail_ingest_transport: { schema: mailIngestTransportSettingSchema, default: 'off' }, + mail_ingest_transport: { + schema: mailIngestTransportSettingSchema, + default: 'off', + adminForm: true, + managedLocked: true, + }, + mail_ingest_imap_host: { schema: text, default: null, adminForm: true, managedLocked: true }, + mail_ingest_imap_port: { schema: intText, default: '993', adminForm: true, managedLocked: true }, + mail_ingest_imap_tls: { schema: mailIngestImapTlsSchema, default: 'tls', adminForm: true, managedLocked: true }, + mail_ingest_imap_user: { schema: text, default: null, adminForm: true, managedLocked: true }, + mail_ingest_imap_pass: { + schema: text, + default: null, + adminForm: true, + masked: true, + encrypted: 'always', + managedLocked: true, + }, + /** The folder the forwarded mails arrive in; read mails move on to TREK/processed. */ + mail_ingest_imap_mailbox: { schema: text, default: 'INBOX', adminForm: true, managedLocked: true }, /** The forwarding address, `{token}` standing where a person's token goes: `trek+{token}@example.com`. */ - mail_ingest_address_template: { schema: text, default: null }, + mail_ingest_address_template: { schema: text, default: null, adminForm: true, managedLocked: true }, + /** The key the webhook's sender signs every mail with (HMAC-SHA256). */ + mail_ingest_webhook_secret: { + schema: text, + default: null, + adminForm: true, + masked: true, + encrypted: 'always', + managedLocked: true, + }, /** How many forwarded mails a day an AI reads for one person. */ - mail_ingest_ai_per_user_daily: { schema: intText, default: '10' }, + mail_ingest_ai_per_user_daily: { schema: intText, default: '10', adminForm: true, managedLocked: true }, } as const satisfies Record; export type AppSettingKey = keyof typeof APP_SETTINGS; diff --git a/server/src/nest/common/managed.ts b/server/src/nest/common/managed.ts index 14efac592..30c35292a 100644 --- a/server/src/nest/common/managed.ts +++ b/server/src/nest/common/managed.ts @@ -88,6 +88,19 @@ export const MANAGED_LOCKED_SETTING_KEYS = [ 'llm_model', 'llm_multimodal', 'llm_provider', + // The booking mail import: whether a plan includes it is the operator's call (never on Start), + // and so is how mail reaches the install (the TREK Cloud relay) and the address it is sent to. + 'mail_ingest_address_template', + 'mail_ingest_ai_per_user_daily', + 'mail_ingest_enabled', + 'mail_ingest_imap_host', + 'mail_ingest_imap_mailbox', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_transport', + 'mail_ingest_webhook_secret', 'mapbox_access_token', 'maps_api_key', // Whether the login page waits or goes straight to the provider is part of the diff --git a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts index 509aa20b8..a52cb0577 100644 --- a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts @@ -1,13 +1,19 @@ import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity'; import type { MailIngestPrefsPatch, MailIngestPrefsRepository } from '../../db/repositories/MailIngestPrefs.repository'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { mailIngestConfig } from '../app-config/tokens'; +import type { AppSettingKey } from '../common/app-settings.registry'; +import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { DEFAULT_ALLOWED_EXTENSIONS } from '../files/files.constants'; import { SettingsService } from '../settings/settings.service'; import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; +import type { ConfigType } from '@nestjs/config'; import { + mailIngestImapTlsSchema, mailIngestTransportSettingSchema, + type MailIngestImapTls, type MailIngestPrefs as MailIngestPrefsView, type MailIngestPrefsUpdateRequest, type MailIngestTransportSetting, @@ -15,6 +21,17 @@ import { /** The daily AI cap when the setting is missing or unreadable. */ const DEFAULT_AI_PER_USER_DAILY = 10; +const DEFAULT_IMAP_MAILBOX = 'INBOX'; + +/** The IMAP box the import reads, complete: a box without a host, a user or a password is no box. */ +export interface MailIngestImapConfig { + host: string; + port: number; + tls: MailIngestImapTls; + user: string; + pass: string; + mailbox: string; +} /** * Whether the mail import runs on this install, how mail reaches it, and the @@ -24,12 +41,16 @@ const DEFAULT_AI_PER_USER_DAILY = 10; * install that setting is the operator's: it is written for the plans that * include the import and never for Start, so on Start every surface reports * the import as unavailable and nothing is read. + * + * The transport's values come from the admin dialog, each overridden by its + * MAIL_INGEST_* variable where that is set (the mailIngestConfig token). */ @Injectable() export class MailIngestSettingsService { constructor( private readonly settings: SettingsService, private readonly env: RuntimeEnvService, + @Inject(mailIngestConfig.KEY) private readonly config: ConfigType, @InjectRepository(MailIngestPrefs) private readonly prefs: MailIngestPrefsRepository, ) {} @@ -40,17 +61,43 @@ export class MailIngestSettingsService { /** How mail reaches the install; a managed one takes it from the TREK Cloud relay alone. */ async transport(): Promise { if (this.env.isManaged()) return 'cloud'; - const stored = await this.settings.getAppSetting('mail_ingest_transport'); - const parsed = mailIngestTransportSettingSchema.safeParse(stored); + const parsed = mailIngestTransportSettingSchema.safeParse( + await this.pick(this.config.transport, 'mail_ingest_transport'), + ); return parsed.success ? parsed.data : 'off'; } /** The forwarding address with `{token}` where a person's token goes; null until an admin set one. */ async addressTemplate(): Promise { - const template = (await this.settings.getAppSetting('mail_ingest_address_template'))?.trim(); + const template = await this.pick(this.config.addressTemplate, 'mail_ingest_address_template'); return template && template.includes('{token}') ? template : null; } + /** The IMAP box to read; null while its host, user or password is missing. */ + async imapConfig(): Promise { + const host = await this.pick(this.config.imapHost, 'mail_ingest_imap_host'); + const user = await this.pick(this.config.imapUser, 'mail_ingest_imap_user'); + const pass = this.config.imapPass ?? decrypt_api_key(await this.settings.getAppSetting('mail_ingest_imap_pass')); + if (!host || !user || !pass) return null; + const tls = mailIngestImapTlsSchema.catch('tls').parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls')); + const port = Number(await this.pick(this.config.imapPort, 'mail_ingest_imap_port')); + const mailbox = await this.pick(this.config.imapMailbox, 'mail_ingest_imap_mailbox'); + return { + host, + port: Number.isInteger(port) && port > 0 && port <= 65535 ? port : defaultImapPort(tls), + tls, + user, + pass, + mailbox: mailbox ?? DEFAULT_IMAP_MAILBOX, + }; + } + + /** The key a webhook sender signs its mails with; while none is set, the webhook takes no mail. */ + async webhookSecret(): Promise { + const stored = await this.settings.getAppSetting('mail_ingest_webhook_secret'); + return this.config.webhookSecret ?? (decrypt_api_key(stored) || null); + } + async aiDailyCap(): Promise { const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily')); return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY; @@ -84,4 +131,13 @@ export class MailIngestSettingsService { await this.prefs.upsertForUser(userId, patch); return this.prefsOf(userId); } + + /** The environment's value where it is set, else the stored one; blank reads as unset. */ + private async pick(override: string | null, key: AppSettingKey): Promise { + return (override ?? (await this.settings.getAppSetting(key)))?.trim() || null; + } +} + +function defaultImapPort(tls: MailIngestImapTls): number { + return tls === 'tls' ? 993 : 143; } diff --git a/server/tests/unit/app-config/derive.test.ts b/server/tests/unit/app-config/derive.test.ts index 00540ab95..1c8e8ba0e 100644 --- a/server/tests/unit/app-config/derive.test.ts +++ b/server/tests/unit/app-config/derive.test.ts @@ -1,6 +1,7 @@ import { deriveHttpBoot, deriveKitinerary, + deriveMailIngest, deriveServices, deriveStorage, deriveTransit, @@ -299,6 +300,23 @@ describe('deriveKitinerary (kitineraryConfig token)', () => { }); }); +describe('deriveMailIngest (mailIngestConfig token)', () => { + it('leaves every value unset unless its variable holds one, trimmed', () => { + expect(Object.values(deriveMailIngest({}))).toEqual(Array(9).fill(null)); + expect(deriveMailIngest({ MAIL_INGEST_IMAP_HOST: ' ', MAIL_INGEST_IMAP_PASS: ' s3cret ' })).toMatchObject({ + imapHost: null, + imapPass: 's3cret', + }); + }); + + it('reads the transport and the encryption in lower case', () => { + expect(deriveMailIngest({ MAIL_INGEST_TRANSPORT: 'IMAP', MAIL_INGEST_IMAP_TLS: 'StartTLS' })).toMatchObject({ + transport: 'imap', + imapTls: 'starttls', + }); + }); +}); + describe('deriveServices (servicesConfig token)', () => { it('registers by itself unless TREK_SERVICES says off, and any value but auto is off', () => { expect(deriveServices({}).mode).toBe('auto'); diff --git a/server/tests/unit/db/migrate-encryption-parity.test.ts b/server/tests/unit/db/migrate-encryption-parity.test.ts index fceec9ad1..d0c5a8210 100644 --- a/server/tests/unit/db/migrate-encryption-parity.test.ts +++ b/server/tests/unit/db/migrate-encryption-parity.test.ts @@ -119,6 +119,13 @@ describe('migrate-encryption.ts app_settings parity', () => { } }); + // A forwarding address shows its token through token_enc; a stored mail is sealed in raw_eml_enc. + it('ROTPAR-016: rotates the address tokens and stored mails of the mail import', () => { + const section = script.slice(script.indexOf('--- mail_ingest:'), script.indexOf('--- addons:')); + expect(section).toContain("['mail_ingest_addresses', 'token_enc']"); + expect(section).toContain("['mail_ingest_messages', 'raw_eml_enc']"); + }); + it('ROTPAR-004: rotates every storage.backends secret field for s3', () => { const fields = storageSecretFields('s3'); expect(fields.length).toBeGreaterThan(0); diff --git a/server/tests/unit/middleware/globalMiddleware.redaction.test.ts b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts index 051d14372..69721cdd4 100644 --- a/server/tests/unit/middleware/globalMiddleware.redaction.test.ts +++ b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts @@ -33,6 +33,21 @@ describe('globalMiddleware request-log redaction', () => { } }); + it('redacts the mail import secrets the admin settings form sends, and keeps the rest readable', () => { + const out = redact({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: 'box-secret', + mail_ingest_webhook_secret: 'hook-secret', + mail_ingest_address_template: 'trek+{token}@example.com', + }); + expect(out).toEqual({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: '[REDACTED]', + mail_ingest_webhook_secret: '[REDACTED]', + mail_ingest_address_template: 'trek+{token}@example.com', + }); + }); + it('redacts nested objects and arrays, leaving non-sensitive fields untouched', () => { const out = redact({ name: 'off-box', diff --git a/server/tests/unit/nest/app-settings.registry.test.ts b/server/tests/unit/nest/app-settings.registry.test.ts index f22148cd2..7cacc6176 100644 --- a/server/tests/unit/nest/app-settings.registry.test.ts +++ b/server/tests/unit/nest/app-settings.registry.test.ts @@ -51,14 +51,33 @@ describe('the derived key lists', () => { 'webauthn_origins', 'places_provider', 'flight_live_enabled', + 'mail_ingest_enabled', + 'mail_ingest_transport', + 'mail_ingest_imap_host', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_mailbox', + 'mail_ingest_address_template', + 'mail_ingest_webhook_secret', + 'mail_ingest_ai_per_user_daily', ]); }); it('APPSET-002: the masked and the encrypted keys', () => { - expect([...MASKED_ADMIN_SETTING_KEYS].sort()).toEqual(['admin_ntfy_token', 'admin_webhook_url', 'smtp_pass']); + expect([...MASKED_ADMIN_SETTING_KEYS].sort()).toEqual([ + 'admin_ntfy_token', + 'admin_webhook_url', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', + 'smtp_pass', + ]); expect([...ENCRYPTED_APP_SETTING_KEYS].sort()).toEqual([ 'admin_ntfy_token', 'admin_webhook_url', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', 'oidc_client_secret', 'service_key', 'smtp_pass', diff --git a/server/tests/unit/nest/auth.service.test.ts b/server/tests/unit/nest/auth.service.test.ts index 80304c4dc..eb54d4587 100644 --- a/server/tests/unit/nest/auth.service.test.ts +++ b/server/tests/unit/nest/auth.service.test.ts @@ -21,6 +21,7 @@ import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service' import { WebauthnConfigService } from '../../../src/nest/auth/webauthn-config.service'; import { BudgetService } from '../../../src/nest/budget/budget.service'; import { ExchangeRatesService } from '../../../src/nest/budget/exchange-rates.service'; +import { encrypt_api_key } from '../../../src/nest/common/crypto/apiKeyCrypto'; import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; import { DEFAULT_ALLOWED_EXTENSIONS } from '../../../src/nest/files/files.constants'; import { MailerService } from '../../../src/nest/notifications/mailer/mailer.service'; @@ -1126,6 +1127,28 @@ describe('updateAppSettings', () => { expect(result.auditDebugDetails).not.toHaveProperty('smtp_pass', 'stored'); await deleteRows(orm, AppSettings, { key: { $in: ['smtp_pass', 'notification_channels'] } }); }); + + it('AUTH-DB-075b: the mail import secrets are stored encrypted and never reach the audit row', async () => { + const { user } = createAdmin(testDb); + const result = await asLegacyResult( + svc.updateAppSettings(user.id, { + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: 'box-secret', + mail_ingest_webhook_secret: 'hook-secret', + }), + ); + expect(result.success).toBe(true); + expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('box-secret'); + expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('hook-secret'); + expect(result.auditDebugDetails).toEqual({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: '***', + mail_ingest_webhook_secret: '***', + }); + expect(result.auditSummary).toMatchObject({ mail_ingest_settings_updated: true }); + const keys = ['mail_ingest_imap_host', 'mail_ingest_imap_pass', 'mail_ingest_webhook_secret']; + await deleteRows(orm, AppSettings, { key: { $in: keys } }); + }); }); describe('MFA success flows', () => { diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts index 5183fba04..51099b00d 100644 --- a/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts @@ -1,10 +1,13 @@ /** * Whether the mail import runs and how: off until switched on, the TREK Cloud * relay as the only transport of a managed install, the address template only - * with its placeholder, and a person's switches all on until they change one. + * with its placeholder, the MAIL_INGEST_* environment over the stored values, + * and a person's switches all on until they change one. */ import { MailIngestPrefs } from '../../../../src/db/entities/MailIngestPrefs.entity'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { mailIngestConfig } from '../../../../src/nest/app-config/tokens'; +import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto'; import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service'; import { SettingsService } from '../../../../src/nest/settings/settings.service'; import { getRepositoryToken } from '@mikro-orm/nestjs'; @@ -16,6 +19,18 @@ const stored = new Map(); const settings = { getAppSetting: vi.fn(async (key: string) => stored.get(key) ?? null) }; const env = { isManaged: vi.fn(() => false) }; const prefs = { findForUser: vi.fn(), upsertForUser: vi.fn(async () => undefined) }; +const UNSET = { + transport: null, + imapHost: null, + imapPort: null, + imapTls: null, + imapUser: null, + imapPass: null, + imapMailbox: null, + addressTemplate: null, + webhookSecret: null, +}; +const config: Record = { ...UNSET }; let service: MailIngestSettingsService; beforeEach(async () => { @@ -23,11 +38,13 @@ beforeEach(async () => { stored.clear(); env.isManaged.mockReturnValue(false); prefs.findForUser.mockResolvedValue(null); + Object.assign(config, UNSET); const moduleRef = await Test.createTestingModule({ providers: [ MailIngestSettingsService, { provide: SettingsService, useValue: settings }, { provide: RuntimeEnvService, useValue: env }, + { provide: mailIngestConfig.KEY, useValue: config }, { provide: getRepositoryToken(MailIngestPrefs), useValue: prefs }, ], }).compile(); @@ -76,6 +93,46 @@ describe('MailIngestSettingsService', () => { expect(await service.allowedFileTypes()).toBe('pdf'); }); + it('takes the environment over every stored transport value', async () => { + stored.set('mail_ingest_transport', 'imap'); + stored.set('mail_ingest_address_template', 'trek+{token}@stored.example'); + config.transport = 'webhook'; + config.addressTemplate = 'in+{token}@env.example'; + expect(await service.transport()).toBe('webhook'); + expect(await service.addressTemplate()).toBe('in+{token}@env.example'); + }); + + it('reads the IMAP box only when complete, with its password decrypted and sound defaults', async () => { + stored.set('mail_ingest_imap_host', 'imap.example.com'); + stored.set('mail_ingest_imap_user', 'trips'); + expect(await service.imapConfig()).toBeNull(); + stored.set('mail_ingest_imap_pass', encrypt_api_key('box-secret')); + expect(await service.imapConfig()).toEqual({ + host: 'imap.example.com', + port: 993, + tls: 'tls', + user: 'trips', + pass: 'box-secret', + mailbox: 'INBOX', + }); + stored.set('mail_ingest_imap_tls', 'starttls'); + stored.set('mail_ingest_imap_port', 'nonsense'); + stored.set('mail_ingest_imap_mailbox', ' Bookings '); + expect(await service.imapConfig()).toMatchObject({ port: 143, tls: 'starttls', mailbox: 'Bookings' }); + config.imapHost = 'mail.lan'; + config.imapPort = '1143'; + config.imapPass = 'env-secret'; + expect(await service.imapConfig()).toMatchObject({ host: 'mail.lan', port: 1143, pass: 'env-secret' }); + }); + + it('reads the webhook secret decrypted, the environment first, none while unset', async () => { + expect(await service.webhookSecret()).toBeNull(); + stored.set('mail_ingest_webhook_secret', encrypt_api_key('stored-secret')); + expect(await service.webhookSecret()).toBe('stored-secret'); + config.webhookSecret = 'env-secret'; + expect(await service.webhookSecret()).toBe('env-secret'); + }); + it('has every switch on for a person who never changed one, and writes only what changes', async () => { expect(await service.prefsOf(4)).toEqual({ enabled: true, ai_enabled: true, auto_add: true }); prefs.findForUser.mockResolvedValue({ id: 1, user_id: 4, enabled: 1, ai_enabled: 0, auto_add: 1 }); diff --git a/server/tests/unit/nest/managed-keys.test.ts b/server/tests/unit/nest/managed-keys.test.ts index 9aee86f36..0436d2a67 100644 --- a/server/tests/unit/nest/managed-keys.test.ts +++ b/server/tests/unit/nest/managed-keys.test.ts @@ -65,6 +65,17 @@ describe('managed key assignment', () => { 'llm_model', 'llm_multimodal', 'llm_provider', + 'mail_ingest_address_template', + 'mail_ingest_ai_per_user_daily', + 'mail_ingest_enabled', + 'mail_ingest_imap_host', + 'mail_ingest_imap_mailbox', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_transport', + 'mail_ingest_webhook_secret', 'mapbox_access_token', 'maps_api_key', 'oidc_auto_redirect', diff --git a/shared/src/mail-ingest/mail-ingest.schema.spec.ts b/shared/src/mail-ingest/mail-ingest.schema.spec.ts index dc6701877..83364f8c5 100644 --- a/shared/src/mail-ingest/mail-ingest.schema.spec.ts +++ b/shared/src/mail-ingest/mail-ingest.schema.spec.ts @@ -8,6 +8,7 @@ import { import { mailIngestAddressSchema, mailIngestAdminStatusResponseSchema, + mailIngestAdminTestResponseSchema, mailIngestApplyRequestSchema, mailIngestCandidateSchema, mailIngestFileRequestSchema, @@ -137,6 +138,13 @@ describe('mail-ingest rows', () => { expect(mailIngestAdminStatusResponseSchema.safeParse(status).success).toBe(true); expect(mailIngestInboundResponseSchema.safeParse({ accepted: true, duplicate: false }).success).toBe(true); }); + + it('MAILC-016: the connection test names its transport and what failed', () => { + expect(mailIngestAdminTestResponseSchema.safeParse({ transport: 'imap', ok: true, error: null }).success).toBe(true); + const failed = { transport: 'webhook', ok: false, error: 'No webhook secret is set' }; + expect(mailIngestAdminTestResponseSchema.safeParse(failed).success).toBe(true); + expect(mailIngestAdminTestResponseSchema.safeParse({ transport: 'smtp', ok: true, error: null }).success).toBe(false); + }); }); describe('mail-ingest requests', () => { @@ -152,6 +160,7 @@ describe('mail-ingest requests', () => { expect(mailIngestApplyRequestSchema.safeParse({ trip_id: 3 }).success).toBe(true); expect(mailIngestApplyRequestSchema.safeParse({ trip_id: 0 }).success).toBe(false); expect(mailIngestMoveRequestSchema.safeParse({}).success).toBe(false); + expect(mailIngestMoveRequestSchema.safeParse({ trip_id: 3, confirm: true }).success).toBe(true); expect(mailIngestFileRequestSchema.safeParse({ trip_id: 3, reservation_id: 4 }).success).toBe(false); }); diff --git a/shared/src/mail-ingest/mail-ingest.schema.ts b/shared/src/mail-ingest/mail-ingest.schema.ts index 16455db58..e467b559d 100644 --- a/shared/src/mail-ingest/mail-ingest.schema.ts +++ b/shared/src/mail-ingest/mail-ingest.schema.ts @@ -229,8 +229,13 @@ export type MailIngestMessagePreviewResponse = z.infer; -/** POST /api/mail-ingest/candidates/:id/move: the added booking into another trip. */ -export const mailIngestMoveRequestSchema = z.strictObject({ trip_id: idSchema }); +/** + * POST /api/mail-ingest/candidates/:id/move: the added booking into another + * trip, written there anew. One whose cost has moved balances, or that was + * edited since, moves only with `confirm`, as an undo does (409 + * {@link mailIngestUndoConflictSchema}). + */ +export const mailIngestMoveRequestSchema = z.strictObject({ trip_id: idSchema, confirm: z.boolean().optional() }); export type MailIngestMoveRequest = z.infer; /** POST /api/mail-ingest/candidates/:id/file: the mail's attachments, or its text, as documents of a trip. */ @@ -331,3 +336,19 @@ export const mailIngestAdminStatusResponseSchema = z.object({ backlog: z.array(countSchema), }); export type MailIngestAdminStatusResponse = z.infer; + +/** + * POST /api/admin/mail-ingest/test: whether the configured transport can + * reach its mail now (the IMAP box opens, the webhook has its secret, the + * TREK Cloud relay is attached). `error` says what failed, never a password. + */ +export const mailIngestAdminTestResponseSchema = z.object({ + transport: mailIngestTransportSettingSchema, + ok: z.boolean(), + error: z.string().nullable(), +}); +export type MailIngestAdminTestResponse = z.infer; + +/** The IMAP box's encryption: TLS from the first byte (port 993), or STARTTLS on a plain port (143). */ +export const mailIngestImapTlsSchema = z.enum(['tls', 'starttls']); +export type MailIngestImapTls = z.infer; diff --git a/wiki/Environment-Variables.md b/wiki/Environment-Variables.md index a78fa2ad5..86401651a 100644 --- a/wiki/Environment-Variables.md +++ b/wiki/Environment-Variables.md @@ -236,6 +236,25 @@ optional (for unauthenticated relays). --- +## Booking mail import + +How forwarded booking mails reach TREK ([Booking mail import](Booking-Mail-Import)). Every variable overrides its field +in the admin dialog; unset, the dialog decides. + +| Variable | Description | Default | +|--------------------------------|-------------------------------------------------------------------------------------------------------------|---------| +| `MAIL_INGEST_TRANSPORT` | `off` (uploads only), `imap` or `webhook`. | `off` | +| `MAIL_INGEST_IMAP_HOST` | The mailbox TREK reads every two minutes. A host on your own network needs `ALLOW_INTERNAL_NETWORK=true`. | — | +| `MAIL_INGEST_IMAP_PORT` | `993` for TLS, `143` for STARTTLS. | `993` | +| `MAIL_INGEST_IMAP_TLS` | `tls` (encrypted from the first byte) or `starttls`. | `tls` | +| `MAIL_INGEST_IMAP_USER` | IMAP login. | — | +| `MAIL_INGEST_IMAP_PASS` | IMAP password. | — | +| `MAIL_INGEST_IMAP_MAILBOX` | The folder the mails arrive in. Read mails move on to `TREK/processed`; nothing is deleted. | `INBOX` | +| `MAIL_INGEST_ADDRESS_TEMPLATE` | The forwarding address, each person's token where `{token}` stands, e.g. `trek+{token}@example.com`. | — | +| `MAIL_INGEST_WEBHOOK_SECRET` | The key the webhook sender signs each mail with (HMAC-SHA256), at least 32 characters. | — | + +--- + ## Web Push [Web Push](Notifications#web-push) signs every message with a VAPID key pair. None of these variables is needed: