diff --git a/server/.env.example b/server/.env.example index 5c476f694..e0bf2fcbe 100644 --- a/server/.env.example +++ b/server/.env.example @@ -106,6 +106,16 @@ DEMO_MODE=false # Demo mode: resets data hourly # SMTP_FROM=TREK # Sender address of every email TREK sends. # SMTP_SKIP_TLS_VERIFY=false # true accepts a self-signed certificate on an internal relay. Not for production. +# MAIL_INGEST_TRANSPORT=off # How forwarded booking mails reach TREK: off (uploads only), imap or webhook. Each MAIL_INGEST_* variable overrides its field in Admin > Integrations > Booking mail import. +# MAIL_INGEST_IMAP_HOST=imap.example.com # The mailbox TREK reads every two minutes. A host on your own network needs ALLOW_INTERNAL_NETWORK=true. +# MAIL_INGEST_IMAP_PORT=993 # 993 for TLS, 143 for STARTTLS. +# MAIL_INGEST_IMAP_TLS=tls # tls (from the first byte) or starttls. +# MAIL_INGEST_IMAP_USER= # IMAP login. +# MAIL_INGEST_IMAP_PASS= # IMAP password. +# MAIL_INGEST_IMAP_MAILBOX=INBOX # The folder the mails arrive in. Read mails move on to TREK/processed and are never deleted. +# MAIL_INGEST_ADDRESS_TEMPLATE=trek+{token}@example.com # The forwarding address; each person's token stands where {token} stands. +# MAIL_INGEST_WEBHOOK_SECRET= # The key the webhook sender signs each mail with (HMAC-SHA256), at least 32 characters. + # WEBAUTHN_RP_ID=trek.example.com # The domain passkeys are bound to. Overrides the setting in the admin panel; unset, it comes from the hostname of APP_URL. An IP address is refused. # WEBAUTHN_ORIGINS=https://trek.example.com # Comma-separated origins allowed for passkey sign-in. Overrides the setting in the admin panel; unset, the origin of APP_URL is used. diff --git a/server/scripts/migrate-encryption.ts b/server/scripts/migrate-encryption.ts index d5bc0b842..8838f65d0 100644 --- a/server/scripts/migrate-encryption.ts +++ b/server/scripts/migrate-encryption.ts @@ -317,6 +317,9 @@ async function main() { // service_key is the installation's key towards the liketrek services. // Missed here it reads back as unusable, the installation registers anew and // loses its id, and a revoked key would come back as a fresh one. + // + // The mail import's IMAP password and webhook secret: missed, the box stops + // being read and every signed mail is refused until an admin types them again. for (const key of [ 'oidc_client_secret', 'smtp_pass', @@ -327,6 +330,8 @@ async function main() { 'amap_api_key', 'web_push_vapid_private_key', 'service_key', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', ]) { const row = db.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as { value: string } | undefined; if (!row?.value) continue; @@ -567,6 +572,28 @@ async function main() { } } + // --- mail_ingest: forwarding address tokens and the stored mails --- + // A person sees their address again through its token_enc; a forwarded mail + // waits for its reading, and for the inbox's ways out, as raw_eml_enc. Missed, + // the addresses still take mail (they are found by hash) but show as unset, + // and every stored mail reads as unreadable. + for (const [table, column] of [ + ['mail_ingest_addresses', 'token_enc'], + ['mail_ingest_messages', 'raw_eml_enc'], + ] as const) { + if (!tableExists(table)) continue; + const rows = db.prepare(`SELECT id, ${column} AS value FROM ${table} WHERE ${column} IS NOT NULL`).all() as { + id: number; + value: string; + }[]; + for (const row of rows) { + const newVal = migrateApiKeyValue(row.value, `${table}[${row.id}].${column}`); + if (newVal !== null) { + db.prepare(`UPDATE ${table} SET ${column} = ? WHERE id = ?`).run(newVal, row.id); + } + } + } + // --- addons: the AI Parsing addon's instance-wide API key --- // Stored encrypted as `apiKey` inside the addon's JSON config // (llm-parse/llm-config.ts). Left out, LLM parsing quietly runs without a diff --git a/server/src/app-config/README.md b/server/src/app-config/README.md index 8c06cd676..e90a95aa6 100644 --- a/server/src/app-config/README.md +++ b/server/src/app-config/README.md @@ -105,8 +105,10 @@ Every variable has exactly one owner, and the owner decides how it is read: `app.get(httpConfig.KEY)` for the pre-init Express layer). Today: `httpConfig` (TRUST_PROXY, HSTS_INCLUDE_SUBDOMAINS, HTTP_KEEP_ALIVE_TIMEOUT_MS), `storageConfig` (TREK_PLACE_PHOTO_DIR), `transitConfig` (TRANSIT_API_URL, - for TransitService) and `kitineraryConfig` (KITINERARY_EXTRACTOR_PATH and - PATH, for KitineraryExtractorService's binary probe). A module whose provider + for TransitService), `kitineraryConfig` (KITINERARY_EXTRACTOR_PATH and + PATH, for KitineraryExtractorService's binary probe), `servicesConfig` + (TREK_SERVICES and its hosts) and `mailIngestConfig` (MAIL_INGEST_*, for + MailIngestSettingsService, each over its stored setting). A module whose provider injects one imports `AppConfigModule`, so it also builds on its own in a test. - **`readEnv()` / `RuntimeEnvService`** (`derive.ts`): everything else, both the runtime-toggled values and the boot-stable ones that code outside a Nest diff --git a/server/src/app-config/boot-derive.ts b/server/src/app-config/boot-derive.ts index ade88329d..0e054e28b 100644 --- a/server/src/app-config/boot-derive.ts +++ b/server/src/app-config/boot-derive.ts @@ -90,6 +90,27 @@ export function deriveServices(raw: RawEnv) { }; } +/** + * How the booking mail import takes mail in, set in the environment rather + * than in the admin dialog: each value wins over its stored setting when it is + * non-empty (MailIngestSettingsService applies them). Unset leaves the dialog + * in charge, so an install that sets none of them behaves as before. + */ +export function deriveMailIngest(raw: RawEnv) { + const read = (value: string | undefined) => value?.trim() || null; + return { + transport: read(raw.MAIL_INGEST_TRANSPORT)?.toLowerCase() ?? null, + imapHost: read(raw.MAIL_INGEST_IMAP_HOST), + imapPort: read(raw.MAIL_INGEST_IMAP_PORT), + imapTls: read(raw.MAIL_INGEST_IMAP_TLS)?.toLowerCase() ?? null, + imapUser: read(raw.MAIL_INGEST_IMAP_USER), + imapPass: read(raw.MAIL_INGEST_IMAP_PASS), + imapMailbox: read(raw.MAIL_INGEST_IMAP_MAILBOX), + addressTemplate: read(raw.MAIL_INGEST_ADDRESS_TEMPLATE), + webhookSecret: read(raw.MAIL_INGEST_WEBHOOK_SECRET), + }; +} + /** One entry per token: the namespace it registers under and the function it derives with. */ export const BOOT_DERIVERS = { http: deriveHttpBoot, @@ -97,6 +118,7 @@ export const BOOT_DERIVERS = { transit: deriveTransit, kitinerary: deriveKitinerary, services: deriveServices, + mailIngest: deriveMailIngest, } as const; export type BootNamespace = keyof typeof BOOT_DERIVERS; diff --git a/server/src/app-config/env.schema.ts b/server/src/app-config/env.schema.ts index 9f1532ff3..e05a2d202 100644 --- a/server/src/app-config/env.schema.ts +++ b/server/src/app-config/env.schema.ts @@ -110,6 +110,20 @@ export const envSchema = z.object({ SMTP_FROM: anyString, SMTP_SKIP_TLS_VERIFY: boolStr, + // Booking mail import: each wins over its field in the admin dialog + MAIL_INGEST_TRANSPORT: oneOf(['off', 'imap', 'webhook', 'cloud']), + MAIL_INGEST_IMAP_HOST: anyString, + MAIL_INGEST_IMAP_PORT: integer(1, 65535, 'must be a port number (1-65535)'), + MAIL_INGEST_IMAP_TLS: oneOf(['tls', 'starttls']), + MAIL_INGEST_IMAP_USER: anyString, + MAIL_INGEST_IMAP_PASS: anyString, + MAIL_INGEST_IMAP_MAILBOX: anyString, + MAIL_INGEST_ADDRESS_TEMPLATE: optionalWith( + (v) => v.includes('{token}') && v.includes('@'), + 'must hold {token} and an @, like trek+{token}@example.com', + ), + MAIL_INGEST_WEBHOOK_SECRET: optionalWith((v) => v.length >= 32, 'must be at least 32 characters'), + // WebAuthn WEBAUTHN_RP_ID: anyString, WEBAUTHN_ORIGINS: anyString, diff --git a/server/src/nest/app-config/tokens.ts b/server/src/nest/app-config/tokens.ts index e8671a8f0..07cf9268e 100644 --- a/server/src/nest/app-config/tokens.ts +++ b/server/src/nest/app-config/tokens.ts @@ -38,4 +38,14 @@ export const kitineraryConfig = registerAs('kitinerary', () => BOOT_DERIVERS.kit */ export const servicesConfig = registerAs('services', () => BOOT_DERIVERS.services(process.env)); -export const BOOT_STABLE_TOKENS = [httpConfig, storageConfig, transitConfig, kitineraryConfig, servicesConfig]; +/** How the booking mail import takes mail in (MAIL_INGEST_*), each value over its stored setting. */ +export const mailIngestConfig = registerAs('mailIngest', () => BOOT_DERIVERS.mailIngest(process.env)); + +export const BOOT_STABLE_TOKENS = [ + httpConfig, + storageConfig, + transitConfig, + kitineraryConfig, + servicesConfig, + mailIngestConfig, +]; diff --git a/server/src/nest/auth/auth.service.ts b/server/src/nest/auth/auth.service.ts index b6aa46ac9..23a1d9096 100644 --- a/server/src/nest/auth/auth.service.ts +++ b/server/src/nest/auth/auth.service.ts @@ -778,7 +778,8 @@ export class AuthService { } }); - const changedKeys = ADMIN_FORM_SETTING_KEYS.filter(k => !blocked.includes(k) && body[k] !== undefined && !(k === 'smtp_pass' && String(body[k]) === '••••••••')); + // A masked key sent back as its mask was not changed, and a secret's value never reaches the audit row. + const changedKeys = ADMIN_FORM_SETTING_KEYS.filter(k => !blocked.includes(k) && body[k] !== undefined && !(MASKED_ADMIN_SETTING_KEYS.has(k) && String(body[k]) === '••••••••')); const summary: Record = {}; const smtpChanged = changedKeys.some(k => k.startsWith('smtp_')); @@ -789,10 +790,11 @@ export class AuthService { if (changedKeys.includes('allow_registration')) summary.allow_registration = body.allow_registration; if (changedKeys.includes('allowed_file_types')) summary.allowed_file_types_updated = true; if (changedKeys.includes('require_mfa')) summary.require_mfa = body.require_mfa; + if (changedKeys.some(k => k.startsWith('mail_ingest_'))) summary.mail_ingest_settings_updated = true; const debugDetails: Record = {}; for (const k of changedKeys) { - debugDetails[k] = k === 'smtp_pass' ? '***' : body[k]; + debugDetails[k] = MASKED_ADMIN_SETTING_KEYS.has(k) ? '***' : body[k]; } // The reminder crons read their enable gates and channels per tick, so a diff --git a/server/src/nest/common/app-settings.registry.ts b/server/src/nest/common/app-settings.registry.ts index a4dca77b0..7627c4760 100644 --- a/server/src/nest/common/app-settings.registry.ts +++ b/server/src/nest/common/app-settings.registry.ts @@ -1,6 +1,6 @@ import { readEnv } from '../../app-config'; import type { AppSettingsRepository } from '../../db/repositories/AppSettings.repository'; -import { aiPolicySchema, mailIngestTransportSettingSchema } from '@trek/shared'; +import { aiPolicySchema, mailIngestImapTlsSchema, mailIngestTransportSettingSchema } from '@trek/shared'; import { z } from 'zod'; @@ -161,15 +161,46 @@ export const APP_SETTINGS = { synology_capture_repair_3019: { schema: text, default: null }, /** How many starts have tried that so far, while some photo could not be confirmed. */ synology_capture_repair_3019_tries: { schema: intText, default: null }, - // The booking mail import (mail-ingest). Off until switched on: on a self-hosted install by its - // admin, on a managed one by the operator, for the plans that include it and never for Start. - mail_ingest_enabled: { schema: flag, default: 'false' }, + // The booking mail import (mail-ingest), set in its own admin dialog. Off until switched on: on a + // self-hosted install by its admin, on a managed one by the operator, for the plans that include + // it and never for Start, so every key is the operator's there. The MAIL_INGEST_* environment + // variables win over the stored transport keys; they belong to the mailIngestConfig token, so + // MailIngestSettingsService applies them rather than the `env` of an entry here. + mail_ingest_enabled: { schema: flag, default: 'false', adminForm: true, managedLocked: true }, /** How mail reaches the install: not at all (uploads only), IMAP, a signed webhook, or the TREK Cloud relay. */ - mail_ingest_transport: { schema: mailIngestTransportSettingSchema, default: 'off' }, + mail_ingest_transport: { + schema: mailIngestTransportSettingSchema, + default: 'off', + adminForm: true, + managedLocked: true, + }, + mail_ingest_imap_host: { schema: text, default: null, adminForm: true, managedLocked: true }, + mail_ingest_imap_port: { schema: intText, default: '993', adminForm: true, managedLocked: true }, + mail_ingest_imap_tls: { schema: mailIngestImapTlsSchema, default: 'tls', adminForm: true, managedLocked: true }, + mail_ingest_imap_user: { schema: text, default: null, adminForm: true, managedLocked: true }, + mail_ingest_imap_pass: { + schema: text, + default: null, + adminForm: true, + masked: true, + encrypted: 'always', + managedLocked: true, + }, + /** The folder the forwarded mails arrive in; read mails move on to TREK/processed. */ + mail_ingest_imap_mailbox: { schema: text, default: 'INBOX', adminForm: true, managedLocked: true }, /** The forwarding address, `{token}` standing where a person's token goes: `trek+{token}@example.com`. */ - mail_ingest_address_template: { schema: text, default: null }, + mail_ingest_address_template: { schema: text, default: null, adminForm: true, managedLocked: true }, + /** The key the webhook's sender signs every mail with (HMAC-SHA256). */ + mail_ingest_webhook_secret: { + schema: text, + default: null, + adminForm: true, + masked: true, + encrypted: 'always', + managedLocked: true, + }, /** How many forwarded mails a day an AI reads for one person. */ - mail_ingest_ai_per_user_daily: { schema: intText, default: '10' }, + mail_ingest_ai_per_user_daily: { schema: intText, default: '10', adminForm: true, managedLocked: true }, } as const satisfies Record; export type AppSettingKey = keyof typeof APP_SETTINGS; diff --git a/server/src/nest/common/managed.ts b/server/src/nest/common/managed.ts index 14efac592..30c35292a 100644 --- a/server/src/nest/common/managed.ts +++ b/server/src/nest/common/managed.ts @@ -88,6 +88,19 @@ export const MANAGED_LOCKED_SETTING_KEYS = [ 'llm_model', 'llm_multimodal', 'llm_provider', + // The booking mail import: whether a plan includes it is the operator's call (never on Start), + // and so is how mail reaches the install (the TREK Cloud relay) and the address it is sent to. + 'mail_ingest_address_template', + 'mail_ingest_ai_per_user_daily', + 'mail_ingest_enabled', + 'mail_ingest_imap_host', + 'mail_ingest_imap_mailbox', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_transport', + 'mail_ingest_webhook_secret', 'mapbox_access_token', 'maps_api_key', // Whether the login page waits or goes straight to the provider is part of the diff --git a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts index 509aa20b8..a52cb0577 100644 --- a/server/src/nest/mail-ingest/mail-ingest-settings.service.ts +++ b/server/src/nest/mail-ingest/mail-ingest-settings.service.ts @@ -1,13 +1,19 @@ import { MailIngestPrefs } from '../../db/entities/MailIngestPrefs.entity'; import type { MailIngestPrefsPatch, MailIngestPrefsRepository } from '../../db/repositories/MailIngestPrefs.repository'; import { RuntimeEnvService } from '../app-config/runtime-env.service'; +import { mailIngestConfig } from '../app-config/tokens'; +import type { AppSettingKey } from '../common/app-settings.registry'; +import { decrypt_api_key } from '../common/crypto/apiKeyCrypto'; import { DEFAULT_ALLOWED_EXTENSIONS } from '../files/files.constants'; import { SettingsService } from '../settings/settings.service'; import { MAIL_INGEST_LIMITS } from './mail-ingest.constants'; import { InjectRepository } from '@mikro-orm/nestjs'; -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; +import type { ConfigType } from '@nestjs/config'; import { + mailIngestImapTlsSchema, mailIngestTransportSettingSchema, + type MailIngestImapTls, type MailIngestPrefs as MailIngestPrefsView, type MailIngestPrefsUpdateRequest, type MailIngestTransportSetting, @@ -15,6 +21,17 @@ import { /** The daily AI cap when the setting is missing or unreadable. */ const DEFAULT_AI_PER_USER_DAILY = 10; +const DEFAULT_IMAP_MAILBOX = 'INBOX'; + +/** The IMAP box the import reads, complete: a box without a host, a user or a password is no box. */ +export interface MailIngestImapConfig { + host: string; + port: number; + tls: MailIngestImapTls; + user: string; + pass: string; + mailbox: string; +} /** * Whether the mail import runs on this install, how mail reaches it, and the @@ -24,12 +41,16 @@ const DEFAULT_AI_PER_USER_DAILY = 10; * install that setting is the operator's: it is written for the plans that * include the import and never for Start, so on Start every surface reports * the import as unavailable and nothing is read. + * + * The transport's values come from the admin dialog, each overridden by its + * MAIL_INGEST_* variable where that is set (the mailIngestConfig token). */ @Injectable() export class MailIngestSettingsService { constructor( private readonly settings: SettingsService, private readonly env: RuntimeEnvService, + @Inject(mailIngestConfig.KEY) private readonly config: ConfigType, @InjectRepository(MailIngestPrefs) private readonly prefs: MailIngestPrefsRepository, ) {} @@ -40,17 +61,43 @@ export class MailIngestSettingsService { /** How mail reaches the install; a managed one takes it from the TREK Cloud relay alone. */ async transport(): Promise { if (this.env.isManaged()) return 'cloud'; - const stored = await this.settings.getAppSetting('mail_ingest_transport'); - const parsed = mailIngestTransportSettingSchema.safeParse(stored); + const parsed = mailIngestTransportSettingSchema.safeParse( + await this.pick(this.config.transport, 'mail_ingest_transport'), + ); return parsed.success ? parsed.data : 'off'; } /** The forwarding address with `{token}` where a person's token goes; null until an admin set one. */ async addressTemplate(): Promise { - const template = (await this.settings.getAppSetting('mail_ingest_address_template'))?.trim(); + const template = await this.pick(this.config.addressTemplate, 'mail_ingest_address_template'); return template && template.includes('{token}') ? template : null; } + /** The IMAP box to read; null while its host, user or password is missing. */ + async imapConfig(): Promise { + const host = await this.pick(this.config.imapHost, 'mail_ingest_imap_host'); + const user = await this.pick(this.config.imapUser, 'mail_ingest_imap_user'); + const pass = this.config.imapPass ?? decrypt_api_key(await this.settings.getAppSetting('mail_ingest_imap_pass')); + if (!host || !user || !pass) return null; + const tls = mailIngestImapTlsSchema.catch('tls').parse(await this.pick(this.config.imapTls, 'mail_ingest_imap_tls')); + const port = Number(await this.pick(this.config.imapPort, 'mail_ingest_imap_port')); + const mailbox = await this.pick(this.config.imapMailbox, 'mail_ingest_imap_mailbox'); + return { + host, + port: Number.isInteger(port) && port > 0 && port <= 65535 ? port : defaultImapPort(tls), + tls, + user, + pass, + mailbox: mailbox ?? DEFAULT_IMAP_MAILBOX, + }; + } + + /** The key a webhook sender signs its mails with; while none is set, the webhook takes no mail. */ + async webhookSecret(): Promise { + const stored = await this.settings.getAppSetting('mail_ingest_webhook_secret'); + return this.config.webhookSecret ?? (decrypt_api_key(stored) || null); + } + async aiDailyCap(): Promise { const value = Number(await this.settings.getAppSetting('mail_ingest_ai_per_user_daily')); return Number.isInteger(value) && value >= 0 ? value : DEFAULT_AI_PER_USER_DAILY; @@ -84,4 +131,13 @@ export class MailIngestSettingsService { await this.prefs.upsertForUser(userId, patch); return this.prefsOf(userId); } + + /** The environment's value where it is set, else the stored one; blank reads as unset. */ + private async pick(override: string | null, key: AppSettingKey): Promise { + return (override ?? (await this.settings.getAppSetting(key)))?.trim() || null; + } +} + +function defaultImapPort(tls: MailIngestImapTls): number { + return tls === 'tls' ? 993 : 143; } diff --git a/server/tests/unit/app-config/derive.test.ts b/server/tests/unit/app-config/derive.test.ts index 00540ab95..1c8e8ba0e 100644 --- a/server/tests/unit/app-config/derive.test.ts +++ b/server/tests/unit/app-config/derive.test.ts @@ -1,6 +1,7 @@ import { deriveHttpBoot, deriveKitinerary, + deriveMailIngest, deriveServices, deriveStorage, deriveTransit, @@ -299,6 +300,23 @@ describe('deriveKitinerary (kitineraryConfig token)', () => { }); }); +describe('deriveMailIngest (mailIngestConfig token)', () => { + it('leaves every value unset unless its variable holds one, trimmed', () => { + expect(Object.values(deriveMailIngest({}))).toEqual(Array(9).fill(null)); + expect(deriveMailIngest({ MAIL_INGEST_IMAP_HOST: ' ', MAIL_INGEST_IMAP_PASS: ' s3cret ' })).toMatchObject({ + imapHost: null, + imapPass: 's3cret', + }); + }); + + it('reads the transport and the encryption in lower case', () => { + expect(deriveMailIngest({ MAIL_INGEST_TRANSPORT: 'IMAP', MAIL_INGEST_IMAP_TLS: 'StartTLS' })).toMatchObject({ + transport: 'imap', + imapTls: 'starttls', + }); + }); +}); + describe('deriveServices (servicesConfig token)', () => { it('registers by itself unless TREK_SERVICES says off, and any value but auto is off', () => { expect(deriveServices({}).mode).toBe('auto'); diff --git a/server/tests/unit/db/migrate-encryption-parity.test.ts b/server/tests/unit/db/migrate-encryption-parity.test.ts index fceec9ad1..d0c5a8210 100644 --- a/server/tests/unit/db/migrate-encryption-parity.test.ts +++ b/server/tests/unit/db/migrate-encryption-parity.test.ts @@ -119,6 +119,13 @@ describe('migrate-encryption.ts app_settings parity', () => { } }); + // A forwarding address shows its token through token_enc; a stored mail is sealed in raw_eml_enc. + it('ROTPAR-016: rotates the address tokens and stored mails of the mail import', () => { + const section = script.slice(script.indexOf('--- mail_ingest:'), script.indexOf('--- addons:')); + expect(section).toContain("['mail_ingest_addresses', 'token_enc']"); + expect(section).toContain("['mail_ingest_messages', 'raw_eml_enc']"); + }); + it('ROTPAR-004: rotates every storage.backends secret field for s3', () => { const fields = storageSecretFields('s3'); expect(fields.length).toBeGreaterThan(0); diff --git a/server/tests/unit/middleware/globalMiddleware.redaction.test.ts b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts index 051d14372..69721cdd4 100644 --- a/server/tests/unit/middleware/globalMiddleware.redaction.test.ts +++ b/server/tests/unit/middleware/globalMiddleware.redaction.test.ts @@ -33,6 +33,21 @@ describe('globalMiddleware request-log redaction', () => { } }); + it('redacts the mail import secrets the admin settings form sends, and keeps the rest readable', () => { + const out = redact({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: 'box-secret', + mail_ingest_webhook_secret: 'hook-secret', + mail_ingest_address_template: 'trek+{token}@example.com', + }); + expect(out).toEqual({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: '[REDACTED]', + mail_ingest_webhook_secret: '[REDACTED]', + mail_ingest_address_template: 'trek+{token}@example.com', + }); + }); + it('redacts nested objects and arrays, leaving non-sensitive fields untouched', () => { const out = redact({ name: 'off-box', diff --git a/server/tests/unit/nest/app-settings.registry.test.ts b/server/tests/unit/nest/app-settings.registry.test.ts index f22148cd2..7cacc6176 100644 --- a/server/tests/unit/nest/app-settings.registry.test.ts +++ b/server/tests/unit/nest/app-settings.registry.test.ts @@ -51,14 +51,33 @@ describe('the derived key lists', () => { 'webauthn_origins', 'places_provider', 'flight_live_enabled', + 'mail_ingest_enabled', + 'mail_ingest_transport', + 'mail_ingest_imap_host', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_mailbox', + 'mail_ingest_address_template', + 'mail_ingest_webhook_secret', + 'mail_ingest_ai_per_user_daily', ]); }); it('APPSET-002: the masked and the encrypted keys', () => { - expect([...MASKED_ADMIN_SETTING_KEYS].sort()).toEqual(['admin_ntfy_token', 'admin_webhook_url', 'smtp_pass']); + expect([...MASKED_ADMIN_SETTING_KEYS].sort()).toEqual([ + 'admin_ntfy_token', + 'admin_webhook_url', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', + 'smtp_pass', + ]); expect([...ENCRYPTED_APP_SETTING_KEYS].sort()).toEqual([ 'admin_ntfy_token', 'admin_webhook_url', + 'mail_ingest_imap_pass', + 'mail_ingest_webhook_secret', 'oidc_client_secret', 'service_key', 'smtp_pass', diff --git a/server/tests/unit/nest/auth.service.test.ts b/server/tests/unit/nest/auth.service.test.ts index 80304c4dc..eb54d4587 100644 --- a/server/tests/unit/nest/auth.service.test.ts +++ b/server/tests/unit/nest/auth.service.test.ts @@ -21,6 +21,7 @@ import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service' import { WebauthnConfigService } from '../../../src/nest/auth/webauthn-config.service'; import { BudgetService } from '../../../src/nest/budget/budget.service'; import { ExchangeRatesService } from '../../../src/nest/budget/exchange-rates.service'; +import { encrypt_api_key } from '../../../src/nest/common/crypto/apiKeyCrypto'; import { AllowedFileTypesService } from '../../../src/nest/files/allowed-file-types.service'; import { DEFAULT_ALLOWED_EXTENSIONS } from '../../../src/nest/files/files.constants'; import { MailerService } from '../../../src/nest/notifications/mailer/mailer.service'; @@ -1126,6 +1127,28 @@ describe('updateAppSettings', () => { expect(result.auditDebugDetails).not.toHaveProperty('smtp_pass', 'stored'); await deleteRows(orm, AppSettings, { key: { $in: ['smtp_pass', 'notification_channels'] } }); }); + + it('AUTH-DB-075b: the mail import secrets are stored encrypted and never reach the audit row', async () => { + const { user } = createAdmin(testDb); + const result = await asLegacyResult( + svc.updateAppSettings(user.id, { + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: 'box-secret', + mail_ingest_webhook_secret: 'hook-secret', + }), + ); + expect(result.success).toBe(true); + expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('box-secret'); + expect(vi.mocked(encrypt_api_key)).toHaveBeenCalledWith('hook-secret'); + expect(result.auditDebugDetails).toEqual({ + mail_ingest_imap_host: 'imap.example.com', + mail_ingest_imap_pass: '***', + mail_ingest_webhook_secret: '***', + }); + expect(result.auditSummary).toMatchObject({ mail_ingest_settings_updated: true }); + const keys = ['mail_ingest_imap_host', 'mail_ingest_imap_pass', 'mail_ingest_webhook_secret']; + await deleteRows(orm, AppSettings, { key: { $in: keys } }); + }); }); describe('MFA success flows', () => { diff --git a/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts index 5183fba04..51099b00d 100644 --- a/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts +++ b/server/tests/unit/nest/mail-ingest/mail-ingest-settings.service.test.ts @@ -1,10 +1,13 @@ /** * Whether the mail import runs and how: off until switched on, the TREK Cloud * relay as the only transport of a managed install, the address template only - * with its placeholder, and a person's switches all on until they change one. + * with its placeholder, the MAIL_INGEST_* environment over the stored values, + * and a person's switches all on until they change one. */ import { MailIngestPrefs } from '../../../../src/db/entities/MailIngestPrefs.entity'; import { RuntimeEnvService } from '../../../../src/nest/app-config/runtime-env.service'; +import { mailIngestConfig } from '../../../../src/nest/app-config/tokens'; +import { encrypt_api_key } from '../../../../src/nest/common/crypto/apiKeyCrypto'; import { MailIngestSettingsService } from '../../../../src/nest/mail-ingest/mail-ingest-settings.service'; import { SettingsService } from '../../../../src/nest/settings/settings.service'; import { getRepositoryToken } from '@mikro-orm/nestjs'; @@ -16,6 +19,18 @@ const stored = new Map(); const settings = { getAppSetting: vi.fn(async (key: string) => stored.get(key) ?? null) }; const env = { isManaged: vi.fn(() => false) }; const prefs = { findForUser: vi.fn(), upsertForUser: vi.fn(async () => undefined) }; +const UNSET = { + transport: null, + imapHost: null, + imapPort: null, + imapTls: null, + imapUser: null, + imapPass: null, + imapMailbox: null, + addressTemplate: null, + webhookSecret: null, +}; +const config: Record = { ...UNSET }; let service: MailIngestSettingsService; beforeEach(async () => { @@ -23,11 +38,13 @@ beforeEach(async () => { stored.clear(); env.isManaged.mockReturnValue(false); prefs.findForUser.mockResolvedValue(null); + Object.assign(config, UNSET); const moduleRef = await Test.createTestingModule({ providers: [ MailIngestSettingsService, { provide: SettingsService, useValue: settings }, { provide: RuntimeEnvService, useValue: env }, + { provide: mailIngestConfig.KEY, useValue: config }, { provide: getRepositoryToken(MailIngestPrefs), useValue: prefs }, ], }).compile(); @@ -76,6 +93,46 @@ describe('MailIngestSettingsService', () => { expect(await service.allowedFileTypes()).toBe('pdf'); }); + it('takes the environment over every stored transport value', async () => { + stored.set('mail_ingest_transport', 'imap'); + stored.set('mail_ingest_address_template', 'trek+{token}@stored.example'); + config.transport = 'webhook'; + config.addressTemplate = 'in+{token}@env.example'; + expect(await service.transport()).toBe('webhook'); + expect(await service.addressTemplate()).toBe('in+{token}@env.example'); + }); + + it('reads the IMAP box only when complete, with its password decrypted and sound defaults', async () => { + stored.set('mail_ingest_imap_host', 'imap.example.com'); + stored.set('mail_ingest_imap_user', 'trips'); + expect(await service.imapConfig()).toBeNull(); + stored.set('mail_ingest_imap_pass', encrypt_api_key('box-secret')); + expect(await service.imapConfig()).toEqual({ + host: 'imap.example.com', + port: 993, + tls: 'tls', + user: 'trips', + pass: 'box-secret', + mailbox: 'INBOX', + }); + stored.set('mail_ingest_imap_tls', 'starttls'); + stored.set('mail_ingest_imap_port', 'nonsense'); + stored.set('mail_ingest_imap_mailbox', ' Bookings '); + expect(await service.imapConfig()).toMatchObject({ port: 143, tls: 'starttls', mailbox: 'Bookings' }); + config.imapHost = 'mail.lan'; + config.imapPort = '1143'; + config.imapPass = 'env-secret'; + expect(await service.imapConfig()).toMatchObject({ host: 'mail.lan', port: 1143, pass: 'env-secret' }); + }); + + it('reads the webhook secret decrypted, the environment first, none while unset', async () => { + expect(await service.webhookSecret()).toBeNull(); + stored.set('mail_ingest_webhook_secret', encrypt_api_key('stored-secret')); + expect(await service.webhookSecret()).toBe('stored-secret'); + config.webhookSecret = 'env-secret'; + expect(await service.webhookSecret()).toBe('env-secret'); + }); + it('has every switch on for a person who never changed one, and writes only what changes', async () => { expect(await service.prefsOf(4)).toEqual({ enabled: true, ai_enabled: true, auto_add: true }); prefs.findForUser.mockResolvedValue({ id: 1, user_id: 4, enabled: 1, ai_enabled: 0, auto_add: 1 }); diff --git a/server/tests/unit/nest/managed-keys.test.ts b/server/tests/unit/nest/managed-keys.test.ts index 9aee86f36..0436d2a67 100644 --- a/server/tests/unit/nest/managed-keys.test.ts +++ b/server/tests/unit/nest/managed-keys.test.ts @@ -65,6 +65,17 @@ describe('managed key assignment', () => { 'llm_model', 'llm_multimodal', 'llm_provider', + 'mail_ingest_address_template', + 'mail_ingest_ai_per_user_daily', + 'mail_ingest_enabled', + 'mail_ingest_imap_host', + 'mail_ingest_imap_mailbox', + 'mail_ingest_imap_pass', + 'mail_ingest_imap_port', + 'mail_ingest_imap_tls', + 'mail_ingest_imap_user', + 'mail_ingest_transport', + 'mail_ingest_webhook_secret', 'mapbox_access_token', 'maps_api_key', 'oidc_auto_redirect', diff --git a/shared/src/mail-ingest/mail-ingest.schema.spec.ts b/shared/src/mail-ingest/mail-ingest.schema.spec.ts index dc6701877..83364f8c5 100644 --- a/shared/src/mail-ingest/mail-ingest.schema.spec.ts +++ b/shared/src/mail-ingest/mail-ingest.schema.spec.ts @@ -8,6 +8,7 @@ import { import { mailIngestAddressSchema, mailIngestAdminStatusResponseSchema, + mailIngestAdminTestResponseSchema, mailIngestApplyRequestSchema, mailIngestCandidateSchema, mailIngestFileRequestSchema, @@ -137,6 +138,13 @@ describe('mail-ingest rows', () => { expect(mailIngestAdminStatusResponseSchema.safeParse(status).success).toBe(true); expect(mailIngestInboundResponseSchema.safeParse({ accepted: true, duplicate: false }).success).toBe(true); }); + + it('MAILC-016: the connection test names its transport and what failed', () => { + expect(mailIngestAdminTestResponseSchema.safeParse({ transport: 'imap', ok: true, error: null }).success).toBe(true); + const failed = { transport: 'webhook', ok: false, error: 'No webhook secret is set' }; + expect(mailIngestAdminTestResponseSchema.safeParse(failed).success).toBe(true); + expect(mailIngestAdminTestResponseSchema.safeParse({ transport: 'smtp', ok: true, error: null }).success).toBe(false); + }); }); describe('mail-ingest requests', () => { @@ -152,6 +160,7 @@ describe('mail-ingest requests', () => { expect(mailIngestApplyRequestSchema.safeParse({ trip_id: 3 }).success).toBe(true); expect(mailIngestApplyRequestSchema.safeParse({ trip_id: 0 }).success).toBe(false); expect(mailIngestMoveRequestSchema.safeParse({}).success).toBe(false); + expect(mailIngestMoveRequestSchema.safeParse({ trip_id: 3, confirm: true }).success).toBe(true); expect(mailIngestFileRequestSchema.safeParse({ trip_id: 3, reservation_id: 4 }).success).toBe(false); }); diff --git a/shared/src/mail-ingest/mail-ingest.schema.ts b/shared/src/mail-ingest/mail-ingest.schema.ts index 16455db58..e467b559d 100644 --- a/shared/src/mail-ingest/mail-ingest.schema.ts +++ b/shared/src/mail-ingest/mail-ingest.schema.ts @@ -229,8 +229,13 @@ export type MailIngestMessagePreviewResponse = z.infer; -/** POST /api/mail-ingest/candidates/:id/move: the added booking into another trip. */ -export const mailIngestMoveRequestSchema = z.strictObject({ trip_id: idSchema }); +/** + * POST /api/mail-ingest/candidates/:id/move: the added booking into another + * trip, written there anew. One whose cost has moved balances, or that was + * edited since, moves only with `confirm`, as an undo does (409 + * {@link mailIngestUndoConflictSchema}). + */ +export const mailIngestMoveRequestSchema = z.strictObject({ trip_id: idSchema, confirm: z.boolean().optional() }); export type MailIngestMoveRequest = z.infer; /** POST /api/mail-ingest/candidates/:id/file: the mail's attachments, or its text, as documents of a trip. */ @@ -331,3 +336,19 @@ export const mailIngestAdminStatusResponseSchema = z.object({ backlog: z.array(countSchema), }); export type MailIngestAdminStatusResponse = z.infer; + +/** + * POST /api/admin/mail-ingest/test: whether the configured transport can + * reach its mail now (the IMAP box opens, the webhook has its secret, the + * TREK Cloud relay is attached). `error` says what failed, never a password. + */ +export const mailIngestAdminTestResponseSchema = z.object({ + transport: mailIngestTransportSettingSchema, + ok: z.boolean(), + error: z.string().nullable(), +}); +export type MailIngestAdminTestResponse = z.infer; + +/** The IMAP box's encryption: TLS from the first byte (port 993), or STARTTLS on a plain port (143). */ +export const mailIngestImapTlsSchema = z.enum(['tls', 'starttls']); +export type MailIngestImapTls = z.infer; diff --git a/wiki/Environment-Variables.md b/wiki/Environment-Variables.md index a78fa2ad5..86401651a 100644 --- a/wiki/Environment-Variables.md +++ b/wiki/Environment-Variables.md @@ -236,6 +236,25 @@ optional (for unauthenticated relays). --- +## Booking mail import + +How forwarded booking mails reach TREK ([Booking mail import](Booking-Mail-Import)). Every variable overrides its field +in the admin dialog; unset, the dialog decides. + +| Variable | Description | Default | +|--------------------------------|-------------------------------------------------------------------------------------------------------------|---------| +| `MAIL_INGEST_TRANSPORT` | `off` (uploads only), `imap` or `webhook`. | `off` | +| `MAIL_INGEST_IMAP_HOST` | The mailbox TREK reads every two minutes. A host on your own network needs `ALLOW_INTERNAL_NETWORK=true`. | — | +| `MAIL_INGEST_IMAP_PORT` | `993` for TLS, `143` for STARTTLS. | `993` | +| `MAIL_INGEST_IMAP_TLS` | `tls` (encrypted from the first byte) or `starttls`. | `tls` | +| `MAIL_INGEST_IMAP_USER` | IMAP login. | — | +| `MAIL_INGEST_IMAP_PASS` | IMAP password. | — | +| `MAIL_INGEST_IMAP_MAILBOX` | The folder the mails arrive in. Read mails move on to `TREK/processed`; nothing is deleted. | `INBOX` | +| `MAIL_INGEST_ADDRESS_TEMPLATE` | The forwarding address, each person's token where `{token}` stands, e.g. `trek+{token}@example.com`. | — | +| `MAIL_INGEST_WEBHOOK_SECRET` | The key the webhook sender signs each mail with (HMAC-SHA256), at least 32 characters. | — | + +--- + ## Web Push [Web Push](Notifications#web-push) signs every message with a VAPID key pair. None of these variables is needed: