diff --git a/server/scripts/repository-owners.json b/server/scripts/repository-owners.json index 776187c78..5da042d8d 100644 --- a/server/scripts/repository-owners.json +++ b/server/scripts/repository-owners.json @@ -3,6 +3,7 @@ "AiConnections": "ai-connections", "AppSettings": "settings", "AssignmentParticipants": "assignments", + "AssistantAttachments": "assistant", "AssistantMessages": "assistant", "AssistantThreads": "assistant", "AssistantToolCalls": "assistant", diff --git a/server/src/db/entities/AssistantAttachments.entity.ts b/server/src/db/entities/AssistantAttachments.entity.ts new file mode 100644 index 000000000..22b8e574b --- /dev/null +++ b/server/src/db/entities/AssistantAttachments.entity.ts @@ -0,0 +1,60 @@ +import { EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core'; +import { AssistantAttachmentsRepository } from '../repositories/AssistantAttachments.repository'; +import { DbTimestampType } from '../types'; +import { AssistantThreads } from './AssistantThreads.entity'; +import { TripFiles } from './TripFiles.entity'; +import { Trips } from './Trips.entity'; +import { Users } from './Users.entity'; + +export class AssistantAttachments { + [EntityRepositoryType]?: AssistantAttachmentsRepository; + id!: string; + thread!: Ref; + thread_id!: number; + user!: Ref; + user_id!: number; + trip!: Ref; + trip_id!: number; + name!: string; + mime!: string; + size!: number; + sha256!: string; + storage_key!: string; + text?: string | null; + text_status!: string; + shared_at?: string | null; + file?: Ref | null; + file_id?: number | null; + created_at!: string & Opt; +} + +export const AssistantAttachmentsSchema = defineEntity({ + class: AssistantAttachments, + repository: () => AssistantAttachmentsRepository, + indexes: [ + { + name: 'idx_assistant_attachments_thread', + properties: ['thread_id', 'created_at'], + }, + ], + properties: { + id: p.text().primary(), + thread: () => p.manyToOne(AssistantThreads).ref().deleteRule('cascade').hidden(), + thread_id: p.integer().persist(false), + user: () => p.manyToOne(Users).ref().deleteRule('cascade').hidden().index('idx_assistant_attachments_user'), + user_id: p.integer().persist(false).index('idx_assistant_attachments_user'), + trip: () => p.manyToOne(Trips).ref().deleteRule('cascade').hidden().index('idx_assistant_attachments_trip'), + trip_id: p.integer().persist(false).index('idx_assistant_attachments_trip'), + name: p.text(), + mime: p.text(), + size: p.integer(), + sha256: p.text(), + storage_key: p.text(), + text: p.text().nullable(), + text_status: p.text(), + shared_at: p.type(DbTimestampType).nullable(), + file: () => p.manyToOne(TripFiles).ref().nullable().hidden().index('idx_assistant_attachments_file'), + file_id: p.integer().nullable().persist(false).index('idx_assistant_attachments_file'), + created_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`).index('idx_assistant_attachments_created'), + }, +}); diff --git a/server/src/db/entities/AssistantThreads.entity.ts b/server/src/db/entities/AssistantThreads.entity.ts index fb8fd654a..bdfccc910 100644 --- a/server/src/db/entities/AssistantThreads.entity.ts +++ b/server/src/db/entities/AssistantThreads.entity.ts @@ -1,6 +1,7 @@ import { Collection, EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core'; import { AssistantThreadsRepository } from '../repositories/AssistantThreads.repository'; import { DbTimestampType } from '../types'; +import { AssistantAttachments } from './AssistantAttachments.entity'; import { AssistantMessages } from './AssistantMessages.entity'; import { AssistantToolCalls } from './AssistantToolCalls.entity'; import { Journeys } from './Journeys.entity'; @@ -19,6 +20,7 @@ export class AssistantThreads { title!: string; created_at!: string & Opt; last_message_at!: string & Opt; + assistant_attachments_collection = new Collection(this); assistant_messages_collection = new Collection(this); assistant_tool_calls_collection = new Collection(this); } @@ -47,6 +49,7 @@ export const AssistantThreadsSchema = defineEntity({ title: p.text(), created_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`), last_message_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`), + assistant_attachments_collection: () => p.oneToMany(AssistantAttachments).mappedBy('thread').hidden(), assistant_messages_collection: () => p.oneToMany(AssistantMessages).mappedBy('thread').hidden(), assistant_tool_calls_collection: () => p.oneToMany(AssistantToolCalls).mappedBy('thread').hidden(), }, diff --git a/server/src/db/entities/TripFiles.entity.ts b/server/src/db/entities/TripFiles.entity.ts index a170af543..ece9b8986 100644 --- a/server/src/db/entities/TripFiles.entity.ts +++ b/server/src/db/entities/TripFiles.entity.ts @@ -1,6 +1,7 @@ import { Collection, EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core'; import { TripFilesRepository } from '../repositories/TripFiles.repository'; import { DbTimestampType } from '../types'; +import { AssistantAttachments } from './AssistantAttachments.entity'; import { CollabMessages } from './CollabMessages.entity'; import { CollabNotes } from './CollabNotes.entity'; import { DocumentSyncItems } from './DocumentSyncItems.entity'; @@ -33,6 +34,7 @@ export class TripFiles { message?: Ref | null; message_id?: number | null; uploadedByRef?: Ref | null; + assistant_attachments_collection = new Collection(this); document_sync_items_collection = new Collection(this); file_links_collection = new Collection(this); } @@ -62,6 +64,7 @@ export const TripFilesSchema = defineEntity({ message: () => p.manyToOne(CollabMessages).ref().deleteRule('cascade').nullable().hidden().index('idx_trip_files_message_id'), message_id: p.integer().nullable().persist(false).index('idx_trip_files_message_id'), uploadedByRef: () => p.manyToOne(Users).ref().joinColumn('uploaded_by').nullable().hidden().index('idx_trip_files_uploaded_by'), + assistant_attachments_collection: () => p.oneToMany(AssistantAttachments).mappedBy('file').hidden(), document_sync_items_collection: () => p.oneToMany(DocumentSyncItems).mappedBy('file').hidden(), file_links_collection: () => p.oneToMany(FileLinks).mappedBy('file').hidden(), }, diff --git a/server/src/db/entities/Trips.entity.ts b/server/src/db/entities/Trips.entity.ts index 0fab80c02..2a2aacc35 100644 --- a/server/src/db/entities/Trips.entity.ts +++ b/server/src/db/entities/Trips.entity.ts @@ -1,6 +1,7 @@ import { Collection, EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core'; import { TripsRepository } from '../repositories/Trips.repository'; import { DbTimestampType } from '../types'; +import { AssistantAttachments } from './AssistantAttachments.entity'; import { AssistantThreads } from './AssistantThreads.entity'; import { BudgetCategoryOrder } from './BudgetCategoryOrder.entity'; import { BudgetItems } from './BudgetItems.entity'; @@ -57,6 +58,7 @@ export class Trips { created_at?: string | null; updated_at?: string | null; reminder_sent_for?: string | null; + assistant_attachments_collection = new Collection(this); assistant_threads_collection = new Collection(this); budget_category_order_collection = new Collection(this); budget_items_collection = new Collection(this); @@ -122,6 +124,7 @@ export const TripsSchema = defineEntity({ created_at: p.type(DbTimestampType).nullable().defaultRaw(`CURRENT_TIMESTAMP`).index('idx_trips_created_at'), updated_at: p.type(DbTimestampType).nullable().defaultRaw(`CURRENT_TIMESTAMP`), reminder_sent_for: p.text().nullable(), + assistant_attachments_collection: () => p.oneToMany(AssistantAttachments).mappedBy('trip').hidden(), assistant_threads_collection: () => p.oneToMany(AssistantThreads).mappedBy('trip').hidden(), budget_category_order_collection: () => p.oneToMany(BudgetCategoryOrder).mappedBy('trip').hidden(), budget_items_collection: () => p.oneToMany(BudgetItems).mappedBy('trip').hidden(), diff --git a/server/src/db/entities/Users.entity.ts b/server/src/db/entities/Users.entity.ts index fd42d4787..1900e7132 100644 --- a/server/src/db/entities/Users.entity.ts +++ b/server/src/db/entities/Users.entity.ts @@ -3,6 +3,7 @@ import { UsersRepository } from '../repositories/Users.repository'; import { DbTimestampType } from '../types'; import { AiConnections } from './AiConnections.entity'; import { AssignmentParticipants } from './AssignmentParticipants.entity'; +import { AssistantAttachments } from './AssistantAttachments.entity'; import { AssistantThreads } from './AssistantThreads.entity'; import { AssistantUsage } from './AssistantUsage.entity'; import { AuditLog } from './AuditLog.entity'; @@ -135,6 +136,7 @@ export class Users { email_self_asserted: number & Opt = 0; ai_connections_collection = new Collection(this); assignment_participants_collection = new Collection(this); + assistant_attachments_collection = new Collection(this); assistant_threads_collection = new Collection(this); assistant_usage_collection = new Collection(this); audit_log_collection = new Collection(this); @@ -293,6 +295,7 @@ export const UsersSchema = defineEntity({ email_self_asserted: p.integer().default(0), ai_connections_collection: () => p.oneToMany(AiConnections).mappedBy('user').hidden(), assignment_participants_collection: () => p.oneToMany(AssignmentParticipants).mappedBy('user').hidden(), + assistant_attachments_collection: () => p.oneToMany(AssistantAttachments).mappedBy('user').hidden(), assistant_threads_collection: () => p.oneToMany(AssistantThreads).mappedBy('user').hidden(), assistant_usage_collection: () => p.oneToMany(AssistantUsage).mappedBy('user').hidden(), audit_log_collection: () => p.oneToMany(AuditLog).mappedBy('user').hidden(), diff --git a/server/src/db/entities/index.ts b/server/src/db/entities/index.ts index 590e6d741..5c0550197 100644 --- a/server/src/db/entities/index.ts +++ b/server/src/db/entities/index.ts @@ -3,6 +3,7 @@ import { AddonsSchema } from './Addons.entity'; import { AiConnectionsSchema } from './AiConnections.entity'; import { AppSettingsSchema } from './AppSettings.entity'; import { AssignmentParticipantsSchema } from './AssignmentParticipants.entity'; +import { AssistantAttachmentsSchema } from './AssistantAttachments.entity'; import { AssistantMessagesSchema } from './AssistantMessages.entity'; import { AssistantThreadsSchema } from './AssistantThreads.entity'; import { AssistantToolCallsSchema } from './AssistantToolCalls.entity'; @@ -166,6 +167,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [ AiConnectionsSchema, AppSettingsSchema, AssignmentParticipantsSchema, + AssistantAttachmentsSchema, AssistantMessagesSchema, AssistantThreadsSchema, AssistantToolCallsSchema, diff --git a/server/src/db/kysely/AssistantAttachments.table.ts b/server/src/db/kysely/AssistantAttachments.table.ts new file mode 100644 index 000000000..7d0374d8d --- /dev/null +++ b/server/src/db/kysely/AssistantAttachments.table.ts @@ -0,0 +1,20 @@ +// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand: +// change the migration, then run `npm run gen:db-types`. +import type { InsertOptional } from './columns'; + +export interface AssistantAttachmentsTable { + id: string; + thread_id: number; + user_id: number; + trip_id: number; + name: string; + mime: string; + size: number; + sha256: string; + storage_key: string; + text: InsertOptional; + text_status: string; + shared_at: InsertOptional; + file_id: InsertOptional; + created_at: InsertOptional; +} diff --git a/server/src/db/kysely/db.ts b/server/src/db/kysely/db.ts index e5fc05d93..01eedd8d8 100644 --- a/server/src/db/kysely/db.ts +++ b/server/src/db/kysely/db.ts @@ -4,6 +4,7 @@ import type { AddonsTable } from './Addons.table'; import type { AiConnectionsTable } from './AiConnections.table'; import type { AppSettingsTable } from './AppSettings.table'; import type { AssignmentParticipantsTable } from './AssignmentParticipants.table'; +import type { AssistantAttachmentsTable } from './AssistantAttachments.table'; import type { AssistantMessagesTable } from './AssistantMessages.table'; import type { AssistantThreadsTable } from './AssistantThreads.table'; import type { AssistantToolCallsTable } from './AssistantToolCalls.table'; @@ -173,6 +174,7 @@ export interface DB { ai_connections: AiConnectionsTable; app_settings: AppSettingsTable; assignment_participants: AssignmentParticipantsTable; + assistant_attachments: AssistantAttachmentsTable; assistant_messages: AssistantMessagesTable; assistant_threads: AssistantThreadsTable; assistant_tool_calls: AssistantToolCallsTable; diff --git a/server/src/db/migrations/Migration20200101046300_an_attached_file_stays_with_its_conversation.ts b/server/src/db/migrations/Migration20200101046300_an_attached_file_stays_with_its_conversation.ts new file mode 100644 index 000000000..0446d2cd0 --- /dev/null +++ b/server/src/db/migrations/Migration20200101046300_an_attached_file_stays_with_its_conversation.ts @@ -0,0 +1,50 @@ +import { Migration } from '@mikro-orm/migrations'; + +/** + * A file the user attaches in the trip chat stays with the conversation. Until + * now it lived only as a buffer for the one request that read it, so the + * assistant could neither look at it again nor keep it in the trip's Files. + * + * `assistant_attachments` keeps one row per attached file: who attached it, + * in which thread of which trip, its name, type, size and hash, the storage + * key its bytes sit under, and the text TREK read out of it with how that + * went (`text_status`). The text reaches the AI provider only after the user + * agreed for this one file (`shared_at`), and a file kept in Files names the + * trip file it became (`file_id`). + * + * The rows go with their thread, their user and their trip; a trip file the + * user deletes later only clears `file_id`. The retention sweep removes a row + * and its bytes after a week, or as soon as its thread is gone. Every foreign + * key leads an index, and `created_at` has one for that sweep. + */ +export class Migration20200101046300_an_attached_file_stays_with_its_conversation extends Migration { + override name = 'Migration20200101046300_an_attached_file_stays_with_its_conversation'; + + override up(): void { + this.addSql(` + CREATE TABLE IF NOT EXISTS assistant_attachments ( + id TEXT PRIMARY KEY NOT NULL, + thread_id INTEGER NOT NULL REFERENCES assistant_threads(id) ON DELETE CASCADE, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE, + name TEXT NOT NULL, + mime TEXT NOT NULL, + size INTEGER NOT NULL, + sha256 TEXT NOT NULL, + storage_key TEXT NOT NULL, + text TEXT, + text_status TEXT NOT NULL CHECK(text_status IN ('ok', 'empty', 'scanned', 'unsupported', 'failed')), + shared_at DATETIME, + file_id INTEGER REFERENCES trip_files(id) ON DELETE SET NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + this.addSql( + 'CREATE INDEX IF NOT EXISTS idx_assistant_attachments_thread ON assistant_attachments(thread_id, created_at)', + ); + this.addSql('CREATE INDEX IF NOT EXISTS idx_assistant_attachments_user ON assistant_attachments(user_id)'); + this.addSql('CREATE INDEX IF NOT EXISTS idx_assistant_attachments_trip ON assistant_attachments(trip_id)'); + this.addSql('CREATE INDEX IF NOT EXISTS idx_assistant_attachments_file ON assistant_attachments(file_id)'); + this.addSql('CREATE INDEX IF NOT EXISTS idx_assistant_attachments_created ON assistant_attachments(created_at)'); + } +} diff --git a/server/src/db/repositories/AssistantAttachments.repository.ts b/server/src/db/repositories/AssistantAttachments.repository.ts new file mode 100644 index 000000000..6ef783295 --- /dev/null +++ b/server/src/db/repositories/AssistantAttachments.repository.ts @@ -0,0 +1,196 @@ +import type { AssistantAttachments } from '../entities/AssistantAttachments.entity'; +import type { DB } from '../kysely/db'; +import type { AssertRowKeys } from './_shared/rows'; +import { TrekRepository } from './_shared/trek-repository'; + +/** How TREK's reading of an attached file went: text it read, none, a scan, a type it does not read, a failure. */ +export type AssistantAttachmentTextStatus = 'ok' | 'empty' | 'scanned' | 'unsupported' | 'failed'; + +/** One `assistant_attachments` row as stored, the text included. */ +export interface AssistantAttachmentRow { + id: string; + thread_id: number; + user_id: number; + trip_id: number; + name: string; + mime: string; + size: number; + sha256: string; + storage_key: string; + text: string | null; + text_status: string; + shared_at: string | null; + file_id: number | null; + created_at: string; +} + +const _assistantAttachmentRowKeys: AssertRowKeys = true; + +/** An attachment as a turn lists it: everything but its text. */ +export type AssistantAttachmentSummary = Omit; + +/** A new attachment, as AssistantAttachmentsService writes it. */ +export interface AssistantAttachmentInsert { + id: string; + threadId: number; + userId: number; + tripId: number; + name: string; + mime: string; + size: number; + sha256: string; + storageKey: string; + text: string | null; + textStatus: AssistantAttachmentTextStatus; + createdAt: string; +} + +/** Where an attachment's bytes sit, for the sweep and a thread's deletion. */ +export interface AssistantAttachmentObject { + id: string; + storage_key: string; +} + +type AttachmentsKyselyDB = Pick; + +const SUMMARY_COLUMNS = [ + 'id', + 'thread_id', + 'user_id', + 'trip_id', + 'name', + 'mime', + 'size', + 'sha256', + 'storage_key', + 'text_status', + 'shared_at', + 'file_id', + 'created_at', +] as const; + +/** + * `assistant_attachments`: the files users attached in a trip chat. Every + * read a turn makes names the thread and the user next to the id, so an + * attachment is only ever reached in the conversation it was attached in, + * by the user who attached it. + */ +export class AssistantAttachmentsRepository extends TrekRepository { + private db() { + return this.kysely(); + } + + async insertAttachment(row: AssistantAttachmentInsert): Promise { + await this.db() + .insertInto('assistant_attachments') + .values({ + id: row.id, + thread_id: row.threadId, + user_id: row.userId, + trip_id: row.tripId, + name: row.name, + mime: row.mime, + size: row.size, + sha256: row.sha256, + storage_key: row.storageKey, + text: row.text, + text_status: row.textStatus, + created_at: row.createdAt, + }) + .execute(); + } + + /** One attachment of the user's in this thread, with its text; null for anyone else's or another thread's. */ + async findInThread(id: string, threadId: number, userId: number): Promise { + const row = await this.db() + .selectFrom('assistant_attachments') + .selectAll() + .where('id', '=', id) + .where('thread_id', '=', threadId) + .where('user_id', '=', userId) + .executeTakeFirst(); + return row ?? null; + } + + /** The user's attachments in this thread, oldest first, without their text. */ + async listForThread(threadId: number, userId: number): Promise { + return this.db() + .selectFrom('assistant_attachments') + .select(SUMMARY_COLUMNS) + .where('thread_id', '=', threadId) + .where('user_id', '=', userId) + .orderBy('created_at', 'asc') + .orderBy('id', 'asc') + .execute(); + } + + /** The texts of this thread's attachments the user let the model read, by id. */ + async listSharedTexts(threadId: number, userId: number): Promise> { + const rows = await this.db() + .selectFrom('assistant_attachments') + .select(['id', 'text']) + .where('thread_id', '=', threadId) + .where('user_id', '=', userId) + .where('shared_at', 'is not', null) + .where('text', 'is not', null) + .execute(); + return rows.flatMap((row) => (row.text === null ? [] : [{ id: row.id, text: row.text }])); + } + + /** Records that the user let the model read the attachment; the first agreement's time stays. */ + async markShared(id: string, threadId: number, userId: number, at: string): Promise { + const result = await this.db() + .updateTable('assistant_attachments') + .set({ shared_at: at }) + .where('id', '=', id) + .where('thread_id', '=', threadId) + .where('user_id', '=', userId) + .where('shared_at', 'is', null) + .executeTakeFirst(); + return Number(result.numUpdatedRows) > 0; + } + + /** Names the trip file the attachment was kept as. */ + async setFile(id: string, fileId: number): Promise { + await this.db().updateTable('assistant_attachments').set({ file_id: fileId }).where('id', '=', id).execute(); + } + + /** Where the bytes of the user's attachments in this thread sit. */ + async listObjectsOfThread(threadId: number, userId: number): Promise { + return this.db() + .selectFrom('assistant_attachments') + .select(['id', 'storage_key']) + .where('thread_id', '=', threadId) + .where('user_id', '=', userId) + .execute(); + } + + /** Retention: the attachments made before `cutoff`, oldest first, at most `limit`. */ + async listCreatedBefore(cutoff: string, limit: number): Promise { + return this.db() + .selectFrom('assistant_attachments') + .select(['id', 'storage_key']) + .where('created_at', '<', cutoff) + .orderBy('created_at', 'asc') + .limit(limit) + .execute(); + } + + /** Which of these storage keys a row still names. */ + async listKnownKeys(keys: string[]): Promise { + if (keys.length === 0) return []; + const rows = await this.db() + .selectFrom('assistant_attachments') + .select('storage_key') + .where('storage_key', 'in', keys) + .execute(); + return rows.map((row) => row.storage_key); + } + + /** Drops these attachments; answers how many went. */ + async deleteByIds(ids: string[]): Promise { + if (ids.length === 0) return 0; + const result = await this.db().deleteFrom('assistant_attachments').where('id', 'in', ids).executeTakeFirst(); + return Number(result.numDeletedRows); + } +} diff --git a/server/src/nest/assistant/assistant-attachments.service.ts b/server/src/nest/assistant/assistant-attachments.service.ts new file mode 100644 index 000000000..9dd7db883 --- /dev/null +++ b/server/src/nest/assistant/assistant-attachments.service.ts @@ -0,0 +1,201 @@ +import { AssistantAttachments } from '../../db/entities/AssistantAttachments.entity'; +import type { + AssistantAttachmentObject, + AssistantAttachmentsRepository, + AssistantAttachmentSummary, + AssistantAttachmentTextStatus, +} from '../../db/repositories/AssistantAttachments.repository'; +import { dbNow } from '../../db/types'; +import { logError, logInfo } from '../audit/audit-log.logger'; +import { LlmParseService } from '../llm-parse/llm-parse.service'; +import { StorageService } from '../storage/storage.service'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Injectable } from '@nestjs/common'; +import { LLM_PHOTO_EXTENSIONS } from '@trek/shared'; + +import { createHash, randomUUID } from 'node:crypto'; +import { extname } from 'node:path'; +import { Readable } from 'node:stream'; + +/** How long an attached file is kept with its conversation. */ +export const ASSISTANT_ATTACHMENT_TTL_DAYS = 7; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** + * Where the bytes sit: under the trip files' category, which is never served + * by path (`/uploads/files` refuses every request), in a folder of their own. + */ +const STORAGE_CATEGORY = 'files'; +const KEY_PREFIX = 'assistant/'; + +/** The most text kept of one file; a model is sent a window of it, never all of it. */ +const MAX_STORED_TEXT_CHARS = 200_000; + +/** Rows one sweep pass takes; the next night takes the rest. */ +const SWEEP_BATCH = 500; + +/** Bytes put moments ago may belong to a turn whose row is not written yet; the sweep leaves them for an hour. */ +const ORPHAN_GRACE_MS = 60 * 60_000; + +/** Files whose bytes hold no text TREK reads: photos, and the zipped wallet passes. */ +const UNREAD_EXTENSIONS: ReadonlySet = new Set([...LLM_PHOTO_EXTENSIONS, '.pkpass']); + +/** A file stored for a turn that has not written its rows yet. */ +export interface PreparedAttachment { + id: string; + name: string; + mime: string; + size: number; + sha256: string; + storageKey: string; + text: string | null; + textStatus: AssistantAttachmentTextStatus; +} + +/** A plain extension of a file name for its storage key, or none when the name carries something else. */ +function keyExtension(name: string): string { + const ext = extname(name).toLowerCase(); + return /^\.[a-z0-9]{1,10}$/.test(ext) ? ext : ''; +} + +/** + * The files users attach in the trip chat, kept with the conversation + * (`assistant_attachments`): the bytes in storage, the text TREK read out of + * them in the row. The text stays on the server; it reaches the model only + * where the user agreed for that one file, and never a message part. + */ +@Injectable() +export class AssistantAttachmentsService { + constructor( + @InjectRepository(AssistantAttachments) private readonly rows: AssistantAttachmentsRepository, + private readonly storage: StorageService, + private readonly llmParse: LlmParseService, + ) {} + + /** + * Puts the file's bytes into storage and reads its text, before the turn + * writes anything: a storage that refuses the bytes refuses the turn. A file + * whose text cannot be read is kept all the same, as `failed`. + */ + async prepare(file: Express.Multer.File, name: string): Promise { + const id = randomUUID(); + const storageKey = `${KEY_PREFIX}${id}${keyExtension(name)}`; + const mime = file.mimetype || 'application/octet-stream'; + await this.storage.put(STORAGE_CATEGORY, storageKey, Readable.from([file.buffer]), { contentType: mime }); + const sha256 = createHash('sha256').update(file.buffer).digest('hex'); + return { id, name, mime, size: file.size, sha256, storageKey, ...(await this.textOf(file.buffer, name)) }; + } + + /** The row of a prepared file, in the thread its turn opened; written inside the turn's opening transaction. */ + async record(prepared: PreparedAttachment, at: { threadId: number; userId: number; tripId: number }): Promise { + await this.rows.insertAttachment({ + id: prepared.id, + threadId: at.threadId, + userId: at.userId, + tripId: at.tripId, + name: prepared.name, + mime: prepared.mime, + size: prepared.size, + sha256: prepared.sha256, + storageKey: prepared.storageKey, + text: prepared.text, + textStatus: prepared.textStatus, + createdAt: dbNow(), + }); + } + + /** The bytes of a file whose turn never began; the sweep would find them otherwise, a night later. */ + async discard(prepared: PreparedAttachment): Promise { + await this.deleteObject(prepared.storageKey); + } + + /** The user's attachments in a thread, without their text. */ + listForThread(threadId: number, userId: number): Promise { + return this.rows.listForThread(threadId, userId); + } + + /** Where the bytes of a thread's attachments sit, read before the thread is deleted. */ + objectsOfThread(threadId: number, userId: number): Promise { + return this.rows.listObjectsOfThread(threadId, userId); + } + + /** + * Deletes the bytes of a thread that is gone; its rows went with it. An + * object that cannot be deleted now is left for the sweep. + */ + async dropObjects(objects: readonly AssistantAttachmentObject[]): Promise { + await Promise.all(objects.map((object) => this.deleteObject(object.storage_key))); + } + + /** + * The nightly sweep: attachments older than the TTL (bytes first, then the + * row, so no row ever names bytes that are gone), then bytes no row names + * any more (a thread or a user deleted with its rows, a turn that never + * began). Answers how many attachments and loose objects went. + * + * @txIndependent a retention sweep in passes: each object is deleted from + * storage on its own, and a row only after its bytes. + */ + async sweep(now: Date = new Date()): Promise<{ expired: number; orphaned: number }> { + const cutoff = dbNow(new Date(now.getTime() - ASSISTANT_ATTACHMENT_TTL_DAYS * DAY_MS)); + const old = await this.rows.listCreatedBefore(cutoff, SWEEP_BATCH); + const gone = await Promise.all( + old.map(async (row) => ((await this.deleteObject(row.storage_key)) ? row.id : null)), + ); + const expired = await this.rows.deleteByIds(gone.filter((id): id is string => id !== null)); + const orphaned = await this.sweepOrphans(now); + if (expired + orphaned > 0) { + logInfo(`Assistant retention: removed ${expired} attached file(s) and ${orphaned} loose object(s)`); + } + return { expired, orphaned }; + } + + /** Objects under the attachments' folder that no row names, older than the grace period. */ + private async sweepOrphans(now: Date): Promise { + const listed: string[] = []; + for await (const stat of this.storage.list(STORAGE_CATEGORY, KEY_PREFIX)) { + if (now.getTime() - stat.mtimeMs > ORPHAN_GRACE_MS) listed.push(stat.key); + } + const pages = Array.from({ length: Math.ceil(listed.length / SWEEP_BATCH) }, (_page, i) => + listed.slice(i * SWEEP_BATCH, (i + 1) * SWEEP_BATCH), + ); + // One page after another, so a large backlog never asks the database for all its keys at once. + return pages.reduce>(async (previous, page) => { + const done = await previous; + const known = new Set(await this.rows.listKnownKeys(page)); + const deleted = await Promise.all(page.filter((key) => !known.has(key)).map((key) => this.deleteObject(key))); + return done + deleted.filter(Boolean).length; + }, Promise.resolve(0)); + } + + /** Deletes one object; a failure is logged and answered as false, the next sweep tries again. */ + private async deleteObject(key: string): Promise { + try { + await this.storage.delete(STORAGE_CATEGORY, key); + return true; + } catch (err) { + logError( + `[assistant] could not delete attached file ${key}: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** The text TREK reads out of a file, and how that went. Never throws. */ + private async textOf( + buffer: Buffer, + name: string, + ): Promise<{ text: string | null; textStatus: AssistantAttachmentTextStatus }> { + const ext = extname(name).toLowerCase(); + if (UNREAD_EXTENSIONS.has(ext)) return { text: null, textStatus: 'unsupported' }; + try { + const text = (await this.llmParse.readText(buffer, name)).trim(); + if (text) return { text: text.slice(0, MAX_STORED_TEXT_CHARS), textStatus: 'ok' }; + return { text: null, textStatus: ext === '.pdf' ? 'scanned' : 'empty' }; + } catch (err) { + logError(`[assistant] could not read the text of ${name}: ${err instanceof Error ? err.message : String(err)}`); + return { text: null, textStatus: 'failed' }; + } + } +} diff --git a/server/src/nest/assistant/assistant-retention.job.ts b/server/src/nest/assistant/assistant-retention.job.ts index 0feefe1c4..8f27fdd23 100644 --- a/server/src/nest/assistant/assistant-retention.job.ts +++ b/server/src/nest/assistant/assistant-retention.job.ts @@ -7,6 +7,7 @@ import { AiConnectionsService } from '../ai-connections/ai-connections.service'; import { AiUsageService } from '../ai-connections/ai-usage.service'; import { logError, logInfo } from '../audit/audit-log.logger'; import { CronRegistrarService } from '../scheduling/cron-registrar.service'; +import { AssistantAttachmentsService } from './assistant-attachments.service'; import { decisionCutoff } from './assistant-tools.constants'; import { InjectRepository } from '@mikro-orm/nestjs'; import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; @@ -20,8 +21,9 @@ const DAY_MS = 24 * 60 * 60 * 1000; * Nightly sweep of the assistant's conversations: threads nobody wrote in * for longer than the admin's retention (none under "forever"), threads of * users who no longer belong to their trip or journey, changes that waited a day - * without a decision (expired, so no card waits forever), and usage days - * older than the usage retention. + * without a decision (expired, so no card waits forever), usage days + * older than the usage retention, and the files attached in the chat once + * they are a week old or their conversation is gone. */ @Injectable() export class AssistantRetentionJob implements OnApplicationBootstrap { @@ -31,6 +33,7 @@ export class AssistantRetentionJob implements OnApplicationBootstrap { private readonly connections: AiConnectionsService, private readonly usage: AiUsageService, private readonly registrar: CronRegistrarService, + private readonly attachments: AssistantAttachmentsService, ) {} onApplicationBootstrap(): void { @@ -54,6 +57,7 @@ export class AssistantRetentionJob implements OnApplicationBootstrap { const expired = await this.toolCalls.expireBefore(decisionCutoff(now)); if (expired.length > 0) logInfo(`Assistant retention: ${expired.length} proposed change(s) expired`); await this.usage.purgeOlderThan(ASSISTANT_USAGE_RETENTION_DAYS); + await this.attachments.sweep(now); } catch (err: unknown) { logError(`Assistant retention: ${err instanceof Error ? err.message : String(err)}`); } diff --git a/server/src/nest/assistant/assistant.module.ts b/server/src/nest/assistant/assistant.module.ts index 128c19874..b2e1263d3 100644 --- a/server/src/nest/assistant/assistant.module.ts +++ b/server/src/nest/assistant/assistant.module.ts @@ -1,3 +1,4 @@ +import { AssistantAttachments } from '../../db/entities/AssistantAttachments.entity'; import { AssistantMessages } from '../../db/entities/AssistantMessages.entity'; import { AssistantThreads } from '../../db/entities/AssistantThreads.entity'; import { AssistantToolCalls } from '../../db/entities/AssistantToolCalls.entity'; @@ -10,14 +11,17 @@ import { CollabModule } from '../collab/collab.module'; import { RateLimitModule } from '../common/rate-limit.module'; import { JourneyBookModule } from '../journey/journey-book.module'; import { JourneyDomainModule } from '../journey/journey-domain.module'; +import { LlmParseModule } from '../llm-parse/llm-parse.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; import { PermissionsModule } from '../permissions/permissions.module'; import { PhotoSignalsModule } from '../photos/photo-signals.module'; import { SchedulingModule } from '../scheduling/scheduling.module'; import { SettingsModule } from '../settings/settings.module'; +import { StorageModule } from '../storage/storage.module'; import { TourFilesModule } from '../tours/tour-files.module'; import { TripMembershipModule } from '../trip-membership/trip-membership.module'; import { AssistantAttachmentsController } from './assistant-attachments.controller'; +import { AssistantAttachmentsService } from './assistant-attachments.service'; import { AssistantBookContext } from './assistant-book-context'; import { AssistantBookGate } from './assistant-book-gate'; import { AssistantBookingImportService } from './assistant-booking-import.service'; @@ -84,7 +88,11 @@ import { Module } from '@nestjs/common'; TripMembershipModule, // P7: a route file attached in the chat, read into its tracks. TourFilesModule, - MikroOrmModule.forFeature([AssistantThreads, AssistantMessages, AssistantToolCalls]), + // A file attached in the chat is kept with its conversation: its bytes in storage, its text read as the + // AI import reads it. + StorageModule, + LlmParseModule, + MikroOrmModule.forFeature([AssistantThreads, AssistantMessages, AssistantToolCalls, AssistantAttachments]), ], controllers: [ AssistantController, @@ -101,6 +109,7 @@ import { Module } from '@nestjs/common'; AssistantToolsService, AssistantBookingImportService, AssistantFileAnswers, + AssistantAttachmentsService, AssistantTourFileService, AssistantFoundPlacesService, AssistantProposalsService, diff --git a/server/src/nest/assistant/assistant.service.ts b/server/src/nest/assistant/assistant.service.ts index 9be2f3b15..e122d8a78 100644 --- a/server/src/nest/assistant/assistant.service.ts +++ b/server/src/nest/assistant/assistant.service.ts @@ -34,6 +34,7 @@ import { UnitOfWork } from '../database/unit-of-work'; import { SettingsService } from '../settings/settings.service'; import { runAgentLoop, type AgentStore, type HeldCall, type StepRecord } from './assistant-agent'; import type { AttachedFileReader } from './assistant-attached-file'; +import { AssistantAttachmentsService } from './assistant-attachments.service'; import { AssistantBookingImportService } from './assistant-booking-import.service'; import { mentionItems } from './assistant-cards.helpers'; import { todayIn } from './assistant-context'; @@ -131,6 +132,15 @@ interface TurnSpec { renders?: boolean; } +/** What a turn opens with: its thread (null for a new one), the user's message, and what is written with them. */ +interface TurnOpening { + threadId: number | null; + titleText: string; + model: string; + userParts: AssistantMessagePart[]; + alongside?: (threadId: number) => Promise; +} + function summaryOf(thread: AssistantThreadRow): AssistantThreadSummary { return { id: thread.id, @@ -170,6 +180,7 @@ export class AssistantService { private readonly tools: AssistantToolsService, private readonly bookings: AssistantBookingImportService, private readonly fileAnswers: AssistantFileAnswers, + private readonly attachments: AssistantAttachmentsService, private readonly tourFiles: AssistantTourFileService, private readonly settings: SettingsService, private readonly rateLimit: RateLimitService, @@ -192,8 +203,11 @@ export class AssistantService { return { thread: summaryOf(thread), messages: toThreadMessages(rows, await this.callsOf(rows), new Date()) }; } + /** Deletes the thread with its rows, then the bytes of the files attached in it. */ async deleteThread(userId: number, scope: AssistantScope, threadId: number): Promise { + const kept = scope.kind === 'trip' ? await this.attachments.objectsOfThread(threadId, userId) : []; if (!(await scopedThreads(this.threads, scope).remove(threadId, userId))) throw threadNotFound(); + await this.attachments.dropObjects(kept); } /** Stops the user's running turn, wherever it runs; nothing to stop is not an error. */ @@ -219,10 +233,15 @@ export class AssistantService { // The labels the message keeps come from the trip, never from the request. const card = mentions.length > 0 && tripId !== null ? await readTripCard(this.tools, userId, tripId) : null; const items = card ? mentionItems(card, mentions, req.locale) : []; - const begun = await this.beginTurn(userId, scope, req.threadId, req.text, connection.model, [ - { type: 'text', text: req.text }, - ...(items.length > 0 ? [{ type: 'data-trek-mentions' as const, data: { items } }] : []), - ]); + const begun = await this.beginTurn(userId, scope, { + threadId: req.threadId, + titleText: req.text, + model: connection.model, + userParts: [ + { type: 'text', text: req.text }, + ...(items.length > 0 ? [{ type: 'data-trek-mentions' as const, data: { items } }] : []), + ], + }); const stream = channel.open(); const w = new UiStreamWriter(stream); w.start(messageIdOf(begun.rowId)); @@ -274,9 +293,18 @@ export class AssistantService { await this.withinDailyLimit(userId, connection); await this.claimed(userId, channel.signal, async (signal) => { const name = file.originalname.slice(0, MAX_FILE_NAME_CHARS); - const begun = await this.beginTurn(userId, tripScope(tripId), req.threadId ?? null, name, connection.model, [ - { type: 'attachment', name, mime: file.mimetype || 'application/octet-stream', size: file.size }, - ]); + // The file is kept before anything is written: a storage that refuses it refuses the turn. + const kept = await this.attachments.prepare(file, name); + const begun = await this.beginTurn(userId, tripScope(tripId), { + threadId: req.threadId ?? null, + titleText: name, + model: connection.model, + userParts: [{ type: 'attachment', name, mime: kept.mime, size: file.size, attachmentId: kept.id }], + alongside: (threadId) => this.attachments.record(kept, { threadId, userId, tripId }), + }).catch(async (err: unknown) => { + await this.attachments.discard(kept); + throw err; + }); const stream = channel.open(); const w = new UiStreamWriter(stream); w.start(messageIdOf(begun.rowId)); @@ -499,17 +527,12 @@ export class AssistantService { /** * A turn's opening rows, in one transaction: the thread (found or, with * `threadId` null, created and titled from `titleText`), the user's message, - * and the row the answer's first step writes into. Changes still waiting in + * and the row the answer's first step writes into, with whatever goes + * `alongside` them (the row of an attached file). Changes still waiting in * the thread are cancelled: the user moved on. */ - private async beginTurn( - userId: number, - scope: AssistantScope, - threadId: number | null, - titleText: string, - model: string, - userParts: AssistantMessagePart[], - ) { + private async beginTurn(userId: number, scope: AssistantScope, opening: TurnOpening) { + const { threadId, titleText, model, userParts, alongside } = opening; const now = dbNow(); const threads = scopedThreads(this.threads, scope); return this.uow.transactional(async () => { @@ -535,6 +558,7 @@ export class AssistantService { }); const rowId = await this.messages.append({ ...base, role: 'assistant', parts: '[]', status: 'aborted', model }); await this.threads.touch(id, now); + await alongside?.(id); return { threadId: id, title, created: threadId === null, rowId }; }); } diff --git a/server/src/nest/llm-parse/llm-parse.service.ts b/server/src/nest/llm-parse/llm-parse.service.ts index 4eef238c5..d21447b15 100644 --- a/server/src/nest/llm-parse/llm-parse.service.ts +++ b/server/src/nest/llm-parse/llm-parse.service.ts @@ -69,6 +69,25 @@ export class LlmParseService { return this.llmConfig.readInstanceEndpoint(); } + /** + * The text of a document as the AI fallback reads it (an e-mail's body, a + * PDF's text layer, a page without its markup), for a caller outside this + * module: the assistant keeps it with a file attached in the chat. A scanned + * PDF answers an empty string. Asks no model. + */ + readText(buffer: Buffer, fileName: string): Promise { + return extractText(buffer, fileName); + } + + /** + * A text cut to what one model request should carry of it, its start and + * its end kept: the cloud window, or the compact one for a server whose + * context may be small (text-window.ts). + */ + fitText(text: string, compact: boolean): string { + return fitTextWindow(text, compact ? COMPACT_TEXT_WINDOW : CLOUD_TEXT_WINDOW).text; + } + /** True when the addon is enabled AND a usable config resolves for this user. */ async isAvailable(userId: number): Promise { return (await this.llmConfig.resolve(userId)) !== null; diff --git a/server/tests/e2e/assistant-attachments.e2e.test.ts b/server/tests/e2e/assistant-attachments.e2e.test.ts new file mode 100644 index 000000000..b80a23e34 --- /dev/null +++ b/server/tests/e2e/assistant-attachments.e2e.test.ts @@ -0,0 +1,257 @@ +/** + * A file attached in the trip chat, end to end: the whole app with an instance + * connection of kind `custom` pointing at a fake OpenAI-compatible server on + * loopback, as in assistant.e2e.test.ts. The booking import's extractor (an + * external binary) is stubbed on BookingImportService; storage and the text + * reading are the real ones. + * + * Covered: the file is kept with its conversation (a row and its bytes) under + * the id its message part carries, and goes with the thread; a storage that + * refuses the bytes refuses the turn before anything is written. + */ +import { buildApp } from '../../src/bootstrap'; +import { AssistantAttachments } from '../../src/db/entities/AssistantAttachments.entity'; +import { AssistantThreads } from '../../src/db/entities/AssistantThreads.entity'; +import { BookingImportService } from '../../src/nest/booking-import/booking-import.service'; +import { StorageService } from '../../src/nest/storage/storage.service'; +import { countRows, findRow } from '../helpers/factories/rows'; +import { setAddonEnabled } from '../helpers/factories/settings'; +import { makeTrip } from '../helpers/factories/trips'; +import { makeUser } from '../helpers/factories/users'; +import { resetRateLimits } from '../helpers/test-db'; +import { sessionCookie } from './harness'; +import { MikroORM } from '@mikro-orm/core'; +import type { INestApplication } from '@nestjs/common'; +import type { BookingImportPreviewResponse, UiStreamChunk } from '@trek/shared'; + +import { createServer, type IncomingMessage, type Server as HttpServer, type ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +// ── The fake provider ───────────────────────────────────────────────────────── + +interface ChatBody { + model: string; + messages: Array<{ role: string; content: string | null; tool_call_id?: string; tool_calls?: unknown[] }>; + tools?: Array<{ function: { name: string; parameters: { properties?: Record } } }>; +} + +type Step = (body: ChatBody) => { chunks: object[] }; + +const provider = { requests: [] as ChatBody[], script: [] as Step[] }; + +const say = + (text: string): Step => + () => ({ + chunks: [ + { choices: [{ index: 0, delta: { content: text } }] }, + { choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] }, + { choices: [], usage: { prompt_tokens: 12, completion_tokens: 4 } }, + ], + }); + +async function readBody(req: IncomingMessage): Promise { + let text = ''; + for await (const chunk of req) text += String(chunk); + return text; +} + +async function answer(req: IncomingMessage, res: ServerResponse): Promise { + if (req.method !== 'POST' || req.url !== '/v1/chat/completions') { + res.writeHead(404).end(); + return; + } + const body = JSON.parse(await readBody(req)) as ChatBody; + provider.requests.push(body); + const reply = (provider.script.shift() ?? say('Nothing more.'))(body); + res.writeHead(200, { 'content-type': 'text/event-stream' }); + for (const chunk of reply.chunks) res.write(`data: ${JSON.stringify(chunk)}\n\n`); + res.end('data: [DONE]\n\n'); +} + +// ── The app ────────────────────────────────────────────────────────────────── + +let nestApp: INestApplication; +let orm: MikroORM; +let fake: HttpServer; + +beforeAll(async () => { + fake = createServer((req, res) => { + answer(req, res).catch(() => res.destroy()); + }); + await new Promise((resolve) => fake.listen(0, '127.0.0.1', resolve)); + nestApp = await buildApp(); + orm = nestApp.get(MikroORM); + await setAddonEnabled(orm, 'ai_assistant', true); + const adminId = (await makeUser(orm, { role: 'admin' })).user.id; + const { port } = fake.address() as AddressInfo; + const res = await request(nestApp.getHttpServer()) + .put('/api/admin/ai/connection') + .set('Cookie', sessionCookie(adminId)) + .send({ kind: 'custom', baseUrl: `http://127.0.0.1:${port}/v1`, model: 'fake-model' }); + expect(res.status).toBe(200); +}); + +beforeEach(async () => { + await resetRateLimits(nestApp); + provider.requests = []; + provider.script = []; +}); + +afterAll(async () => { + vi.restoreAllMocks(); + await nestApp.close(); + await new Promise((resolve) => fake.close(() => resolve())); +}); + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const CTX = { locale: 'en', timeZone: 'Europe/Berlin' }; + +interface Answer { + status: number; + chunks: UiStreamChunk[]; + json: Record | null; +} + +function asText(req: request.Test): request.Test { + return req.buffer(true).parse((res, done) => { + let text = ''; + res.setEncoding('utf8'); + res.on('data', (chunk: string) => (text += chunk)); + res.on('end', () => done(null, text)); + }); +} + +function parseAnswer(status: number, contentType: string, text: string): Answer { + if (!contentType.includes('text/event-stream')) { + return { status, chunks: [], json: text ? (JSON.parse(text) as Record) : null }; + } + const events = text + .split('\n\n') + .filter((event) => event.startsWith('data: ')) + .map((event) => event.slice('data: '.length)); + expect(events.at(-1)).toBe('[DONE]'); + return { status, chunks: events.slice(0, -1).map((event) => JSON.parse(event) as UiStreamChunk), json: null }; +} + +async function post(path: string, userId: number, body: object): Promise { + const res = await asText(request(nestApp.getHttpServer()).post(path).set('Cookie', sessionCookie(userId)).send(body)); + return parseAnswer(res.status, String(res.headers['content-type']), res.body as string); +} + +/** The booking file as the client uploads it: multipart, the context as text fields. */ +const BOOKING_TEXT = 'Your stay at Casa Lisboa\nCheck-in 2026-06-11\nConfirmation HM4X2'; + +async function upload(path: string, userId: number, opts: { name?: string; threadId?: number } = {}): Promise { + const name = opts.name ?? 'AirBnB Buchung.txt'; + let req = request(nestApp.getHttpServer()) + .post(path) + .set('Cookie', sessionCookie(userId)) + .field('locale', CTX.locale) + .field('timeZone', CTX.timeZone); + if (opts.threadId !== undefined) req = req.field('threadId', String(opts.threadId)); + req = req.attach('file', Buffer.from(BOOKING_TEXT), { filename: name, contentType: 'text/plain' }); + const res = await asText(req); + return parseAnswer(res.status, String(res.headers['content-type']), res.body as string); +} + +function get(path: string, userId: number) { + return request(nestApp.getHttpServer()).get(path).set('Cookie', sessionCookie(userId)); +} + +const ofType = (chunks: UiStreamChunk[], type: T) => + chunks.filter((chunk): chunk is Extract => chunk.type === type); + +async function travelerWithTrip() { + const { user } = await makeUser(orm); + const trip = await makeTrip(orm, user.id, { title: 'Lisbon', start_date: '2026-06-11', end_date: '2026-06-14' }); + return { userId: user.id, tripId: trip.id, base: `/api/trips/${trip.id}/assistant` }; +} + +/** The booking import with its extractor on, the AI off, answering `preview`. */ +function stubImport(preview: BookingImportPreviewResponse) { + const bookingImport = nestApp.get(BookingImportService); + const spies = [ + vi.spyOn(bookingImport, 'isAvailable').mockReturnValue(true), + vi.spyOn(bookingImport, 'aiAvailable').mockResolvedValue(false), + vi.spyOn(bookingImport, 'preview').mockResolvedValue(preview), + ]; + return () => spies.forEach((spy) => spy.mockRestore()); +} + +const ONE_BOOKING: BookingImportPreviewResponse = { + items: [{ type: 'hotel', title: 'Casa Lisboa', source: { fileName: 'AirBnB Buchung.txt', index: 0 } }], + warnings: [], +}; + +/** The attachment the thread's first message names, and its row. */ +async function keptAttachment(t: { userId: number; base: string }, threadId: number) { + const shown = await get(`${t.base}/threads/${threadId}`, t.userId); + const part = (shown.body.messages[0].parts as Array<{ type: string; attachmentId?: string }>)[0]; + const row = await findRow(orm, AssistantAttachments, { id: part.attachmentId ?? '' }); + return { part, row }; +} + +// ── Cases ──────────────────────────────────────────────────────────────────── + +describe('Trip assistant: attached files', () => { + it('ATTE2E-001: an attached file is kept with its conversation under the id its message names, and goes with it', async () => { + const t = await travelerWithTrip(); + const restore = stubImport(ONE_BOOKING); + let threadId: number; + try { + const turn = await upload(`${t.base}/attachments`, t.userId); + expect(turn.status).toBe(200); + threadId = ofType(turn.chunks, 'data-trek-thread')[0].data.threadId; + } finally { + restore(); + } + const { part, row } = await keptAttachment(t, threadId); + expect(part).toMatchObject({ type: 'attachment', name: 'AirBnB Buchung.txt', mime: 'text/plain' }); + expect(row).toMatchObject({ + thread_id: threadId, + user_id: t.userId, + trip_id: t.tripId, + name: 'AirBnB Buchung.txt', + text: BOOKING_TEXT, + text_status: 'ok', + shared_at: null, + file_id: null, + }); + const storage = nestApp.get(StorageService); + const key = String(row?.storage_key); + expect(key.startsWith('assistant/')).toBe(true); + expect(await storage.exists('files', key)).toBe(true); + // The text never rides along in the thread. + const shown = await get(`${t.base}/threads/${threadId}`, t.userId); + expect(JSON.stringify(shown.body)).not.toContain('HM4X2'); + + const other = await makeUser(orm); + expect((await get(`${t.base}/threads/${threadId}`, other.user.id)).status).toBe(404); + + const removed = await request(nestApp.getHttpServer()) + .delete(`${t.base}/threads/${threadId}`) + .set('Cookie', sessionCookie(t.userId)); + expect(removed.status).toBe(200); + expect(await countRows(orm, AssistantAttachments, { thread: threadId })).toBe(0); + expect(await storage.exists('files', key)).toBe(false); + }); + + it('ATTE2E-002: a storage that refuses the bytes refuses the turn before anything is written', async () => { + const t = await travelerWithTrip(); + const restore = stubImport(ONE_BOOKING); + const put = vi.spyOn(nestApp.get(StorageService), 'put').mockRejectedValue(new Error('bucket is gone')); + try { + const turn = await upload(`${t.base}/attachments`, t.userId); + expect(turn.status).toBe(500); + expect(turn.chunks).toEqual([]); + } finally { + put.mockRestore(); + restore(); + } + expect(await countRows(orm, AssistantThreads, { trip: t.tripId })).toBe(0); + expect(await countRows(orm, AssistantAttachments, { trip: t.tripId })).toBe(0); + }); +}); diff --git a/server/tests/e2e/assistant.e2e.test.ts b/server/tests/e2e/assistant.e2e.test.ts index 571825837..4bdcca0df 100644 --- a/server/tests/e2e/assistant.e2e.test.ts +++ b/server/tests/e2e/assistant.e2e.test.ts @@ -732,7 +732,7 @@ describe('Trip assistant e2e', () => { expect(provider.requests).toHaveLength(0); const shown = await get(`${t.base}/threads/${threadId}`, t.userId); expect(shown.body.messages[0].parts).toEqual([ - { type: 'attachment', name: 'ticket.pdf', mime: 'application/pdf', size: 9 }, + { type: 'attachment', name: 'ticket.pdf', mime: 'application/pdf', size: 9, attachmentId: expect.any(String) }, ]); expect(shown.body.messages[1].status).toBe('awaiting_approval'); diff --git a/server/tests/unit/db/entities-index.test.ts b/server/tests/unit/db/entities-index.test.ts index cf549c779..a003f4c80 100644 --- a/server/tests/unit/db/entities-index.test.ts +++ b/server/tests/unit/db/entities-index.test.ts @@ -56,14 +56,15 @@ describe('ALL_ENTITIES', () => { * the trip assistant's conversations, the changes it proposed and what it used * + trek_photo_signals, what a photo's pixels say, for picking photos for a book * + journey_book_revisions, the books a whole-book design replaced, for its undo - * + trip_member_removals, who another member took off a trip, so its invite link refuses them) + * + trip_member_removals, who another member took off a trip, so its invite link refuses them + * + assistant_attachments, the files attached in the trip chat, kept with their conversation) * is registered, and no two share a class name (a silent duplicate would * shadow one of them in `orm.getMetadata()`, exactly the kind of drift * `entity-schema-parity.test.ts` can't see — it iterates the metadata map, * which de-duplicates by class name). */ - it('ENT-003: exactly 161 entities, every class name unique', () => { - expect(ALL_ENTITIES.length).toBe(161); + it('ENT-003: exactly 162 entities, every class name unique', () => { + expect(ALL_ENTITIES.length).toBe(162); const names = ALL_ENTITIES.map((s) => s.meta.className); expect(new Set(names).size).toBe(names.length); }); diff --git a/server/tests/unit/db/entity-class-fields.test.ts b/server/tests/unit/db/entity-class-fields.test.ts index 4807240c5..4808431a1 100644 --- a/server/tests/unit/db/entity-class-fields.test.ts +++ b/server/tests/unit/db/entity-class-fields.test.ts @@ -57,7 +57,9 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; // - assistant_threads.trip_id: a thread may be about a journey instead, so it is nullable now. // + journey_book_revisions: id, journey_id, version, document (a revision is only ever written whole). // + trip_member_removals: id, trip_id, user_id (a removal note always names its trip and its person). -const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 575; +// + assistant_attachments: id, thread_id, user_id, trip_id, name, mime, size, sha256, storage_key, text_status +// (an attached file is only ever written whole, by the turn it was attached in). +const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 585; describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => { it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => { diff --git a/server/tests/unit/db/legacy-baseline.test.ts b/server/tests/unit/db/legacy-baseline.test.ts index 4bbe32e74..ac0acfbf3 100644 --- a/server/tests/unit/db/legacy-baseline.test.ts +++ b/server/tests/unit/db/legacy-baseline.test.ts @@ -97,6 +97,7 @@ const UNNUMBERED = [ 'Migration20200101046270_a_photo_knows_its_colours_and_calm_areas', 'Migration20200101046280_an_assistant_conversation_can_be_about_no_one_trip', 'Migration20200101046290_a_removed_member_cannot_rejoin_by_invite_link', + 'Migration20200101046300_an_attached_file_stays_with_its_conversation', ]; // The last step the positional runner ever had; planLegacyBaseline refuses a // schema_version past it. diff --git a/server/tests/unit/nest/assistant/assistant-attachments.service.test.ts b/server/tests/unit/nest/assistant/assistant-attachments.service.test.ts new file mode 100644 index 000000000..e52f6d78f --- /dev/null +++ b/server/tests/unit/nest/assistant/assistant-attachments.service.test.ts @@ -0,0 +1,172 @@ +/** + * The files users attach in the trip chat, kept with the conversation: the + * bytes go to storage before the turn writes anything, the text TREK reads + * stays in the row and never fails the turn, a thread's bytes go with it, and + * the nightly sweep removes the week-old ones and the bytes no row names. + */ +import { AssistantAttachments } from '../../../../src/db/entities/AssistantAttachments.entity'; +import { + ASSISTANT_ATTACHMENT_TTL_DAYS, + AssistantAttachmentsService, +} from '../../../../src/nest/assistant/assistant-attachments.service'; +import { LlmParseService } from '../../../../src/nest/llm-parse/llm-parse.service'; +import { StorageService } from '../../../../src/nest/storage/storage.service'; +import { getRepositoryToken } from '@mikro-orm/nestjs'; +import { Test } from '@nestjs/testing'; + +import { Readable } from 'node:stream'; +import { describe, expect, it, vi } from 'vitest'; + +const logMock = vi.hoisted(() => ({ logInfo: vi.fn(), logError: vi.fn(), logWarn: vi.fn(), logDebug: vi.fn() })); +vi.mock('../../../../src/nest/audit/audit-log.logger', () => logMock); + +const NOW = new Date('2026-10-24T03:45:00.000Z'); +const HOUR = 60 * 60_000; + +function file(name: string, content = 'Booking ABC123', mimetype = 'application/pdf'): Express.Multer.File { + const buffer = Buffer.from(content); + return { originalname: name, buffer, size: buffer.length, mimetype } as Express.Multer.File; +} + +interface Listed { + key: string; + mtimeMs: number; +} + +async function make(opts: { text?: string | Error; listed?: Listed[]; failDelete?: string; failPut?: boolean } = {}) { + const rows = { + insertAttachment: vi.fn(async () => undefined), + listForThread: vi.fn(async () => []), + listObjectsOfThread: vi.fn(async () => [{ id: 'a1', storage_key: 'assistant/a1.pdf' }]), + listCreatedBefore: vi.fn(async () => [ + { id: 'old1', storage_key: 'assistant/old1.pdf' }, + { id: 'old2', storage_key: 'assistant/old2.pdf' }, + ]), + deleteByIds: vi.fn(async (ids: string[]) => ids.length), + listKnownKeys: vi.fn(async (keys: string[]) => keys.filter((key) => key.startsWith('assistant/kept'))), + }; + const put = vi.fn(async (_category: string, _key: string, source: Readable) => { + if (opts.failPut) throw new Error('bucket is gone'); + for await (const _chunk of source) { + // drained, as a driver would + } + }); + const storage = { + put, + delete: vi.fn(async (_category: string, key: string) => { + if (key === opts.failDelete) throw new Error('permission denied'); + }), + list: vi.fn(async function* list() { + for (const entry of opts.listed ?? []) yield { key: entry.key, size: 1, mtimeMs: entry.mtimeMs }; + }), + }; + const llmParse = { + readText: vi.fn(async () => { + if (opts.text instanceof Error) throw opts.text; + return opts.text ?? ' Booking ABC123 '; + }), + }; + const moduleRef = await Test.createTestingModule({ + providers: [ + AssistantAttachmentsService, + { provide: getRepositoryToken(AssistantAttachments), useValue: rows }, + { provide: StorageService, useValue: storage }, + { provide: LlmParseService, useValue: llmParse }, + ], + }).compile(); + return { service: moduleRef.get(AssistantAttachmentsService), rows, storage, llmParse }; +} + +describe('AssistantAttachmentsService', () => { + it('ASTATT-001: keeps the bytes under the attachments folder and reads the text, before any row is written', async () => { + const t = await make(); + const kept = await t.service.prepare(file('AirBnB Buchung.pdf'), 'AirBnB Buchung.pdf'); + expect(kept).toMatchObject({ + name: 'AirBnB Buchung.pdf', + mime: 'application/pdf', + size: 14, + text: 'Booking ABC123', + textStatus: 'ok', + }); + expect(kept.id).toMatch(/^[0-9a-f-]{36}$/); + expect(kept.storageKey).toBe(`assistant/${kept.id}.pdf`); + expect(kept.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(t.storage.put).toHaveBeenCalledWith('files', kept.storageKey, expect.any(Readable), { + contentType: 'application/pdf', + }); + expect(t.rows.insertAttachment).not.toHaveBeenCalled(); + }); + + it('ASTATT-002: a storage that refuses the bytes refuses the turn', async () => { + const t = await make({ failPut: true }); + await expect(t.service.prepare(file('a.pdf'), 'a.pdf')).rejects.toThrow('bucket is gone'); + }); + + it('ASTATT-003: says how reading the text went, and a failure never fails the file', async () => { + const statusOf = async (name: string, text?: string | Error) => { + const t = await make({ text }); + return (await t.service.prepare(file(name), name)).textStatus; + }; + expect(await statusOf('scan.pdf', ' ')).toBe('scanned'); + expect(await statusOf('empty.txt', '')).toBe('empty'); + expect(await statusOf('broken.pdf', new Error('bad xref'))).toBe('failed'); + expect(await statusOf('photo.JPG')).toBe('unsupported'); + expect(await statusOf('pass.pkpass')).toBe('unsupported'); + expect(logMock.logError).toHaveBeenCalledWith(expect.stringContaining('bad xref')); + }); + + it('ASTATT-004: a name with an odd extension gets a plain key, and a long text is kept to its cap', async () => { + const t = await make({ text: 'x'.repeat(250_000) }); + const kept = await t.service.prepare(file('ticket.p d f'), 'ticket.p d f'); + expect(kept.storageKey).toBe(`assistant/${kept.id}`); + expect(kept.text).toHaveLength(200_000); + }); + + it('ASTATT-005: records the row in the thread the turn opened, and drops the bytes of a turn that never began', async () => { + const t = await make(); + const kept = await t.service.prepare(file('a.pdf'), 'a.pdf'); + await t.service.record(kept, { threadId: 4, userId: 7, tripId: 9 }); + expect(t.rows.insertAttachment).toHaveBeenCalledWith( + expect.objectContaining({ id: kept.id, threadId: 4, userId: 7, tripId: 9, storageKey: kept.storageKey }), + ); + await t.service.discard(kept); + expect(t.storage.delete).toHaveBeenCalledWith('files', kept.storageKey); + }); + + it('ASTATT-006: a deleted thread takes its bytes along; one that cannot go now is left for the sweep', async () => { + const t = await make({ failDelete: 'assistant/a1.pdf' }); + const objects = await t.service.objectsOfThread(4, 7); + expect(t.rows.listObjectsOfThread).toHaveBeenCalledWith(4, 7); + await expect(t.service.dropObjects(objects)).resolves.toBeUndefined(); + expect(logMock.logError).toHaveBeenCalledWith(expect.stringContaining('assistant/a1.pdf')); + }); + + it(`ASTATT-007: the sweep removes what is ${ASSISTANT_ATTACHMENT_TTL_DAYS} days old, bytes first, then loose bytes`, async () => { + const t = await make({ + failDelete: 'assistant/old2.pdf', + listed: [ + { key: 'assistant/kept.pdf', mtimeMs: NOW.getTime() - 3 * HOUR }, + { key: 'assistant/loose.pdf', mtimeMs: NOW.getTime() - 3 * HOUR }, + { key: 'assistant/fresh.pdf', mtimeMs: NOW.getTime() - 60_000 }, + ], + }); + const swept = await t.service.sweep(NOW); + expect(t.rows.listCreatedBefore).toHaveBeenCalledWith('2026-10-17 03:45:00', 500); + // The row whose bytes could not be deleted stays, so it can be tried again. + expect(t.rows.deleteByIds).toHaveBeenCalledWith(['old1']); + expect(t.rows.listKnownKeys).toHaveBeenCalledWith(['assistant/kept.pdf', 'assistant/loose.pdf']); + expect(t.storage.delete).toHaveBeenCalledWith('files', 'assistant/loose.pdf'); + expect(t.storage.delete).not.toHaveBeenCalledWith('files', 'assistant/kept.pdf'); + expect(t.storage.delete).not.toHaveBeenCalledWith('files', 'assistant/fresh.pdf'); + expect(swept).toEqual({ expired: 1, orphaned: 1 }); + expect(logMock.logInfo).toHaveBeenCalledWith( + 'Assistant retention: removed 1 attached file(s) and 1 loose object(s)', + ); + }); + + it('ASTATT-008: lists a thread for the user only', async () => { + const t = await make(); + await t.service.listForThread(4, 7); + expect(t.rows.listForThread).toHaveBeenCalledWith(4, 7); + }); +}); diff --git a/server/tests/unit/nest/assistant/assistant-retention.job.test.ts b/server/tests/unit/nest/assistant/assistant-retention.job.test.ts index 35d7b67d2..eca2a2944 100644 --- a/server/tests/unit/nest/assistant/assistant-retention.job.test.ts +++ b/server/tests/unit/nest/assistant/assistant-retention.job.test.ts @@ -2,6 +2,7 @@ import type { AssistantThreadsRepository } from '../../../../src/db/repositories import type { AssistantToolCallsRepository } from '../../../../src/db/repositories/AssistantToolCalls.repository'; import type { AiConnectionsService } from '../../../../src/nest/ai-connections/ai-connections.service'; import type { AiUsageService } from '../../../../src/nest/ai-connections/ai-usage.service'; +import type { AssistantAttachmentsService } from '../../../../src/nest/assistant/assistant-attachments.service'; import { AssistantRetentionJob, ASSISTANT_USAGE_RETENTION_DAYS, @@ -20,14 +21,16 @@ function make(retention: AiRetention, enabled = true) { const connections = { readSettings: vi.fn().mockResolvedValue({ policy: 'user', dailyTurns: 50, retention }) }; const usage = { purgeOlderThan: vi.fn().mockResolvedValue(3) }; const registrar = { isEnabled: vi.fn(() => enabled), register: vi.fn() }; + const attachments = { sweep: vi.fn().mockResolvedValue({ expired: 0, orphaned: 0 }) }; const job = new AssistantRetentionJob( threads as unknown as AssistantThreadsRepository, toolCalls as unknown as AssistantToolCallsRepository, connections as unknown as AiConnectionsService, usage as unknown as AiUsageService, registrar as unknown as CronRegistrarService, + attachments as unknown as AssistantAttachmentsService, ); - return { job, threads, toolCalls, usage, registrar }; + return { job, threads, toolCalls, usage, registrar, attachments }; } const NOW = new Date('2026-10-21T03:45:00.000Z'); @@ -49,6 +52,7 @@ describe('AssistantRetentionJob', () => { expect(t.threads.deleteWithoutAccess).toHaveBeenCalled(); expect(t.toolCalls.expireBefore).toHaveBeenCalledWith('2026-10-20 03:45:00'); expect(t.usage.purgeOlderThan).toHaveBeenCalledWith(ASSISTANT_USAGE_RETENTION_DAYS); + expect(t.attachments.sweep).toHaveBeenCalledWith(NOW); expect(logMock.logInfo).toHaveBeenCalledWith('Assistant retention: removed 3 conversation(s)'); expect(logMock.logInfo).toHaveBeenCalledWith('Assistant retention: 1 proposed change(s) expired'); }); diff --git a/shared/src/assistant/assistant-attachments.schema.ts b/shared/src/assistant/assistant-attachments.schema.ts new file mode 100644 index 000000000..954a677c7 --- /dev/null +++ b/shared/src/assistant/assistant-attachments.schema.ts @@ -0,0 +1,12 @@ +import { z } from 'zod'; + +/** + * A file the user attached in the trip chat, as the conversation knows it. + * + * The server keeps the file with the thread under a random id, which the + * message's `attachment` part carries. The id never reaches anything but the + * thread it was attached in: every read answers an id of another thread or + * another user the way it answers one that does not exist. + */ +export const assistantAttachmentIdSchema = z.uuid(); +export type AssistantAttachmentId = z.infer; diff --git a/shared/src/assistant/assistant.schema.spec.ts b/shared/src/assistant/assistant.schema.spec.ts index 481ba4d80..ebcb28a0d 100644 --- a/shared/src/assistant/assistant.schema.spec.ts +++ b/shared/src/assistant/assistant.schema.spec.ts @@ -180,6 +180,15 @@ describe('assistantMessageSchema', () => { const negative = { ...attachment, size: -1 }; expect(assistantMessageSchema.safeParse({ ...message, parts: [negative] }).success).toBe(false); }); + + it('reads the id a kept attachment goes by, and only a uuid as one', () => { + const message = { id: '14', role: 'user', status: 'complete', createdAt: '2026-10-21T08:00:00.000Z' }; + const attachment = { type: 'attachment', name: 'ticket.pdf', mime: 'application/pdf', size: 2048 }; + const kept = { ...attachment, attachmentId: '0b7c2f0e-6f6d-4a8e-9b1a-3d2c1e0f9a87' }; + expect(assistantMessageSchema.parse({ ...message, parts: [kept] }).parts[0]).toEqual(kept); + const forged = { ...attachment, attachmentId: '../other-thread' }; + expect(assistantMessageSchema.safeParse({ ...message, parts: [forged] }).success).toBe(false); + }); }); describe('uiStreamChunkSchema', () => { diff --git a/shared/src/assistant/assistant.schema.ts b/shared/src/assistant/assistant.schema.ts index eec7af102..7391149c8 100644 --- a/shared/src/assistant/assistant.schema.ts +++ b/shared/src/assistant/assistant.schema.ts @@ -1,5 +1,6 @@ import { bookElementIdSchema, bookSpreadIdSchema } from '../book/book-ops.schema'; import { idParamSchema, idSchema } from '../common/primitives.schema'; +import { assistantAttachmentIdSchema } from './assistant-attachments.schema'; import { assistantPageRenderSchema, assistantPageViewSchema } from './assistant-vision.schema'; import { assistantCardSchema, @@ -648,12 +649,14 @@ function messagePartSchema< >(change: Change, applied: Applied, card: Card) { return z.discriminatedUnion('type', [ z.object({ type: z.literal('text'), text: z.string() }), - // A file the user handed the assistant. Only its description is kept, never its content. + // A file the user handed the assistant. Only its description is kept here, never its content; the + // server keeps the file under `attachmentId` (left out on messages from before it kept files). z.object({ type: z.literal('attachment'), name: z.string(), mime: z.string(), size: z.number().int().nonnegative(), + attachmentId: assistantAttachmentIdSchema.optional(), }), z.object({ type: z.literal('tool-call'), diff --git a/shared/src/index.ts b/shared/src/index.ts index 73e0340d3..d450cedd9 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -232,6 +232,7 @@ export * from './oauth/oauth.schema'; export * from './admin/admin.schema'; export * from './ai/ai-connections.schema'; export * from './assistant/assistant.schema'; +export * from './assistant/assistant-attachments.schema'; export * from './assistant/cards'; export * from './assistant/assistant-share.schema'; export * from './assistant/assistant-scope.schema';