Merge pull request #2391 from liketrek/feat/document-providers

feat(documents): two-way sync with Paperless, Papra, Nextcloud, OpenCloud and Synology
This commit is contained in:
Maurice
2026-09-19 19:57:16 +02:00
committed by GitHub
226 changed files with 36514 additions and 460 deletions
+1
View File
@@ -42,6 +42,7 @@ export const MCP_GATED_ADDON_IDS: readonly AddonId[] = [
ADDON_IDS.COLLAB,
ADDON_IDS.COLLECTIONS,
ADDON_IDS.DAWARICH,
ADDON_IDS.DOCUMENTS,
ADDON_IDS.JOURNEY,
ADDON_IDS.PACKING,
ADDON_IDS.ROADTRIP,
+137
View File
@@ -0,0 +1,137 @@
import type Database from 'better-sqlite3';
/**
* The five document providers and the fields their connection form asks for.
*
* Shared by the migration that introduces the tables and by `seeds.ts`, so a
* fresh install and an upgraded one end up with the same rows. Both call sites
* use INSERT OR IGNORE, which is what lets an operator rename or re-sort a row
* without a later boot undoing it.
*
* `label` and `hint` are i18n key suffixes, never display text: the client
* resolves them as `docsync.<label>`, the same way the photo provider form
* resolves `memories.<label>`.
*
* Nextcloud and OpenCloud are separate rows although one WebDAV adapter serves
* both: the fields differ (a Nextcloud login name and app password against a
* folder, an OpenCloud username and app token against a space), and an operator
* scanning the addon shelf should find the product they actually run.
*/
interface ProviderRow {
id: string;
name: string;
description: string;
icon: string;
sort_order: number;
}
interface ProviderFieldRow {
provider_id: string;
field_key: string;
label: string;
input_type: 'text' | 'url' | 'password' | 'checkbox';
placeholder: string | null;
hint: string | null;
required: 0 | 1;
secret: 0 | 1;
sort_order: number;
}
const PROVIDERS: ProviderRow[] = [
{
id: 'paperless',
name: 'Paperless-ngx',
description: 'Two-way document sync with a Paperless-ngx instance, scoped by tag',
icon: 'FileText',
sort_order: 0,
},
{
id: 'papra',
name: 'Papra',
description: 'Two-way document sync with a Papra organisation, scoped by tag',
icon: 'FileText',
sort_order: 1,
},
{
id: 'nextcloud',
name: 'Nextcloud',
description: 'Two-way document sync with a Nextcloud folder over WebDAV',
icon: 'Cloud',
sort_order: 2,
},
{
id: 'opencloud',
name: 'OpenCloud',
description: 'Two-way document sync with an OpenCloud space over WebDAV',
icon: 'Cloud',
sort_order: 3,
},
{
id: 'synologydrive',
name: 'Synology Drive',
description: 'Two-way document sync with a folder on a Synology NAS',
icon: 'HardDrive',
sort_order: 4,
},
];
/**
* `allow_insecure_tls` defaults to 0 on every provider, unlike the Synology
* photo provider, where skipping verification is the stored default. A LAN
* instance with a self-signed certificate is common enough to need the switch
* and not common enough to justify shipping it on.
*/
const FIELDS: ProviderFieldRow[] = [
// Paperless-ngx: token auth, tag scope.
{ provider_id: 'paperless', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://paperless.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
{ provider_id: 'paperless', field_key: 'api_token', label: 'providerApiToken', input_type: 'password', placeholder: 'API token', hint: 'hintPaperlessToken', required: 1, secret: 1, sort_order: 1 },
{ provider_id: 'paperless', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 2 },
// Papra: bearer key, and an organisation the key must already belong to.
{ provider_id: 'papra', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://papra.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
{ provider_id: 'papra', field_key: 'api_key', label: 'providerApiKey', input_type: 'password', placeholder: 'ppapi_…', hint: 'hintPapraKey', required: 1, secret: 1, sort_order: 1 },
{ provider_id: 'papra', field_key: 'organization_id', label: 'providerOrganization', input_type: 'text', placeholder: 'org_…', hint: 'hintPapraOrg', required: 1, secret: 0, sort_order: 2 },
{ provider_id: 'papra', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 3 },
// Nextcloud: an app password, never the account password. The latter breaks
// the moment the account turns on 2FA or logs in through OIDC.
{ provider_id: 'nextcloud', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://cloud.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
{ provider_id: 'nextcloud', field_key: 'login_name', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: 'hintNextcloudLogin', required: 1, secret: 0, sort_order: 1 },
{ provider_id: 'nextcloud', field_key: 'app_password', label: 'providerAppPassword', input_type: 'password', placeholder: 'app password', hint: 'hintNextcloudAppPassword', required: 1, secret: 1, sort_order: 2 },
{ provider_id: 'nextcloud', field_key: 'base_path', label: 'providerBasePath', input_type: 'text', placeholder: '/TREK', hint: 'hintBasePath', required: 0, secret: 0, sort_order: 3 },
{ provider_id: 'nextcloud', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 4 },
// OpenCloud: auth-app token, spaces instead of folders.
{ provider_id: 'opencloud', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://opencloud.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
{ provider_id: 'opencloud', field_key: 'username', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: null, required: 1, secret: 0, sort_order: 1 },
{ provider_id: 'opencloud', field_key: 'app_token', label: 'providerAppToken', input_type: 'password', placeholder: 'app token', hint: 'hintOpenCloudToken', required: 1, secret: 1, sort_order: 2 },
{ provider_id: 'opencloud', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 3 },
// Synology: DSM has no app tokens at all, so this is a real account. The OTP
// is used once to obtain a device token, which is what gets stored.
{ provider_id: 'synologydrive', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://nas.example.com:5001', hint: 'hintSynologyUrl', required: 1, secret: 0, sort_order: 0 },
{ provider_id: 'synologydrive', field_key: 'username', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: 'hintSynologyUser', required: 1, secret: 0, sort_order: 1 },
{ provider_id: 'synologydrive', field_key: 'password', label: 'providerPassword', input_type: 'password', placeholder: 'password', hint: null, required: 1, secret: 1, sort_order: 2 },
{ provider_id: 'synologydrive', field_key: 'otp_code', label: 'providerOTP', input_type: 'text', placeholder: '123456', hint: 'hintSynologyOtp', required: 0, secret: 1, sort_order: 3 },
{ provider_id: 'synologydrive', field_key: 'base_path', label: 'providerBasePath', input_type: 'text', placeholder: '/trek', hint: 'hintBasePath', required: 0, secret: 0, sort_order: 4 },
{ provider_id: 'synologydrive', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 5 },
];
export function seedDocumentProviders(db: Database.Database): void {
const insertProvider = db.prepare(
'INSERT OR IGNORE INTO document_providers (id, name, description, icon, enabled, sort_order) VALUES (?, ?, ?, ?, 0, ?)',
);
for (const p of PROVIDERS) insertProvider.run(p.id, p.name, p.description, p.icon, p.sort_order);
const insertField = db.prepare(
`INSERT OR IGNORE INTO document_provider_fields
(provider_id, field_key, label, input_type, placeholder, hint, required, secret, sort_order)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
);
for (const f of FIELDS) {
insertField.run(f.provider_id, f.field_key, f.label, f.input_type, f.placeholder, f.hint, f.required, f.secret, f.sort_order);
}
}
export const DOCUMENT_PROVIDER_SEED_IDS = PROVIDERS.map((p) => p.id);
+197
View File
@@ -1,6 +1,7 @@
import { ROADTRIP_PREFERENCE_KEYS } from '@trek/shared';
import { readEnv } from '../app-config';
import { encrypt_api_key } from '../nest/common/crypto/apiKeyCrypto';
import { seedDocumentProviders } from './document-provider-seed';
import Database from 'better-sqlite3';
import fs from 'fs';
@@ -4949,6 +4950,202 @@ function runMigrations(db: Database.Database): void {
'CREATE INDEX IF NOT EXISTS idx_journey_entries_source_assignment ON journey_entries(source_place_id, source_assignment_id)',
);
},
/*
* Document providers, part 1 of 3: the registry (#214).
*
* Deliberately its own pair of tables rather than a `kind` column on
* `photo_providers`. The client filters on `type === 'photo_provider'`, the
* Journey cascade runs `UPDATE photo_providers SET enabled = 0` with no
* WHERE clause, and migrations are append-only. Reusing those tables would
* change the behaviour of three existing paths, which is exactly what the
* no-breaking-changes rule forbids. The field columns follow
* `photo_provider_fields` except for `settings_key` and `payload_key`.
* Those map a photo field onto a settings key and a request key; a document
* field is stored under its own `field_key` in one JSON column, and the
* connect form takes its fields from /docsync/providers rather than from
* the generic settings form, so nothing would ever read them.
*/
() => {
db.exec(`
CREATE TABLE IF NOT EXISTS document_providers (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
icon TEXT DEFAULT 'FileText',
enabled INTEGER DEFAULT 0,
sort_order INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS document_provider_fields (
id INTEGER PRIMARY KEY AUTOINCREMENT,
provider_id TEXT NOT NULL REFERENCES document_providers(id) ON DELETE CASCADE,
field_key TEXT NOT NULL,
label TEXT NOT NULL,
input_type TEXT NOT NULL DEFAULT 'text',
placeholder TEXT,
hint TEXT,
required INTEGER DEFAULT 0,
secret INTEGER DEFAULT 0,
sort_order INTEGER DEFAULT 0,
UNIQUE(provider_id, field_key)
);
`);
seedDocumentProviders(db);
},
/*
* Document providers, part 2 of 3: the connection, and the trip binding.
*
* The connection carries a `trip_id`, and that is the one place this design
* departs from every integration already in the repo. Immich, Synology
* Photos, AirTrail and Dawarich all hang off a user, and even
* `trip_album_links` carries a `user_id`; photos become visible to the rest
* of a trip only through an opt-in `shared` flag. None of that can satisfy
* "everyone on the trip sees the same documents": it would make a
* document's visibility depend on whose credentials fetched it. So the trip
* admin binds the trip once, the server talks to the provider under that
* single identity, and TREK's own membership decides who sees what.
*
* `owner_user_id` stays separate from `trip_id` so the credential holder is
* always explicit: when that person leaves the trip the binding goes to
* `orphaned` rather than silently continuing to use an ex-member's token.
*
* Secrets live in one encrypted JSON blob instead of per-provider columns:
* a sixth provider then needs no migration, and the key rotation in
* scripts/migrate-encryption.ts stays one line instead of a field list that
* someone will forget, and a forgotten column does not survive a rotation.
*/
() => {
db.exec(`
CREATE TABLE IF NOT EXISTS document_connections (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
provider_id TEXT NOT NULL REFERENCES document_providers(id) ON DELETE CASCADE,
owner_user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
base_url TEXT NOT NULL,
secrets TEXT,
settings TEXT NOT NULL DEFAULT '{}',
allow_insecure_tls INTEGER NOT NULL DEFAULT 0,
capabilities TEXT,
last_probe_at TEXT,
last_probe_state TEXT NOT NULL DEFAULT 'never',
last_probe_error TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(trip_id, provider_id)
);
CREATE INDEX IF NOT EXISTS idx_document_connections_trip ON document_connections(trip_id);
CREATE INDEX IF NOT EXISTS idx_document_connections_owner ON document_connections(owner_user_id);
CREATE TABLE IF NOT EXISTS trip_document_links (
id INTEGER PRIMARY KEY AUTOINCREMENT,
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
connection_id INTEGER NOT NULL REFERENCES document_connections(id) ON DELETE CASCADE,
provider_id TEXT NOT NULL,
remote_scope_key TEXT NOT NULL,
remote_root_id TEXT,
remote_root_path TEXT,
remote_label TEXT NOT NULL DEFAULT '',
direction TEXT NOT NULL DEFAULT 'both',
delete_policy TEXT NOT NULL DEFAULT 'unlink',
conflict_policy TEXT NOT NULL DEFAULT 'manual',
sync_enabled INTEGER NOT NULL DEFAULT 1,
webhook_token TEXT,
webhook_secret TEXT,
webhook_subscription_id TEXT,
remote_cursor TEXT,
last_sync_at TEXT,
last_sync_state TEXT NOT NULL DEFAULT 'never',
last_sync_error TEXT,
failure_count INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
created_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE(trip_id, connection_id, remote_scope_key)
);
CREATE INDEX IF NOT EXISTS idx_trip_document_links_trip ON trip_document_links(trip_id);
CREATE INDEX IF NOT EXISTS idx_trip_document_links_due ON trip_document_links(sync_enabled, next_attempt_at);
CREATE UNIQUE INDEX IF NOT EXISTS idx_trip_document_links_token
ON trip_document_links(webhook_token) WHERE webhook_token IS NOT NULL;
`);
},
/*
* Document providers, part 3 of 3: the pairing and its sync state.
*
* `content_sha256` and `pushed_sha256` are two columns on purpose. The
* first is the bytes both sides last agreed on, the second is what TREK
* itself last uploaded. Collapsing them into one is precisely the mistake
* that builds an echo loop: a webhook fires for TREK's own write, the core
* cannot tell it from a stranger's edit, and the file bounces.
*
* `remote_missing_at` records that something vanished upstream instead of
* acting on it, the rule Dawarich already follows with `source_missing_at`.
* An unmounted share answers with an empty listing, and reading that as
* "everything was deleted" would empty a trip.
*
* `file_id ON DELETE SET NULL` keeps a tombstone behind after a document is
* permanently deleted in TREK, so the next run does not cheerfully download
* it again.
*/
() => {
db.exec(`
CREATE TABLE IF NOT EXISTS document_sync_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
link_id INTEGER NOT NULL REFERENCES trip_document_links(id) ON DELETE CASCADE,
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
file_id INTEGER REFERENCES trip_files(id) ON DELETE SET NULL,
trek_doc_uid TEXT NOT NULL,
remote_id TEXT,
remote_name TEXT,
remote_version TEXT,
remote_size INTEGER,
remote_modified_at TEXT,
content_sha256 TEXT,
pushed_sha256 TEXT,
state TEXT NOT NULL DEFAULT 'pending',
error_code TEXT,
attempts INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
remote_missing_at TEXT,
first_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
synced_at TEXT
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_remote
ON document_sync_items(link_id, remote_id) WHERE remote_id IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_file
ON document_sync_items(link_id, file_id) WHERE file_id IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_uid
ON document_sync_items(link_id, trek_doc_uid);
CREATE INDEX IF NOT EXISTS idx_document_sync_items_trip_state ON document_sync_items(trip_id, state);
CREATE INDEX IF NOT EXISTS idx_document_sync_items_hash ON document_sync_items(link_id, content_sha256);
CREATE INDEX IF NOT EXISTS idx_document_sync_items_due ON document_sync_items(link_id, next_attempt_at);
`);
},
/*
* Document providers: when TREK itself put a provider copy in the bin.
*
* A file deleted in TREK under the `trash` policy takes its provider copy
* with it. Taken back out of TREK's trash, it has to go up again; a copy
* somebody else deleted in the meantime must not. Both leave the same gap
* in a listing, so the difference is written down when TREK acts rather
* than guessed at later.
*
* No backfill from the binding's current policy: that policy may not be
* the one the deletion ran under, and reading it back is the retroactive
* mistake this column exists to avoid. A row left NULL is treated like a
* copy somebody else deleted, which flags it instead of uploading it.
*/
() => {
const hasColumn = db
.prepare("SELECT 1 FROM pragma_table_info('document_sync_items') WHERE name = 'remote_trashed_at'")
.get();
if (!hasColumn) db.exec('ALTER TABLE document_sync_items ADD COLUMN remote_trashed_at TEXT');
},
];
if (currentVersion < migrations.length) {
+8
View File
@@ -1,6 +1,7 @@
import Database from 'better-sqlite3';
import crypto from 'crypto';
import { readEnv } from '../app-config';
import { seedDocumentProviders } from './document-provider-seed';
// bcrypt cost factor for the seeded admin password — kept in sync with authService.
const BCRYPT_COST = 12;
@@ -164,6 +165,13 @@ function seedAddons(db: Database.Database): void {
for (const f of providerFields) {
insertProviderField.run(f.provider_id, f.field_key, f.label, f.input_type, f.placeholder, f.hint, f.required, f.secret, f.settings_key, f.payload_key, f.sort_order);
}
// Document providers live in their own pair of tables (see the migration
// for why they are not a `kind` column on photo_providers). Seeded from the
// same helper the migration uses, so a fresh install and an upgraded one
// agree.
seedDocumentProviders(db);
console.log('Default addons seeded');
} catch (err: unknown) {
console.error('Error seeding addons:', err instanceof Error ? err.message : err);
+50 -2
View File
@@ -598,6 +598,28 @@ export class AdminService {
fieldsByProvider.set(field.provider_id, arr);
}
// Document providers are the Documents addon's shelf rows, the same way
// photo providers are Journey's. Hidden on a managed instance for the same
// reason: Paperless, Papra, Nextcloud, OpenCloud and a Synology NAS are all
// servers the admin runs at home, and a hosted TREK has no route to one.
//
// They carry no `config` routes, unlike photo providers: the credentials do
// not belong to a user here but to a trip, so they are entered in the trip
// rather than in settings. The admin decides only whether a provider may be
// offered at all.
const docProviders = this.db.all<{
id: string;
name: string;
description?: string | null;
icon: string;
enabled: number;
sort_order: number;
}>(`
SELECT id, name, description, icon, enabled, sort_order
FROM document_providers
ORDER BY sort_order, id
`).filter(() => !readEnv().managed.enabled);
return [
...addons.map((a) => ({
...a,
@@ -628,6 +650,17 @@ export class AdminService {
})),
sort_order: p.sort_order,
})),
...docProviders.map((p) => ({
id: p.id,
name: p.name,
description: p.description,
type: 'document_provider',
icon: p.icon,
enabled: !!p.enabled,
config: {},
fields: [],
sort_order: p.sort_order,
})),
];
}
@@ -635,7 +668,8 @@ export class AdminService {
type ProviderRow = { id: string; name: string; description?: string | null; icon: string; enabled: number; sort_order: number };
const addon = this.db.get<Addon>('SELECT * FROM addons WHERE id = ?', id);
const provider = this.db.get<ProviderRow>('SELECT * FROM photo_providers WHERE id = ?', id);
if (!addon && !provider) return { error: 'Addon not found', status: 404 };
const docProvider = this.db.get<ProviderRow>('SELECT * FROM document_providers WHERE id = ?', id);
if (!addon && !provider && !docProvider) return { error: 'Addon not found', status: 404 };
// The whole addon, not just its config: on a centrally administered install
// the operator owns the endpoint, the model and the per-document cost, so
@@ -652,6 +686,13 @@ export class AdminService {
return { error: 'Enable the Journey addon first', status: 409 };
}
// Same rule one shelf down: a document provider only exists to serve the
// file manager, so switching one on under a disabled Documents addon would
// advertise a sync nothing can reach.
if (docProvider && data.enabled === true && !this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS)) {
return { error: 'Enable the Documents addon first', status: 409 };
}
this.db.transaction(() => {
if (addon) {
if (data.enabled !== undefined) {
@@ -660,6 +701,10 @@ export class AdminService {
// resurface the moment journey returns, which nobody switched on.
if (id === ADDON_IDS.JOURNEY && !data.enabled)
this.db.run('UPDATE photo_providers SET enabled = 0');
// Documents off takes its providers with it, for the reason above: a
// row left enabled would resurface the moment the addon returns.
if (id === ADDON_IDS.DOCUMENTS && !data.enabled)
this.db.run('UPDATE document_providers SET enabled = 0');
}
if (data.config !== undefined) {
// The AI-parsing addon holds an API key — encrypt it at rest and preserve
@@ -670,9 +715,12 @@ export class AdminService {
: data.config;
this.db.run('UPDATE addons SET config = ? WHERE id = ?', JSON.stringify(configToStore), id);
}
} else {
} else if (provider) {
if (data.enabled !== undefined)
this.db.run('UPDATE photo_providers SET enabled = ? WHERE id = ?', data.enabled ? 1 : 0, id);
} else {
if (data.enabled !== undefined)
this.db.run('UPDATE document_providers SET enabled = ? WHERE id = ?', data.enabled ? 1 : 0, id);
}
});
+2 -1
View File
@@ -45,6 +45,7 @@ import { PhotosModule } from './photos/photos.module';
import { MemoriesModule } from './memories/memories.module';
import { AirtrailModule } from './integrations/airtrail.module';
import { DawarichModule } from './integrations/dawarich.module';
import { DocSyncModule } from './doc-sync/doc-sync.module';
import { JourneyModule } from './journey/journey.module';
import { CollectionsModule } from './collections/collections.module';
import { ShareModule } from './share/share.module';
@@ -81,7 +82,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
* migrated.
*/
@Module({
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
providers: [
// Default-deny: a route is authenticated unless it carries @Public() or
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
@@ -167,8 +167,17 @@ export class AssignmentOpsController {
if (!this.assignments.getAssignmentForTrip(id, tripId)) {
throw new HttpException({ error: 'Assignment not found' }, 404);
}
const assignment = this.assignments.updateTime(id, body.place_time, body.end_time);
const { assignment, reordered, vias } = this.assignments.updateTime(id, body.place_time, body.end_time);
this.assignments.broadcast(tripId, 'assignment:updated', { assignment }, socketId);
// The whole day when a start moved stops, or collaborators apply the one row
// they were sent to their old order and end up with a third one.
//
// Both to every socket, the writer's included, so the order and the vias pinned
// to it arrive together. The planner holds its vias in memory and would route the
// new anchors on the old order until its reload of the day came back, and a save
// replayed from the offline queue has no reload after it at all.
if (reordered) this.assignments.broadcast(tripId, 'assignment:reordered', reordered, undefined);
if (vias) this.assignments.broadcast(tripId, 'roadtripVia:changed', vias, undefined);
this.assignments.reconcile(tripId, socketId);
return { assignment };
}
@@ -130,12 +130,15 @@ export class AssignmentsMcp {
if (!this.guards.hasTripPermission('day_edit', tripId, ctx.userId)) return permissionDenied();
const existing = this.assignments.getAssignmentForTrip(assignmentId, tripId);
if (!existing) return errorResult('Assignment not found.');
const assignment = this.assignments.updateTime(
const { assignment, reordered, vias } = this.assignments.updateTime(
assignmentId,
place_time !== undefined ? place_time : existing.assignment_time,
end_time !== undefined ? end_time : existing.assignment_end_time
);
// Same three events as PUT /assignments/:id/time.
this.guards.safeBroadcast(tripId, 'assignment:updated', { assignment });
if (reordered) this.guards.safeBroadcast(tripId, 'assignment:reordered', reordered);
if (vias) this.guards.safeBroadcast(tripId, 'roadtripVia:changed', vias);
this.assignments.reconcile(tripId);
return ok({ assignment });
}
@@ -1,5 +1,6 @@
import { Injectable } from '@nestjs/common';
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
import { chronoOrder, type RoadtripVia, type TrekWsPayload, type TrekWsTripEventName } from '@trek/shared';
import { isEmptyReanchoring, reanchorByStopOrder, type AnchoredVia } from '@trek/shared/roadtrip';
import { RealtimeService } from '../realtime/realtime.service';
import { DatabaseService, type TripAccess } from '../database/database.service';
import { PermissionsService } from '../permissions/permissions.service';
@@ -10,6 +11,39 @@ import { JourneyDomainService } from '../journey/journey-domain.service';
type Trip = TripAccess;
/** One stop of a day as the time sort reads it. */
interface DayStopRow {
id: number;
order_index: number;
effective_time: string | null;
located: number;
}
/**
* What saving a time changed besides the stop itself, so each caller can tell the
* trip. `reordered` carries the day's whole order and `vias` the day's re-pinned
* vias; both stay null when the save left every stop where it was.
*/
export interface AssignmentTimeUpdate {
assignment: ReturnType<AssignmentsService['getAssignmentWithPlace']>;
reordered: { dayId: number; orderedIds: number[] } | null;
vias: { dayId: number; vias: RoadtripVia[] } | null;
}
/**
* Where the time sort puts a value it cannot read as a clock time: after every real
* time, which is where the '99:99' sentinel has always put a time without a colon. A
* legacy "morning" keeps sorting where it did. The client reads such a value as no
* time at all; that difference is older than this rule and left alone.
*/
const UNREADABLE_TIME = 99 * 60 + 99;
function sortMinutes(time: string | null): number | null {
if (!time) return null;
const clock = /(?:^|T)(\d{1,2}):(\d{2})/.exec(time);
return clock ? Number(clock[1]) * 60 + Number(clock[2]) : UNREADABLE_TIME;
}
/**
* Assignments domain service — owns the day-assignment SQL (relocated from the
* legacy services/assignmentService.ts, then hardened: every multi-statement
@@ -206,48 +240,136 @@ export class AssignmentsService {
`, assignmentId);
}
updateTime(id: string | number, placeTime: unknown, endTime: unknown) {
this.dbs.transaction(() => {
/**
* Saves a visit's own start and end, and puts the day back in time order when the
* start changed. The rule is the planner's (`chronoOrder`): an untimed stop stays
* behind the stop it followed. This used to append every untimed stop after the
* timed ones, so one start time pulled a stop planned last to the top of the day.
*
* The rule is shared, what it reads is not. This sorts the day's stops alone, with a
* booked night timed by its check-in. The planner sorts day notes and bookings in
* between them and never draws the night's row. An untimed stop behind a timed note
* or train takes that item's time there and the previous stop's time here, so on
* such a day the order stored and the order drawn can differ. The old sort did the
* same.
*/
updateTime(id: string | number, placeTime: unknown, endTime: unknown): AssignmentTimeUpdate {
const sorted = this.dbs.transaction(() => {
const stored = this.dbs.get<{ day_id: number; start: string | null }>(`
SELECT da.day_id, COALESCE(da.assignment_time, p.place_time, acc.check_in) AS start
FROM day_assignments da
JOIN places p ON da.place_id = p.id
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
WHERE da.id = ?
`, id);
// Falsy times (null, undefined, '') all clear the override — an empty
// string is a clear, not a stored value.
this.dbs.run('UPDATE day_assignments SET assignment_time = ?, assignment_end_time = ? WHERE id = ?',
placeTime || null, endTime || null, id);
// Auto-sort: reorder timed assignments chronologically within the day
if (placeTime) {
const assignment = this.dbs.get<{ day_id: number }>('SELECT day_id FROM day_assignments WHERE id = ?', id);
if (assignment) {
// A booked night's hour lives on the booking, not on the stop: nobody types a
// time into a hotel row, they type a check-in. Left out of this, the night
// counted as untimed and stayed wherever it had been dropped, so pinning an
// afternoon stop sorted that one and left the hotel sitting in front of or
// behind it by accident.
const dayAssignments = this.dbs.all<{ id: number; effective_time: string | null }>(`
SELECT da.id, COALESCE(da.assignment_time, p.place_time, acc.check_in) as effective_time
FROM day_assignments da
JOIN places p ON da.place_id = p.id
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
WHERE da.day_id = ?
ORDER BY da.order_index ASC
`, assignment.day_id);
// Separate timed and untimed, sort timed by time
const timed = dayAssignments.filter(a => a.effective_time).sort((a, b) => {
const ta = a.effective_time!.includes(':') ? a.effective_time! : '99:99';
const tb = b.effective_time!.includes(':') ? b.effective_time! : '99:99';
return ta.localeCompare(tb);
});
const untimed = dayAssignments.filter(a => !a.effective_time);
// Interleave: timed in chronological order, untimed keep relative position
const reordered = [...timed, ...untimed];
const update = this.dbs.prepare('UPDATE day_assignments SET order_index = ? WHERE id = ?');
reordered.forEach((a, i) => update.run(i, a.id));
}
}
// Only a start that moved sorts. An end is a label. A start sent again as it
// stood (the place form saving an End, the stay dialog taking one off, an MCP
// call that names only the end) leaves the day the way the traveller left it,
// which can be out of time order on purpose. Compared the way the sort reads
// it, so a visit given the time its place already had moves nothing either. A
// cleared start leaves the day alone too.
if (!placeTime || !stored) return null;
if (sortMinutes(String(placeTime)) === sortMinutes(stored.start)) return null;
return this.sortDayByTime(stored.day_id);
});
return this.getAssignmentWithPlace(Number(id));
return {
assignment: this.getAssignmentWithPlace(Number(id)),
reordered: sorted ? { dayId: sorted.dayId, orderedIds: sorted.orderedIds } : null,
vias: sorted?.viasMoved ? { dayId: sorted.dayId, vias: this.listDayVias(sorted.dayId) } : null,
};
}
/**
* Puts one day in time order. Writes nothing when it already is, which is the usual
* case: most starts are typed in the order the day is planned.
*/
private sortDayByTime(dayId: number): { dayId: number; orderedIds: number[]; viasMoved: boolean } | null {
// A booked night's hour lives on the booking, not on the stop: nobody types a
// time into a hotel row, they type a check-in. Left out of this, the night
// counted as untimed and stayed wherever it had been dropped, so pinning an
// afternoon stop sorted that one and left the hotel sitting in front of or
// behind it by accident.
const rows = this.dbs.all<DayStopRow>(`
SELECT da.id, da.order_index, COALESCE(da.assignment_time, p.place_time, acc.check_in) as effective_time,
(p.lat IS NOT NULL AND p.lng IS NOT NULL) as located
FROM day_assignments da
JOIN places p ON da.place_id = p.id
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
WHERE da.day_id = ?
ORDER BY da.order_index ASC, da.created_at ASC, da.id ASC
`, dayId);
const sorted = chronoOrder(rows, row => sortMinutes(row.effective_time));
if (sorted.every((row, i) => row === rows[i])) return null;
// Numbered from 0, the way a drag stores a day (`reorderAssignments`). The order
// goes out as a list of ids and every client numbers it by position, so keys kept
// with their gaps would put the day notes and bookings that sort between stops in
// one place for the writer, who reads the day back, and in another for everyone
// else. Only a stop whose key changes is written.
const update = this.dbs.prepare('UPDATE day_assignments SET order_index = ? WHERE id = ?');
sorted.forEach((row, i) => {
if (row.order_index !== i) update.run(i, row.id);
});
return { dayId, orderedIds: sorted.map(row => row.id), viasMoved: this.reanchorVias(dayId, rows, sorted) };
}
/**
* Keeps every drawn road behind the stop it was drawn after, the rule the planner
* applies when stops are dragged (`reanchorByStopOrder`). A via is pinned to a
* POSITION among the day's located stops, so a sort that moves a stop would
* otherwise hand the vias behind it to other legs. Inside the sort's transaction:
* an order without its vias is a road the traveller never drew.
*
* No sequence renumbering, unlike RoadtripService.reanchor: a reorder maps each leg
* onto a different one, so two legs' vias never end up on the same leg.
*/
private reanchorVias(dayId: number, before: DayStopRow[], after: DayStopRow[]): boolean {
const located = (rows: DayStopRow[]) => rows.filter(row => row.located).map(row => row.id);
const previousIds = located(before);
const nextIds = located(after);
// Only stops without coordinates moved. The router never sees those, so every leg
// is still the one it was.
if (previousIds.every((stopId, i) => stopId === nextIds[i])) return false;
// A via behind the day's last stop bends the drive into the next day, on a trip
// with connected days or a night drive (the planner's `anchorFor` files it there).
// `reanchorByStopOrder` gives a last stop no leg and deletes what follows it, which
// is right for a stop the sort made last and wrong for one that was last already.
const lastAt = previousIds.length - 1;
const seam = previousIds[lastAt] === nextIds[lastAt] ? lastAt : null;
const vias = this.dbs.all<AnchoredVia>('SELECT id, after_order_index, lat, lng FROM roadtrip_vias WHERE day_id = ?', dayId)
.filter(via => via.after_order_index !== seam);
const plan = reanchorByStopOrder(vias, previousIds, nextIds);
for (const viaId of plan.remove) {
this.dbs.run('DELETE FROM roadtrip_vias WHERE id = ? AND day_id = ?', viaId, dayId);
}
for (const via of plan.vias) {
this.dbs.run('UPDATE roadtrip_vias SET after_order_index = ? WHERE id = ? AND day_id = ?', via.after_order_index, via.id, dayId);
}
return !isEmptyReanchoring(plan);
}
/**
* The day's vias in the shape the road trip routes broadcast them. RoadtripService
* has this query too, but its module imports this one, so it cannot be injected here.
*/
private listDayVias(dayId: number): RoadtripVia[] {
return this.dbs.all<RoadtripVia>(
`SELECT id, day_id, after_order_index, sequence, lat, lng, created_at
FROM roadtrip_vias
WHERE day_id = ?
ORDER BY after_order_index, sequence, id`,
dayId,
);
}
setEndDay(id: string | number, endDay: boolean) {
@@ -107,6 +107,11 @@ export const PUBLIC_ROUTE_ALLOW_LIST: string[] = [
'DiscoveryController.protectedResource',
'DiscoveryController.wellKnownFallback',
'DiscoveryController.wellKnownRoot',
// A document provider cannot hold a TREK session. The per-binding token in
// the path is the credential, a shared secret is checked on top where the
// provider can send one, and the handler's only effect is to schedule a sync
// run. It never reads the request body as data.
'DocSyncWebhookController.nudge',
'FeaturesController.features',
// The container/uptime probe.
'FeaturesController.health',
@@ -0,0 +1,504 @@
import { Injectable } from '@nestjs/common';
import crypto from 'crypto';
import { DOCSYNC_SECRET_MASK, type DocsyncConnectionInput, type DocsyncLinkInput } from '@trek/shared';
import { DatabaseService } from '../database/database.service';
import { checkSsrf } from '../../utils/ssrfGuard';
import { DocumentProviderRegistry } from './document-provider.registry';
import type { DocumentConnectionRef, DocumentScopeRef, DocResult } from './document-provider';
import { docFailed } from './document-provider';
import { decryptSecrets, encryptSecrets, maskSecrets, mergeSecrets } from './doc-sync-secrets';
/**
* Connections and trip bindings: everything a human configures, as opposed to
* what the reconciler does with it.
*
* The connection carries a `trip_id`, and that is the one place this design
* leaves the pattern every other integration in the repo follows. Immich,
* Synology Photos, AirTrail and Dawarich all store credentials per user, and
* `trip_album_links` carries a `user_id` so that photos are shared only through
* an explicit flag. None of that can deliver "everyone on the trip sees the
* same documents": visibility would depend on whose credentials fetched a file,
* and a member without their own Paperless account would see nothing.
*
* So the trip admin binds the trip once, the server talks to the provider under
* that single identity, and TREK's own membership decides who sees what. The
* provider never learns that TREK has members. `owner_user_id` stays explicit
* beside `trip_id` precisely so that the credential holder is nameable: when
* they leave the trip the binding goes to `orphaned` rather than quietly
* carrying on with an ex-member's token.
*/
export interface ConnectionRow {
id: number;
trip_id: number;
provider_id: string;
owner_user_id: number;
base_url: string;
secrets: string | null;
settings: string;
allow_insecure_tls: number;
capabilities: string | null;
last_probe_at: string | null;
last_probe_state: string;
last_probe_error: string | null;
created_at: string;
}
export interface LinkRow {
id: number;
trip_id: number;
connection_id: number;
provider_id: string;
remote_scope_key: string;
remote_root_id: string | null;
remote_root_path: string | null;
remote_label: string;
direction: string;
delete_policy: string;
conflict_policy: string;
sync_enabled: number;
webhook_token: string | null;
webhook_secret: string | null;
webhook_subscription_id: string | null;
remote_cursor: string | null;
last_sync_at: string | null;
last_sync_state: string;
last_sync_error: string | null;
failure_count: number;
next_attempt_at: string | null;
}
interface ProviderFieldRow {
field_key: string;
/** i18n key suffix, never display text: the client resolves `docsync.<label>`. */
label: string;
input_type: string;
placeholder: string | null;
hint: string | null;
secret: number;
required: number;
sort_order: number;
}
@Injectable()
export class DocSyncConfigService {
constructor(
private readonly db: DatabaseService,
private readonly registry: DocumentProviderRegistry,
) {}
// ── Provider metadata ──────────────────────────────────────────────────────
/** Only providers the instance admin switched on may be configured. */
enabledProviderIds(): string[] {
return this.db.connection
.prepare('SELECT id FROM document_providers WHERE enabled = 1 ORDER BY sort_order')
.all()
.map((r) => (r as { id: string }).id);
}
providerFields(providerId: string): ProviderFieldRow[] {
// Every column the form needs, not just the three the secret bookkeeping
// uses: the client renders the field from this row, and a missing `label`
// turns into a literal "docsync.undefined" on screen.
return this.db.connection
.prepare(
`SELECT field_key, label, input_type, placeholder, hint, secret, required, sort_order
FROM document_provider_fields WHERE provider_id = ? ORDER BY sort_order`,
)
.all(providerId) as ProviderFieldRow[];
}
private secretKeys(providerId: string): string[] {
return this.providerFields(providerId).filter((f) => f.secret === 1).map((f) => f.field_key);
}
/**
* The display name whatever the enabled flag says. A binding outlives the
* admin switching its provider off, and the client only gets names for the
* providers that are still on, so without this it would show the raw id.
*/
private providerName(providerId: string): string {
const row = this.db.connection
.prepare('SELECT name FROM document_providers WHERE id = ?')
.get(providerId) as { name: string } | undefined;
return row?.name ?? providerId;
}
// ── Connections ────────────────────────────────────────────────────────────
getConnection(id: number): ConnectionRow | undefined {
return this.db.connection.prepare('SELECT * FROM document_connections WHERE id = ?').get(id) as ConnectionRow | undefined;
}
listConnections(tripId: number): ConnectionRow[] {
return this.db.connection
.prepare('SELECT * FROM document_connections WHERE trip_id = ? ORDER BY id')
.all(tripId) as ConnectionRow[];
}
/**
* Turn a stored row into what an adapter needs. Secrets are decrypted here
* and nowhere else, so there is one place to audit and one place that could
* leak them into a log. The ref can write back what the adapter earns, which
* is what makes it a saved connection's ref; see `saveEarnedSecret`.
*/
toRef(row: ConnectionRow): DocumentConnectionRef {
let settings: Record<string, string> = {};
try {
const parsed: unknown = JSON.parse(row.settings || '{}');
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (typeof v === 'string') settings[k] = v;
}
}
} catch {
settings = {};
}
return {
connectionId: row.id,
createdAt: row.created_at,
ownerId: row.owner_user_id,
baseUrl: row.base_url,
secrets: decryptSecrets(row.secrets),
settings,
allowInsecureTls: row.allow_insecure_tls === 1,
saveSecret: (key, value) => this.saveEarnedSecret(row.id, key, value),
};
}
/**
* Keep a secret the provider earned itself rather than one typed into the
* form, or drop it with null. DSM's device token is the one there is: without
* it, a restart would leave every two-factor NAS waiting for its owner to
* type a fresh code.
*
* It lives in the same encrypted blob as the form's secrets, under a key no
* form field has, and that is what keeps it out of every response
* (`maskSecrets` reports form fields only) and out of the form's reach
* (`mergeSecrets` writes form fields only).
*
* The blob is read back inside the transaction rather than taken from the
* caller's ref: a sync run holds its ref for minutes, and writing that
* snapshot back would undo a password the owner changed in the meantime.
*/
saveEarnedSecret(connectionId: number, key: string, value: string | null): void {
this.db.transaction(() => {
const row = this.getConnection(connectionId);
if (!row) return;
const secrets = decryptSecrets(row.secrets);
if (value === null) delete secrets[key];
else secrets[key] = value;
this.db.connection
.prepare('UPDATE document_connections SET secrets = ? WHERE id = ?')
.run(encryptSecrets(secrets), connectionId);
});
}
/** What a client may see: no secret values, only whether each one is set. */
publicConnection(row: ConnectionRow): Record<string, unknown> {
const ref = this.toRef(row);
return {
id: row.id,
tripId: row.trip_id,
providerId: row.provider_id,
ownerUserId: row.owner_user_id,
baseUrl: row.base_url,
settings: ref.settings,
secrets: maskSecrets(ref.secrets, this.secretKeys(row.provider_id)),
allowInsecureTls: row.allow_insecure_tls === 1,
capabilities: row.capabilities ? safeParse(row.capabilities) : null,
lastProbeAt: row.last_probe_at,
lastProbeState: row.last_probe_state,
lastProbeError: row.last_probe_error,
};
}
/**
* Validate a base URL before it is stored.
*
* A private address is stored WITH a warning rather than refused, because
* self-hosting is the normal case here: a Paperless on 192.168.x is the
* point of the feature, not an attack. That is the same call airtrail.service
* and dawarich.service already made. What is refused is an address that the
* SSRF guard rejects for a reason other than being private.
*/
async validateBaseUrl(raw: string): Promise<DocResult<{ url: string; isPrivate: boolean }>> {
const trimmed = raw.trim().replace(/\/+$/, '');
if (!trimmed) return { success: false, error: { code: 'unreachable', detail: 'empty url' } };
const verdict = await checkSsrf(trimmed);
if (!verdict.allowed && !verdict.isPrivate) {
return { success: false, error: { code: 'ssrf_blocked', detail: verdict.error } };
}
return { success: true, data: { url: trimmed, isPrivate: verdict.isPrivate } };
}
/**
* Create or update the trip's connection for one provider.
*
* Secrets that arrive blank or masked keep their stored value, so a client
* that renders the form from a GET never has to hold the real credential.
* A secret the provider earned itself is kept too, whatever the form sends.
* DSM's device token is stored with the address and account it was issued
* for, so an edit that points the connection somewhere else leaves it
* unusable, and the adapter drops it on its next call.
*/
async upsertConnection(
tripId: number,
userId: number,
input: DocsyncConnectionInput,
): Promise<DocResult<ConnectionRow>> {
const urlCheck = await this.validateBaseUrl(input.baseUrl);
if (docFailed(urlCheck)) return { success: false, error: urlCheck.error };
const fields = this.providerFields(input.providerId);
const secretKeys = fields.filter((f) => f.secret === 1).map((f) => f.field_key);
const plainKeys = fields.filter((f) => f.secret !== 1).map((f) => f.field_key);
const existing = this.db.connection
.prepare('SELECT * FROM document_connections WHERE trip_id = ? AND provider_id = ?')
.get(tripId, input.providerId) as ConnectionRow | undefined;
const storedSecrets = existing ? decryptSecrets(existing.secrets) : {};
const nextSecrets = mergeSecrets(storedSecrets, input.credentials, secretKeys);
const settings: Record<string, string> = {};
for (const key of plainKeys) {
if (key === 'allow_insecure_tls') continue;
const submitted = input.credentials[key];
if (submitted !== undefined) settings[key] = submitted;
else if (existing) {
const prev = this.toRef(existing).settings[key];
if (prev !== undefined) settings[key] = prev;
}
}
for (const f of fields) {
if (f.required !== 1) continue;
const present = f.secret === 1 ? !!nextSecrets[f.field_key] : !!settings[f.field_key] || f.field_key === 'base_url';
if (!present) {
return { success: false, error: { code: 'unauthorized', detail: `missing required field ${f.field_key}` } };
}
}
const encrypted = encryptSecrets(nextSecrets);
const settingsJson = JSON.stringify(settings);
const insecure = input.allowInsecureTls ? 1 : 0;
const row = this.db.transaction(() => {
if (existing) {
this.db.connection
.prepare(
`UPDATE document_connections
SET base_url = ?, secrets = ?, settings = ?, allow_insecure_tls = ?,
owner_user_id = ?, updated_at = CURRENT_TIMESTAMP
WHERE id = ?`,
)
.run(urlCheck.data.url, encrypted, settingsJson, insecure, userId, existing.id);
return this.getConnection(existing.id) as ConnectionRow;
}
const info = this.db.connection
.prepare(
`INSERT INTO document_connections (trip_id, provider_id, owner_user_id, base_url, secrets, settings, allow_insecure_tls)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
)
.run(tripId, input.providerId, userId, urlCheck.data.url, encrypted, settingsJson, insecure);
return this.getConnection(Number(info.lastInsertRowid)) as ConnectionRow;
});
return { success: true, data: row };
}
recordProbe(connectionId: number, state: 'ok' | 'failed', error: string | null, capabilities: unknown): void {
this.db.connection
.prepare(
`UPDATE document_connections
SET last_probe_at = CURRENT_TIMESTAMP, last_probe_state = ?, last_probe_error = ?,
capabilities = COALESCE(?, capabilities), updated_at = CURRENT_TIMESTAMP
WHERE id = ?`,
)
.run(state, error, capabilities ? JSON.stringify(capabilities) : null, connectionId);
}
/**
* Deleting a connection leaves every document in place. It unbinds, nothing
* more: the same promise Dawarich's disconnect makes, and the only version
* of this action that is safe to offer without a confirmation dialog.
*/
deleteConnection(id: number): void {
this.db.connection.prepare('DELETE FROM document_connections WHERE id = ?').run(id);
}
// ── Trip bindings ──────────────────────────────────────────────────────────
listLinks(tripId: number): LinkRow[] {
return this.db.connection
.prepare('SELECT * FROM trip_document_links WHERE trip_id = ? ORDER BY id')
.all(tripId) as LinkRow[];
}
getLink(id: number): LinkRow | undefined {
return this.db.connection.prepare('SELECT * FROM trip_document_links WHERE id = ?').get(id) as LinkRow | undefined;
}
getLinkByToken(token: string): LinkRow | undefined {
return this.db.connection
.prepare('SELECT * FROM trip_document_links WHERE webhook_token = ?')
.get(token) as LinkRow | undefined;
}
toScopeRef(link: LinkRow): DocumentScopeRef {
return {
linkId: link.id,
tripId: link.trip_id,
scopeKey: link.remote_scope_key,
remoteRootId: link.remote_root_id,
remoteRootPath: link.remote_root_path,
cursor: link.remote_cursor,
};
}
/**
* The anchor fields (`remoteRootId`, `remoteRootPath`, `remoteLabel`) are
* what the picker was handed by `listScopes` or `createScope` a moment
* earlier, sent back by the client. Taking them on trust is fine: only the
* trip owner or an instance admin gets here, and an anchor typed in by hand
* still goes through the connection's own credential, so it cannot open
* anything that credential could not open anyway.
*/
createLink(tripId: number, userId: number, input: DocsyncLinkInput): DocResult<LinkRow> {
const conn = this.getConnection(input.connectionId);
if (!conn || conn.trip_id !== tripId) {
return { success: false, error: { code: 'not_found', detail: 'connection not found for this trip' } };
}
const existing = this.db.connection
.prepare('SELECT * FROM trip_document_links WHERE trip_id = ? AND connection_id = ? AND remote_scope_key = ?')
.get(tripId, input.connectionId, input.scopeKey) as LinkRow | undefined;
if (existing) return { success: true, data: existing };
// A per-link token, so revoking one binding cannot be used to poke another,
// and so a leaked webhook URL identifies exactly one trip.
const token = crypto.randomBytes(24).toString('base64url');
const secret = crypto.randomBytes(24).toString('base64url');
const info = this.db.connection
.prepare(
`INSERT INTO trip_document_links
(trip_id, connection_id, provider_id, remote_scope_key, remote_root_id, remote_root_path,
remote_label, direction, delete_policy, conflict_policy, sync_enabled,
webhook_token, webhook_secret, created_by, next_attempt_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
)
.run(
tripId,
input.connectionId,
conn.provider_id,
input.scopeKey,
input.remoteRootId ?? null,
input.remoteRootPath ?? null,
input.remoteLabel ?? '',
input.direction,
input.deletePolicy,
input.conflictPolicy,
input.syncEnabled ? 1 : 0,
token,
String(encryptSecrets({ webhook: secret })),
userId,
);
return { success: true, data: this.getLink(Number(info.lastInsertRowid)) as LinkRow };
}
updateLink(id: number, patch: Partial<DocsyncLinkInput>): LinkRow | undefined {
const sets: string[] = [];
const values: unknown[] = [];
if (patch.direction !== undefined) { sets.push('direction = ?'); values.push(patch.direction); }
if (patch.deletePolicy !== undefined) { sets.push('delete_policy = ?'); values.push(patch.deletePolicy); }
if (patch.conflictPolicy !== undefined) { sets.push('conflict_policy = ?'); values.push(patch.conflictPolicy); }
if (patch.syncEnabled !== undefined) { sets.push('sync_enabled = ?'); values.push(patch.syncEnabled ? 1 : 0); }
if (patch.remoteLabel !== undefined) { sets.push('remote_label = ?'); values.push(patch.remoteLabel); }
if (sets.length === 0) return this.getLink(id);
sets.push('updated_at = CURRENT_TIMESTAMP');
this.db.connection.prepare(`UPDATE trip_document_links SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
return this.getLink(id);
}
/**
* Unbinding keeps both copies and drops the pairing rows. It never deletes a
* document on either side. A user who wants that does it deliberately, in
* the system that holds the file.
*/
deleteLink(id: number): void {
this.db.transaction(() => {
this.db.connection.prepare('DELETE FROM document_sync_items WHERE link_id = ?').run(id);
this.db.connection.prepare('DELETE FROM trip_document_links WHERE id = ?').run(id);
});
}
webhookSecret(link: LinkRow): string {
return decryptSecrets(link.webhook_secret).webhook ?? '';
}
/** The masked view a client gets, with the webhook URL it may need to paste. */
publicLink(link: LinkRow, webhookBaseUrl: string | null): Record<string, unknown> {
return {
id: link.id,
tripId: link.trip_id,
connectionId: link.connection_id,
providerId: link.provider_id,
providerName: this.providerName(link.provider_id),
scopeKey: link.remote_scope_key,
remoteRootId: link.remote_root_id,
remoteRootPath: link.remote_root_path,
remoteLabel: link.remote_label,
direction: link.direction,
deletePolicy: link.delete_policy,
conflictPolicy: link.conflict_policy,
syncEnabled: link.sync_enabled === 1,
lastSyncAt: link.last_sync_at,
lastSyncState: link.last_sync_state,
lastSyncError: link.last_sync_error,
failureCount: link.failure_count,
// Shown so a user can paste it into a provider that will not let TREK
// subscribe on its own (Papra, and Nextcloud without admin rights).
webhookUrl: webhookBaseUrl && link.webhook_token
? `${webhookBaseUrl}/api/docsync/webhook/${link.webhook_token}`
: null,
webhookSecret: link.webhook_secret ? DOCSYNC_SECRET_MASK : null,
};
}
/**
* Bindings whose credential owner is no longer on the trip.
*
* Checked on every run rather than hooked to member removal, because a
* membership can also disappear through a trip transfer or a direct DB edit,
* and a binding that keeps using an ex-member's token is the kind of thing
* nobody notices until it is a complaint.
*/
markOrphanedLinks(): number {
const info = this.db.connection
.prepare(
`UPDATE trip_document_links
SET last_sync_state = 'orphaned', sync_enabled = 0, updated_at = CURRENT_TIMESTAMP
WHERE sync_enabled = 1
AND last_sync_state != 'orphaned'
AND connection_id IN (
SELECT c.id FROM document_connections c
WHERE c.owner_user_id NOT IN (
SELECT tm.user_id FROM trip_members tm WHERE tm.trip_id = c.trip_id
UNION SELECT t.user_id FROM trips t WHERE t.id = c.trip_id
)
)`,
)
.run();
return info.changes;
}
}
function safeParse(value: string): unknown {
try {
return JSON.parse(value);
} catch {
return null;
}
}
@@ -0,0 +1,83 @@
import { DOCSYNC_SECRET_MASK } from '@trek/shared';
import { decrypt_api_key, maybe_encrypt_api_key } from '../common/crypto/apiKeyCrypto';
/**
* Credential handling for document connections.
*
* All secret fields of a connection live in ONE encrypted JSON column rather
* than a column per provider. A sixth provider then needs no migration, and the
* key rotation in scripts/migrate-encryption.ts stays a single line instead of
* a field list somebody will extend and forget. A forgotten column does not
* survive a rotation, and the failure only shows up months later as a
* connection that cannot authenticate any more.
*/
export type SecretMap = Record<string, string>;
/** Encrypt the whole secret map for storage. Empty maps store as NULL. */
export function encryptSecrets(secrets: SecretMap): string | null {
const entries = Object.entries(secrets).filter(([, v]) => typeof v === 'string' && v.length > 0);
if (entries.length === 0) return null;
return String(maybe_encrypt_api_key(JSON.stringify(Object.fromEntries(entries))));
}
/**
* Decrypt for use. A value that cannot be decrypted returns an empty map rather
* than throwing: the caller then reports `unauthorized` and the user re-enters
* the credential, which is the recoverable outcome. That is the same choice
* synology.service.ts makes for a stale SID: a decryption failure after an
* ENCRYPTION_KEY rotation must not turn into a boot-time crash.
*/
export function decryptSecrets(stored: string | null | undefined): SecretMap {
if (!stored) return {};
const plain = decrypt_api_key(stored);
if (typeof plain !== 'string' || plain.length === 0) return {};
try {
const parsed: unknown = JSON.parse(plain);
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {};
const out: SecretMap = {};
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (typeof v === 'string') out[k] = v;
}
return out;
} catch {
return {};
}
}
/**
* Merge submitted form values over the stored ones.
*
* A secret field that arrives blank, absent, or as the mask keeps its stored
* value. That rule is what lets the client render a connection form without
* ever holding the secret it was given back, the same contract the photo
* provider form and Dawarich already use, so the generic component needs no
* special case here.
*
* A stored key that is no form field is a secret the provider earned itself
* (DSM's device token). It is carried over untouched and cannot be set from
* here: the form never saw it, so an edit can neither wipe it nor plant one.
*/
export function mergeSecrets(stored: SecretMap, submitted: Record<string, string>, secretKeys: readonly string[]): SecretMap {
const next: SecretMap = { ...stored };
for (const key of secretKeys) {
const value = submitted[key];
if (value === undefined) continue;
if (value === '' || value === DOCSYNC_SECRET_MASK) continue;
next[key] = value;
}
return next;
}
/**
* What a client is allowed to see: whether a secret is set, never its value.
* Form fields only, so a secret the provider earned itself is not even
* reported as present.
*/
export function maskSecrets(secrets: SecretMap, secretKeys: readonly string[]): Record<string, string> {
const out: Record<string, string> = {};
for (const key of secretKeys) {
if (secrets[key]) out[key] = DOCSYNC_SECRET_MASK;
}
return out;
}
@@ -0,0 +1,155 @@
import { Controller, HttpCode, OnModuleDestroy, Param, Post, Req } from '@nestjs/common';
import type { Request } from 'express';
import crypto from 'crypto';
import { Public } from '../auth/public.decorator';
import { DatabaseService } from '../database/database.service';
import { SETTING_SYNC_ENABLED, WEBHOOK_NUDGE_DEBOUNCE_SECONDS } from './doc-sync.constants';
import { DocSyncConfigService, type LinkRow } from './doc-sync-config.service';
import { DocSyncService } from './doc-sync.service';
/**
* `/api/docsync/webhook/:token`: the one endpoint a provider calls.
*
* Deliberately the thinnest thing in the domain. It answers 200 and schedules a
* run; it never reads the body as truth. Every provider here has a different
* payload, none of them signs it in a way all five share, Paperless gives its
* webhook five seconds before it retries, and Nextcloud's payload carries a
* path that is different for every member of a share. Treating any of that as
* data would mean trusting an unauthenticated stranger's description of what
* changed. So the webhook means exactly one thing: look now.
*
* `@Public` because a provider cannot hold a TREK session. The token in the URL
* is the authentication, one per binding, so a leaked URL can only ever nudge
* the one trip it belongs to, and nudging is all it can do. Where the provider
* supports it, a shared secret is checked as well.
*/
@Controller('api/docsync/webhook')
export class DocSyncWebhookController implements OnModuleDestroy {
/**
* One pending run per binding, so a burst folds into a single pass.
*
* Providers fire per document: dropping twenty files into a watched Nextcloud
* folder is twenty calls within a second or two. Each one used to start its
* own run, which the service's in-flight guard then answered with `busy`,
* so nineteen changes were announced and thrown away, and the one run that
* did start had begun before most of them landed. Collecting them for a beat
* and then running once is both less work and more correct.
*/
private readonly pending = new Map<number, ReturnType<typeof setTimeout>>();
constructor(
private readonly config: DocSyncConfigService,
private readonly sync: DocSyncService,
private readonly db: DatabaseService,
) {}
/**
* The same switches the scheduler obeys: the Documents addon, the binding's
* provider, and the instance-wide kill switch.
*
* Without them "switch document sync off" meant "stop the poll", while every
* provider holding a webhook kept driving full runs: an admin turning the
* addon off would have watched it carry on. Checked when the timer fires as
* well as on arrival, so a switch thrown during the debounce window still
* takes effect.
*/
private syncIsOn(link: LinkRow): boolean {
if (this.sync.isSwitchedOff(link)) return false;
const killSwitch = this.db.get<{ value: string }>(
'SELECT value FROM app_settings WHERE key = ?', SETTING_SYNC_ENABLED,
)?.value;
// Unrecognised values mean ON: the setting is absent by default, and only an
// explicit 'false' stops the sync. Same rule as the job.
return killSwitch !== 'false';
}
onModuleDestroy(): void {
for (const timer of this.pending.values()) clearTimeout(timer);
this.pending.clear();
}
@Post(':token')
@Public('A provider cannot hold a TREK session; the per-link token in the URL is the authentication, and the call can only ever trigger a sync run.')
@HttpCode(200)
nudge(@Param('token') token: string, @Req() req: Request) {
const link = this.config.getLinkByToken(token);
// Always 200, even for an unknown token: a 404 here would let anyone probe
// which tokens exist, and a provider that gets an error will retry anyway.
if (!link || link.sync_enabled !== 1) return { received: true };
if (!this.syncIsOn(link)) return { received: true };
const secret = this.config.webhookSecret(link);
if (secret && !this.secretMatches(req, secret)) return { received: true };
// Fire and forget. Paperless allows five seconds before it counts the call
// as failed and retries, and a sync run takes longer than that whenever
// there is anything to do, so the answer goes out now and the run happens
// after the debounce window, by which time the rest of the burst has
// arrived and been folded into this same timer.
this.schedule(link.id, () => this.config.getLink(link.id));
return { received: true };
}
/**
* Start one run per binding per window, trailing rather than leading.
*
* Trailing on purpose: the first call of a burst is the least informed one,
* because the provider is usually still writing the rest. The link is looked
* up again when the timer fires, so a binding switched off or deleted in the
* meantime does not get one last run out of a stale row.
*/
private schedule(linkId: number, reload: () => ReturnType<DocSyncConfigService['getLink']>, isRetry = false): void {
if (this.pending.has(linkId)) return;
const timer = setTimeout(() => {
this.pending.delete(linkId);
const fresh = reload();
if (!fresh || fresh.sync_enabled !== 1) return;
if (!this.syncIsOn(fresh)) return;
void this.sync.syncLink(fresh).then((res) => {
// A run that was already in flight answers `busy`, and the changes this
// nudge was about may have landed after that run read the folder. Ask
// again once rather than waiting out a whole poll interval: once, and
// only for `busy`, so this cannot become a loop.
if (res?.state === 'busy' && !isRetry) this.schedule(linkId, reload, true);
});
}, WEBHOOK_NUDGE_DEBOUNCE_SECONDS * 1000);
// A pending nudge must not hold the process open at shutdown.
if (typeof timer.unref === 'function') timer.unref();
this.pending.set(linkId, timer);
}
/**
* Accept either a plain shared-secret header (Paperless workflows, Nextcloud
* `authMethod: header`) or Papra's standard-webhooks HMAC. Compared in
* constant time, and a mismatch is silently ignored rather than reported, so
* the endpoint tells a prober nothing either way.
*/
private secretMatches(req: Request, secret: string): boolean {
const header = req.get('x-trek-docsync-secret');
if (header && timingSafeEqualStr(header, secret)) return true;
const sig = req.get('webhook-signature');
const id = req.get('webhook-id');
const ts = req.get('webhook-timestamp');
if (sig && id && ts) {
const raw = (req as Request & { rawBody?: Buffer }).rawBody;
const body = raw ? raw.toString('utf8') : JSON.stringify(req.body ?? {});
const expected = crypto
.createHmac('sha256', Buffer.from(secret))
.update(`${id}.${ts}.${body}`)
.digest('base64');
for (const part of sig.split(' ')) {
const value = part.startsWith('v1,') ? part.slice(3) : part;
if (timingSafeEqualStr(value, expected)) return true;
}
}
return false;
}
}
function timingSafeEqualStr(a: string, b: string): boolean {
const ab = Buffer.from(a);
const bb = Buffer.from(b);
if (ab.length !== bb.length) return false;
return crypto.timingSafeEqual(ab, bb);
}
@@ -0,0 +1,94 @@
/**
* Tuning constants for the document sync core.
*
* Kept out of the service so the pure planner can be unit-tested against them
* without a container, and so a reviewer can see every limit in one place
* instead of hunting for magic numbers.
*/
/** Per-request ceiling for provider calls that return metadata. */
export const PROVIDER_TIMEOUT_MS = 15_000;
/** Transfers get longer, because a 40 MB PDF over a home upstream is not fast. */
export const PROVIDER_TRANSFER_TIMEOUT_MS = 120_000;
/** Metadata responses are JSON; nothing legitimate here is megabytes. */
export const PROVIDER_JSON_MAX_BYTES = 8 * 1024 * 1024;
/**
* Page cap per listing run, mirroring the reason dawarich.client.ts states: a
* provider that keeps answering `next` must not hold a worker forever. Hitting
* it sets `truncated`, which the core surfaces rather than silently treating a
* partial listing as the whole truth.
*/
export const PROVIDER_MAX_PAGES = 40;
/**
* The share of a link's known documents that may disappear in one run before
* the run refuses to act on any of it.
*
* An unmounted Synology share, a revoked Nextcloud share and an expired
* Paperless token all produce the same thing: a listing far shorter than the
* last one. Without this guard the reconciler would read that as a mass
* deletion. Below the threshold, vanished documents are recorded as
* `remote_missing` and wait for a human either way. This only decides whether
* the run trusts the listing enough to record anything at all.
*/
export const MASS_DELETE_RATIO = 0.4;
/** Below this count the ratio is meaningless, so the guard only counts. */
export const MASS_DELETE_MIN_ITEMS = 5;
/**
* Backoff for a failing link, in seconds, indexed by consecutive failures.
* Flat 30 minutes after the sixth, because a provider that has been down for
* three hours is a configuration problem, not a blip, and hammering it earns a
* DSM auto-block.
*/
export const LINK_BACKOFF_SECONDS = [60, 300, 900, 1800, 3600, 7200] as const;
/** Per-document retry curve, shorter: usually a transient upload failure. */
export const ITEM_BACKOFF_SECONDS = [30, 120, 600, 3600] as const;
/** After this many attempts a document stops retrying and waits for a human. */
export const ITEM_MAX_ATTEMPTS = 6;
/**
* Consecutive link failures after which the circuit opens and the job stops
* scheduling it at all until someone presses "sync now".
*/
export const LINK_CIRCUIT_OPEN_AFTER = 10;
/** Default poll interval. Clamped, overridable per instance in app_settings. */
export const DEFAULT_POLL_INTERVAL_SECONDS = 300;
export const MIN_POLL_INTERVAL_SECONDS = 60;
export const MAX_POLL_INTERVAL_SECONDS = 3600;
/**
* How long a webhook may accelerate a link before the poll takes over again.
* A webhook only ever means "look now". This keeps a link whose subscription
* silently died from going stale forever.
*/
export const WEBHOOK_NUDGE_DEBOUNCE_SECONDS = 5;
/** Documents pulled or pushed per run, so one huge trip cannot starve others. */
export const MAX_TRANSFERS_PER_RUN = 25;
/**
* What a manual run answers for a binding an admin has switched off, through
* its provider or through the Documents addon.
*
* Not one of the shared DocsyncErrorCodes: those say what went wrong between
* TREK and a provider, and here nothing has. It is never written to the binding
* either, which is what lets the binding resume as it was left.
*/
export const PROVIDER_DISABLED = 'provider_disabled';
/** app_settings keys, read per tick so an admin toggle needs no restart. */
export const SETTING_POLL_INTERVAL = 'docsync_poll_interval_seconds';
export const SETTING_SYNC_ENABLED = 'docsync_sync_enabled';
// There is deliberately no size key here: a document that arrives through a
// binding is bound by the same ceiling as one somebody uploads (MAX_FILE_SIZE in
// files.constants), checked against the announced size and again while the
// bytes stream. A second, sync-only limit would be a number nobody maintains
// and a way for the two paths to disagree about what fits.
@@ -0,0 +1,439 @@
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpException,
Param,
Patch,
Post,
Put,
Query,
Req,
UseGuards,
} from '@nestjs/common';
import type { Request } from 'express';
import type { User } from '../../types';
import { ADDON_IDS } from '../../addons';
import { AddonGuard } from '../addons/addon.guard';
import { RequireAddon } from '../addons/require-addon.decorator';
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
import { CurrentUser } from '../auth/current-user.decorator';
import { TripAccessGuard } from '../permissions/trip-access.guard';
import { DatabaseService } from '../database/database.service';
import { RealtimeService } from '../realtime/realtime.service';
import { docFailed } from './document-provider';
import { DocSyncConfigService, type LinkRow } from './doc-sync-config.service';
import { DocSyncService } from './doc-sync.service';
import { DocumentProviderRegistry } from './document-provider.registry';
import { PROVIDER_DISABLED } from './doc-sync.constants';
import {
DocsyncConnectionDto,
DocsyncConnectionTestDto,
DocsyncLinkDto,
DocsyncLinkUpdateDto,
DocsyncResolveConflictDto,
DocsyncScopeCreateDto,
DocsyncSyncNowDto,
} from './doc-sync.dto';
/**
* `/api/trips/:tripId/docsync`: a trip's document provider binding.
*
* Trip-scoped rather than user-scoped, and that is the whole point of the
* feature: one connection per trip, which every member reads through. The
* routes therefore sit under the trip, not under `/api/integrations`.
*
* Two shapes of answer, the split Dawarich already established: connection
* routes (`test`, `status`) answer 200 even when the provider is unreachable,
* with the reason in the body, because a form showing someone their own typo
* needs a field to render rather than an exception. Everything else answers the
* status the failure deserves.
*
* Only the trip owner may change a binding (plus an instance admin, who
* overrides trip permissions everywhere else too). It hands TREK a credential
* that usually reaches the owner's entire document archive, so widening this to
* every member would let any member point the trip at a folder the owner never
* meant to share. Reading the status is open to all members: they need to know
* where their documents are going.
*/
@Controller('api/trips/:tripId/docsync')
@UseGuards(AddonGuard, JwtAuthGuard, TripAccessGuard)
@RequireAddon(ADDON_IDS.DOCUMENTS, 'Documents')
export class DocSyncController {
constructor(
private readonly config: DocSyncConfigService,
private readonly sync: DocSyncService,
private readonly registry: DocumentProviderRegistry,
private readonly db: DatabaseService,
private readonly realtime: RealtimeService,
) {}
/**
* Tell the trip that a binding came or went.
*
* Whether a trip is bound decides whether its members see the sync button
* at all, and a run only pings when it moved a document. Binding two empty
* sides, or unbinding, moves nothing, so a member sitting on the Files tab
* kept the old answer until they left it. The whole room, the caller
* included: it is a ping to re-read, not a change to apply, and the owner's
* own button depends on it too once every provider is switched off.
*/
private announceBinding(link: Pick<LinkRow, 'id' | 'trip_id'>): void {
this.realtime.broadcast(link.trip_id, 'docsync:changed', { linkId: link.id, pulled: 0, pushed: 0 });
}
/**
* The trip owner or an instance admin, the same rule as `canManageDocSync`
* on the client. Checked here rather than through @RequirePermission: the
* permission table has no "manage integrations" action and inventing one
* would grant it to every trip_member by default, which is the opposite of
* what a credential this broad needs.
*
* The 403 still says "owner": an admin never sees it, and to everybody who
* does, the owner is the person to ask.
*/
private assertCanManage(tripId: string, user: User): void {
if (user.role === 'admin') return;
const trip = this.db.get<{ user_id: number }>('SELECT user_id FROM trips WHERE id = ?', tripId);
if (!trip) throw new HttpException('Trip not found', 404);
if (Number(trip.user_id) !== Number(user.id)) {
throw new HttpException('Only the trip owner can change document sync', 403);
}
}
// ── Providers and status ───────────────────────────────────────────────────
/** Which providers this instance has switched on, with their form fields. */
@Get('providers')
providers() {
const rows = this.db.connection
.prepare('SELECT id, name, description, icon FROM document_providers WHERE enabled = 1 ORDER BY sort_order')
.all() as Array<{ id: string; name: string; description: string | null; icon: string }>;
return rows.map((p) => ({
...p,
// A provider row with no registered adapter would render a form that
// cannot work, so it is reported rather than hidden.
available: !!this.registry.get(p.id),
fields: this.config.providerFields(p.id).map((f) => ({ ...f, secret: f.secret === 1, required: f.required === 1 })),
}));
}
@Get('status')
status(@Param('tripId') tripId: string) {
return this.sync.status(Number(tripId));
}
// ── Connections ────────────────────────────────────────────────────────────
@Get('connections')
listConnections(@Param('tripId') tripId: string) {
return this.config.listConnections(Number(tripId)).map((c) => this.config.publicConnection(c));
}
@Put('connections')
@HttpCode(200)
async upsertConnection(
@Param('tripId') tripId: string,
@CurrentUser() user: User,
@Body() body: DocsyncConnectionDto,
) {
this.assertCanManage(tripId, user);
if (!this.config.enabledProviderIds().includes(body.providerId)) {
throw new HttpException(`Provider: "${body.providerId}" is not enabled, contact server administrator`, 400);
}
const res = await this.config.upsertConnection(Number(tripId), Number(user.id), body);
if (docFailed(res)) throw new HttpException(res.error.detail || res.error.code, 400);
return this.config.publicConnection(res.data);
}
/**
* Probe form values. Always 200: the result is in the body, because this is
* a form field, not a failure of the request. The photo providers pin the
* same contract with a test their e2e suite marks CRITICAL.
*/
@Post('connections/test')
@HttpCode(200)
async testConnection(
@Param('tripId') tripId: string,
@CurrentUser() user: User,
@Body() body: DocsyncConnectionTestDto,
) {
this.assertCanManage(tripId, user);
const provider = this.registry.get(body.providerId);
if (!provider) return { connected: false, error: 'unknown_provider' };
const urlCheck = await this.config.validateBaseUrl(body.baseUrl);
if (docFailed(urlCheck)) return { connected: false, error: urlCheck.error.code };
// Probing with the values on screen, merged over whatever is stored, so a
// user testing an unchanged connection does not have to retype the secret.
const existing = this.config
.listConnections(Number(tripId))
.find((c) => c.provider_id === body.providerId);
// Only for the address the credential was stored against. Merging it into a
// probe of an arbitrary baseUrl turns this route into a way to have TREK
// post a stored API token at a server of the caller's choosing, which is
// exactly what somebody who inherited a trip but not its credentials would
// reach for. Same host, same scheme, same port, or the caller types it in.
const sameTarget = !!existing && sameOrigin(existing.base_url, urlCheck.data.url);
const stored = sameTarget && existing ? this.config.toRef(existing) : null;
const fields = this.config.providerFields(body.providerId);
const secrets: Record<string, string> = { ...(stored?.secrets ?? {}) };
const settings: Record<string, string> = { ...(stored?.settings ?? {}) };
for (const f of fields) {
const submitted = body.credentials[f.field_key];
if (submitted === undefined || submitted === '') continue;
if (f.secret === 1) secrets[f.field_key] = submitted;
else settings[f.field_key] = submitted;
}
const res = await provider.probe({
connectionId: existing?.id ?? 0,
createdAt: existing?.created_at ?? '',
ownerId: Number(user.id),
baseUrl: urlCheck.data.url,
secrets,
settings,
allowInsecureTls: body.allowInsecureTls,
});
if (docFailed(res)) return { connected: false, error: res.error.code, detail: res.error.detail };
if (existing) this.config.recordProbe(existing.id, 'ok', null, res.data.capabilities);
return { connected: true, account: res.data.account, capabilities: res.data.capabilities };
}
@Delete('connections/:connectionId')
@HttpCode(200)
deleteConnection(
@Param('tripId') tripId: string,
@Param('connectionId') connectionId: string,
@CurrentUser() user: User,
) {
this.assertCanManage(tripId, user);
const conn = this.config.getConnection(Number(connectionId));
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
// Its bindings go with it (ON DELETE CASCADE), so read them first.
const unbound = this.config.listLinks(conn.trip_id).filter((l) => l.connection_id === conn.id);
this.config.deleteConnection(conn.id);
for (const link of unbound) this.announceBinding(link);
return { success: true };
}
// ── Scopes ─────────────────────────────────────────────────────────────────
/** Folders, tags or spaces the trip could be bound to. */
@Get('connections/:connectionId/scopes')
async listScopes(
@Param('tripId') tripId: string,
@Param('connectionId') connectionId: string,
@CurrentUser() user: User,
@Query('q') q?: string,
) {
this.assertCanManage(tripId, user);
const conn = this.config.getConnection(Number(connectionId));
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
const provider = this.registry.get(conn.provider_id);
if (!provider) throw new HttpException('Provider not available', 400);
const res = await provider.listScopes(this.config.toRef(conn), q);
if (docFailed(res)) return { scopes: [], error: res.error.code };
return { scopes: res.data };
}
@Post('connections/:connectionId/scopes')
async createScope(
@Param('tripId') tripId: string,
@Param('connectionId') connectionId: string,
@CurrentUser() user: User,
@Body() body: DocsyncScopeCreateDto,
) {
this.assertCanManage(tripId, user);
const conn = this.config.getConnection(Number(connectionId));
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
const provider = this.registry.get(conn.provider_id);
if (!provider) throw new HttpException('Provider not available', 400);
const res = await provider.createScope(this.config.toRef(conn), body.name);
if (docFailed(res)) throw new HttpException(res.error.code, 400);
return res.data;
}
// ── Bindings ───────────────────────────────────────────────────────────────
@Get('links')
listLinks(@Param('tripId') tripId: string, @Req() req: Request) {
const base = publicOrigin(req);
return this.config.listLinks(Number(tripId)).map((l) => this.config.publicLink(l, base));
}
@Post('links')
@HttpCode(200)
async createLink(
@Param('tripId') tripId: string,
@CurrentUser() user: User,
@Body() body: DocsyncLinkDto,
@Req() req: Request,
) {
this.assertCanManage(tripId, user);
const res = this.config.createLink(Number(tripId), Number(user.id), body);
if (docFailed(res)) throw new HttpException(res.error.detail || res.error.code, 400);
this.announceBinding(res.data);
// Subscribe where the provider lets TREK do it itself. A failure here is
// not a failure of the binding (polling still carries it), so it is logged
// into the link state rather than thrown at the user.
const conn = this.config.getConnection(res.data.connection_id);
const provider = conn ? this.registry.get(conn.provider_id) : undefined;
if (conn && provider?.registerWebhook) {
const base = publicOrigin(req);
if (base) {
const hook = await provider.registerWebhook(
this.config.toRef(conn),
this.config.toScopeRef(res.data),
`${base}/api/docsync/webhook/${res.data.webhook_token}`,
this.config.webhookSecret(res.data),
);
if (hook.success) {
this.db.connection
.prepare('UPDATE trip_document_links SET webhook_subscription_id = ? WHERE id = ?')
.run(hook.data.subscriptionId, res.data.id);
}
}
}
// A first run right away, so the user sees something happen instead of
// waiting out a poll interval and wondering whether it worked.
void this.sync.syncLink(res.data, { full: true });
return this.config.publicLink(this.config.getLink(res.data.id) ?? res.data, publicOrigin(req));
}
@Patch('links/:linkId')
@HttpCode(200)
updateLink(
@Param('tripId') tripId: string,
@Param('linkId') linkId: string,
@CurrentUser() user: User,
@Body() body: DocsyncLinkUpdateDto,
@Req() req: Request,
) {
this.assertCanManage(tripId, user);
const link = this.config.getLink(Number(linkId));
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
const updated = this.config.updateLink(link.id, body);
return this.config.publicLink(updated ?? link, publicOrigin(req));
}
@Delete('links/:linkId')
@HttpCode(200)
async deleteLink(
@Param('tripId') tripId: string,
@Param('linkId') linkId: string,
@CurrentUser() user: User,
) {
this.assertCanManage(tripId, user);
const link = this.config.getLink(Number(linkId));
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
const conn = this.config.getConnection(link.connection_id);
const provider = conn ? this.registry.get(conn.provider_id) : undefined;
if (conn && provider?.unregisterWebhook && link.webhook_subscription_id) {
await provider.unregisterWebhook(this.config.toRef(conn), link.webhook_subscription_id);
}
this.config.deleteLink(link.id);
this.announceBinding(link);
// Unbinding keeps both copies. Nothing is deleted anywhere.
return { success: true, documentsKept: true };
}
@Post('links/:linkId/sync')
@HttpCode(200)
async syncNow(
@Param('tripId') tripId: string,
@Param('linkId') linkId: string,
@Body() body: DocsyncSyncNowDto,
) {
const link = this.config.getLink(Number(linkId));
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
// An orphaned binding stays orphaned: its credential belongs to somebody who
// is no longer on this trip, and a manual run would use it anyway. The
// scheduler already refuses it (sync_enabled is 0), so without this the
// button was a way around that.
if (link.last_sync_state === 'orphaned') {
throw new HttpException({ error: 'This binding lost its owner and has to be reconnected' }, 409);
}
// Refused before the shelved rows are touched, so a binding an admin
// switched off stays exactly as it was and resumes where it stopped. A code
// rather than a sentence: the client says it in the reader's language.
if (this.sync.isSwitchedOff(link)) {
throw new HttpException({ error: PROVIDER_DISABLED }, 409);
}
// A person asking for a run is also asking for the rows that gave up to be
// tried once more; the scheduler gets no such reprieve.
this.sync.retryShelvedItems(link.id);
return this.sync.syncLink(link, { full: body.full });
}
// ── Documents and conflicts ────────────────────────────────────────────────
@Get('items')
items(@Param('tripId') tripId: string, @Query('state') state?: string) {
const rows = state
? this.db.connection
.prepare(
`SELECT i.*, f.original_name AS file_name FROM document_sync_items i
LEFT JOIN trip_files f ON f.id = i.file_id
WHERE i.trip_id = ? AND i.state = ? ORDER BY i.id DESC LIMIT 500`,
)
.all(Number(tripId), state)
: this.db.connection
.prepare(
`SELECT i.*, f.original_name AS file_name FROM document_sync_items i
LEFT JOIN trip_files f ON f.id = i.file_id
WHERE i.trip_id = ? ORDER BY i.id DESC LIMIT 500`,
)
.all(Number(tripId));
return rows;
}
@Post('items/:itemId/resolve')
@HttpCode(200)
async resolve(
@Param('tripId') tripId: string,
@Param('itemId') itemId: string,
@CurrentUser() user: User,
@Body() body: DocsyncResolveConflictDto,
) {
this.assertCanManage(tripId, user);
const ok = await this.sync.resolveConflict(Number(itemId), body.keep, Number(tripId));
if (!ok) throw new HttpException('Item is not in conflict', 400);
return { success: true };
}
}
/** Whether two URLs address the same server, for deciding if a stored secret may be reused. */
function sameOrigin(a: string | null | undefined, b: string): boolean {
if (!a) return false;
try {
const x = new URL(a);
const y = new URL(b);
return x.protocol === y.protocol && x.host === y.host;
} catch {
return false;
}
}
/**
* The origin a provider has to call back on.
*
* Derived from the request rather than from configuration because a self-hosted
* TREK is reached under whatever name its operator chose, and asking them to
* maintain a second copy of it would get it wrong. Returns null behind a proxy
* that strips the host, in which case the webhook is simply not offered and
* polling carries the binding.
*/
function publicOrigin(req: Request): string | null {
const host = req.get('x-forwarded-host') || req.get('host');
if (!host) return null;
const proto = (req.get('x-forwarded-proto') || req.protocol || 'http').split(',')[0].trim();
return `${proto}://${host}`;
}
+24
View File
@@ -0,0 +1,24 @@
import { createZodDto } from 'nestjs-zod';
import {
docsyncConnectionInputSchema,
docsyncConnectionTestSchema,
docsyncLinkInputSchema,
docsyncLinkUpdateSchema,
docsyncResolveConflictSchema,
docsyncScopeCreateSchema,
docsyncSyncNowSchema,
} from '@trek/shared';
/**
* Zod DTOs for every mutating route in this domain.
*
* The boot-time ratchet in common/validate-body-contracts.ts refuses to start
* if a mutation takes an unwrapped body, so these are not optional decoration.
*/
export class DocsyncConnectionDto extends createZodDto(docsyncConnectionInputSchema) {}
export class DocsyncConnectionTestDto extends createZodDto(docsyncConnectionTestSchema) {}
export class DocsyncLinkDto extends createZodDto(docsyncLinkInputSchema) {}
export class DocsyncLinkUpdateDto extends createZodDto(docsyncLinkUpdateSchema) {}
export class DocsyncScopeCreateDto extends createZodDto(docsyncScopeCreateSchema) {}
export class DocsyncResolveConflictDto extends createZodDto(docsyncResolveConflictSchema) {}
export class DocsyncSyncNowDto extends createZodDto(docsyncSyncNowSchema) {}
@@ -0,0 +1,677 @@
import crypto from 'crypto';
import path from 'path';
import { BLOCKED_EXTENSIONS } from '../files/files.constants';
import { MASS_DELETE_MIN_ITEMS, MASS_DELETE_RATIO } from './doc-sync.constants';
import type { RemoteDocument } from './document-provider';
/**
* The pure half of the sync core.
*
* `planReconcile` takes a snapshot of both sides and returns the actions to
* take. It touches no database, no HTTP and no container, which is what lets
* the hard cases (conflict, echo, mass deletion, rename, a document that
* exists on neither side any more) be tested exhaustively as plain data. The
* service does nothing but execute the plan.
*/
// ── Identity and incoming names ──────────────────────────────────────────────
/** Per-document anchor, written into provider metadata where there is room. */
export function newTrekDocUid(): string {
return crypto.randomUUID();
}
/**
* Names arriving from a provider are hostile input: they can carry path
* separators, control characters, leading dots and NTFS-illegal characters, and
* they end up both in a DB column and in a Content-Disposition header.
*
* The extension is preserved deliberately: it is what TREK's download route
* derives the Content-Type from, and what the blocklist check reads.
*/
export function sanitizeIncomingName(raw: string): string {
// Split on separators by hand rather than through path.basename: the win32
// flavour reads `a:` as a drive letter and silently drops it, so a perfectly
// ordinary `a:b.pdf` from a Linux provider would arrive as `b.pdf`. Only
// slashes are directory separators as far as this is concerned; the colon is
// dealt with below, as an illegal character.
const base = (raw || '').split(/[\\/]/).pop()?.trim() ?? '';
const cleaned = base
// eslint-disable-next-line no-control-regex
.replace(/[\u0000-\u001f\u007f]/g, '')
.replace(/[\\/:*?"<>|]/g, '_')
.replace(/^\.+/, '')
.slice(0, 200);
return cleaned || 'document';
}
/**
* The blocklist is enforced on incoming provider documents for exactly the same
* reason it is enforced on uploads: TREK serves downloads inline with an
* extension-derived Content-Type, so an .svg or .html from a Nextcloud folder
* would be a stored XSS. A rejected document becomes a visible `rejected_type`
* row rather than a silent skip, so the person who put it there can find out.
*/
export function isBlockedName(name: string): boolean {
const ext = path.extname(name).toLowerCase();
return !!ext && BLOCKED_EXTENSIONS.includes(ext);
}
/**
* The operator's own allowlist, applied to incoming provider documents.
*
* Same rule as an upload: `*` admits anything the blocklist above still lets
* through, and an empty extension is refused rather than waved past, because a
* file with no extension is served with a Content-Type TREK had to guess.
*/
export function isAllowedByOperator(name: string, allowedCsv: string): boolean {
const ext = path.extname(name).toLowerCase().replace(/^\./, '');
if (!ext) return false;
if (allowedCsv.trim() === '*') return true;
return allowedCsv
.split(',')
.map((e) => e.trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean)
.includes(ext);
}
// ── The plan ─────────────────────────────────────────────────────────────────
/** What the core knows about one pairing before the run. */
export interface SyncItemState {
id: number;
fileId: number | null;
trekDocUid: string;
remoteId: string | null;
remoteVersion: string | null;
/** The name the provider had at the last agreed state: the rename arbiter. */
remoteName: string | null;
/**
* Size and modification time of the provider copy as last seen. Only read
* where ids are paths, to recognise the same copy after a rename or a move.
*/
remoteSize: number | null;
remoteModifiedAt: string | null;
/** The bytes both sides last agreed on. */
contentSha256: string | null;
/** What TREK itself last uploaded: the echo guard. */
pushedSha256: string | null;
state: string;
attempts: number;
/** When the backoff lets this row be tried again; null once it is shelved. */
nextAttemptAt: string | null;
remoteMissingAt: string | null;
/**
* When TREK itself put the provider copy in the recycle bin, under the
* `trash` policy. A copy binned by TREK and one deleted by somebody else
* leave the same gap in a listing; only this tells them apart once the file
* comes back out of TREK's trash.
*/
remoteTrashedAt: string | null;
}
/** What TREK currently holds for this trip. */
export interface LocalDocument {
fileId: number;
name: string;
size: number;
mimeType: string | null;
sha256: string | null;
deletedAt: string | null;
}
export type PlanAction =
/** Bytes only exist upstream: download and create a trip_files row. */
| { kind: 'pull'; remote: RemoteDocument; itemId: number | null }
/** Bytes only exist in TREK: upload. */
| { kind: 'push'; local: LocalDocument; itemId: number | null; remoteId: string | null }
/** Upstream moved on while TREK did not: replace local bytes. */
| { kind: 'pull_update'; remote: RemoteDocument; itemId: number }
/** TREK moved on while upstream did not: push a new revision. */
| { kind: 'push_update'; local: LocalDocument; itemId: number; remoteId: string }
/**
* Where the id is the path: the paired copy is listed under a new one,
* because it was renamed or moved. The pairing follows; nothing else changes.
*/
| { kind: 'relocate'; itemId: number; remote: RemoteDocument }
/** A name changed on one side only. */
| { kind: 'rename_remote'; itemId: number; remoteId: string; name: string }
| { kind: 'rename_local'; itemId: number; fileId: number; name: string }
/** Both sides changed since the agreed state. */
| { kind: 'conflict'; itemId: number; remote: RemoteDocument | null; local: LocalDocument | null }
/** Present upstream last run, absent now. Recorded, never acted on. */
| { kind: 'mark_remote_missing'; itemId: number }
/** Deleted in TREK; what happens upstream is the link's delete policy. */
| { kind: 'local_deleted'; itemId: number; remoteId: string }
/**
* Back out of TREK's trash while the provider copy stayed: the pairing counts
* again. With `missing`, the copy is gone and TREK did not bin it, so the row
* comes back flagged the way a vanished copy is.
*/
| { kind: 'local_restored'; itemId: number; missing?: true }
/** Back out of TREK's trash after TREK binned the copy: unpaired, so it goes up as a new document. */
| { kind: 'detach'; itemId: number }
/** The copy TREK binned is listed again: somebody restored it at the provider. */
| { kind: 'remote_restored'; itemId: number }
/** Nothing to do but the row should stop looking stale. */
| { kind: 'touch'; itemId: number; remote: RemoteDocument | null };
export interface ReconcilePlan {
actions: PlanAction[];
/** True when the listing looked like a mass deletion and was not trusted. */
massDeleteGuardTripped: boolean;
/** Newly gone upstream: vanished since the last run, or found gone on a restore in TREK. */
missingCount: number;
}
export interface ReconcileInput {
items: readonly SyncItemState[];
remote: readonly RemoteDocument[];
local: readonly LocalDocument[];
direction: 'both' | 'pull' | 'push';
/** A truncated listing must never be read as "the rest was deleted". */
remoteTruncated: boolean;
/**
* The provider says its side has not moved since the stored cursor.
*
* Every adapter derives that cursor from the scope's contents (a digest over
* the documents for Paperless, Papra and Synology, the root ETag for WebDAV),
* so the flag means "nothing changed upstream" for all of them. The WebDAV
* adapter acts on it by returning an EMPTY list instead of walking the folder,
* which is only safe if this is read: otherwise a quiet binding looks like a
* folder somebody emptied. It was not read, and the third run of every idle
* Nextcloud or OpenCloud binding tripped the mass-delete guard for good.
*/
remoteUnchanged: boolean;
/**
* What to do when both sides moved since they last agreed.
*
* `manual` parks the row for a person. The other two are the reason this
* field exists: the binding was asked at setup, the answer was stored, and
* nothing ever read it, so every conflict was manual whatever the binding
* said, and a person who had chosen a side still had to pick it again.
*/
conflictPolicy: 'manual' | 'trek_wins' | 'provider_wins';
/** Providers without stable ids need the rename heuristic. */
stableRemoteIds: boolean;
/** How many failures a row gets before it is shelved for a person to look at. */
maxAttempts: number;
/** Injected so the backoff window is testable; defaults to now. */
now?: string;
}
/**
* Decide what has to happen, from a snapshot of both sides.
*
* The three-way comparison is the whole point: `contentSha256` is the state
* both sides last agreed on, so "changed here" and "changed there" are separate
* questions and a change on one side alone is never a conflict. Collapsing that
* into a two-way comparison is what makes naive sync engines either lose edits
* or ping-pong forever.
*/
export function planReconcile(input: ReconcileInput): ReconcilePlan {
const { remote, local, direction, remoteTruncated, stableRemoteIds, maxAttempts, conflictPolicy } = input;
const now = input.now ?? new Date().toISOString().replace('T', ' ').slice(0, 19);
const actions: PlanAction[] = [];
/**
* Rows that must not be tried again on this run.
*
* A provider that refuses a document refuses it every time: a file it reads
* as corrupt, a type it does not accept, a quota that is full. Without this
* the row is re-planned on every pass, so a single bad document turns into an
* upload attempt every cron tick forever; rows were found at 46 attempts
* against a limit of 6, because the limit was written to the database and
* never read back out.
*
* Shelved is not dead: a manual run clears the counter (see resetItemAttempts),
* which is what "waits for a person" means.
*/
const blocked = new Set<number>();
for (const it of input.items) {
if (it.state !== 'error') continue;
if (it.attempts >= maxAttempts) { blocked.add(it.id); continue; }
if (it.nextAttemptAt !== null && it.nextAttemptAt > now) blocked.add(it.id);
}
/**
* Every action goes through here so one rule covers all of them.
*
* No exception for `mark_remote_missing`: a row only counts as vanished if it
* was `synced` last run, and a row is only shelved if it is in `error`, so the
* two can never describe the same row.
*/
const add = (action: PlanAction): void => {
const itemId = 'itemId' in action ? action.itemId : null;
if (itemId !== null && blocked.has(itemId)) return;
actions.push(action);
};
/**
* Both sides changed the bytes. The binding's answer decides, and `manual`
* parks it.
*
* A side that wins takes the whole document, name and bytes together, because
* the two halves of one document coming from different sides is not a state
* anybody asked for. Direction still has the last word: a pull-only binding
* cannot push even when TREK is meant to win, so it parks instead of doing
* the opposite of what was chosen.
*/
const settle = (it: SyncItemState, r: RemoteDocument, l: LocalDocument | null): void => {
// Only the two answers that name a winner act; anything else (`manual`, a
// value from a future version, a column somebody edited by hand) parks the
// row. Overwriting one side is the destructive move, so it needs a yes.
const park = (): void => { add({ kind: 'conflict', itemId: it.id, remote: r, local: l }); };
if (conflictPolicy === 'provider_wins' && direction !== 'push') {
return add({ kind: 'pull_update', remote: r, itemId: it.id });
}
if (conflictPolicy === 'trek_wins' && direction !== 'pull' && l !== null) {
return add({ kind: 'push_update', local: l, itemId: it.id, remoteId: it.remoteId });
}
return park();
};
/**
* Both sides renamed a document whose bytes still agree. Only the name is in
* dispute, so the winning side's name is carried over and nothing is sent.
*
* This went through `settle` once, and `trek_wins` re-uploaded identical
* bytes. Where the provider replaces in place (WebDAV, Synology) the upload
* also kept the provider's name, so the next run read that as a rename made
* upstream and gave TREK's file the name TREK was supposed to win with.
*/
const settleName = (it: SyncItemState, r: RemoteDocument, l: LocalDocument, incoming: string): void => {
if (conflictPolicy === 'provider_wins' && direction !== 'push') {
return add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
}
if (conflictPolicy === 'trek_wins' && direction !== 'pull') {
return add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
}
return add({ kind: 'conflict', itemId: it.id, remote: r, local: l });
};
const remoteById = new Map<string, RemoteDocument>();
for (const r of remote) if (!r.isDeleted) remoteById.set(r.remoteId, r);
const localById = new Map<number, LocalDocument>();
for (const l of local) localById.set(l.fileId, l);
/**
* Where the id is the path, a rename or a move upstream reads as one copy
* gone and another one new.
*
* This used to be patched up in the upstream-only loop with a `touch`, which
* went wrong twice. The pairing had already been counted as vanished by then,
* so the same run flagged it missing (and counted it towards the mass-delete
* guard); and `touch` wrote the provider's new name into the arbiter, so the
* next run read the name TREK still had as a rename made in TREK and renamed
* the provider's copy back.
*
* So the pairing is moved first, before anything else looks at it, `reopen`
* included: onto a new entry that is recognisably the same copy (see
* `sameCopy`). Only the id and the version move. The agreed name stays, so
* the pair loop reads the new name the way it reads any rename, and the rest
* of the plan sees a document that is listed. A rename then ends exactly as
* it does where ids are stable. A rename TREK sent upstream is found again
* the same way: the adapter cannot say what the new path is, so the stored id
* is stale until here.
*
* Only a one-to-one match moves anything. Two copies of the same bytes
* renamed at once cannot be told apart, and pairing them crosswise puts each
* name on the other file; unless one of them kept its name (a move), neither
* is moved, and both are handled like any other gap and new document.
*
* A copy TREK binned itself only gets an entry nobody else fits. Its gap is
* TREK's own doing, and letting it make a tie meant that deleting one of two
* identical files in TREK stopped a rename of the other from being followed,
* which came back as a missing flag plus a second download. It is still
* recognised on its own, as a copy restored at the provider somewhere else.
*/
const followMoves = (list: readonly SyncItemState[]): SyncItemState[] => {
const paired = new Set(list.map((it) => it.remoteId));
const fresh = remote.filter((r) => !r.isDeleted && !paired.has(r.remoteId));
const gone = list.filter((it) => it.remoteId !== null && !remoteById.has(it.remoteId));
const owner = (r: RemoteDocument): SyncItemState | null => {
const fits = gone.filter((o) => sameCopy(o, r));
return soleMatch(fits, (o) => keptName(o, r))
?? soleMatch(fits.filter((o) => o.remoteTrashedAt === null), (o) => keptName(o, r));
};
const moved = new Map<number, RemoteDocument>();
for (const it of gone) {
const r = soleMatch(fresh.filter((d) => sameCopy(it, d)), (d) => keptName(it, d));
if (!r || owner(r) !== it) continue;
// Counted above so a shelved row still makes a tie a tie, but not moved:
// `add` would drop the action while the plan went on with the new id.
if (blocked.has(it.id)) continue;
moved.set(it.id, r);
}
return list.map((it) => {
const r = moved.get(it.id);
if (!r) return it;
add({ kind: 'relocate', itemId: it.id, remote: r });
// Same bytes, so the new version marker is not a change: on a path-keyed
// provider it differs only because the path is part of it.
return { ...it, remoteId: r.remoteId, remoteVersion: r.remoteVersion };
});
};
// Restored rows whose copy turned out to be gone: missing, but not vanished.
let foundGone = 0;
/**
* A file taken back out of TREK's trash.
*
* A deletion TREK has acted on is otherwise left alone for good (see the
* pair loop), so this is the one way out of `local_deleted`, and which way
* depends on the provider copy. If it is listed, the pairing counts again and
* the row is judged like any synced one from here on: an edit made upstream
* in the meantime comes down. If TREK binned it, TREK took it away and the
* person asked for it back, so the pairing is dropped and the file goes up as
* a new document through the ordinary push.
*
* A copy gone by somebody else's hand is flagged as missing rather than
* uploaded over their deletion, and flagged right here rather than counted
* as vanished. Nothing is recorded about a copy while its row sits in
* `local_deleted`, so it may have been gone for weeks, and counting it
* towards the mass-delete guard read a handful of such restores as a listing
* that shrank: the guard tripped, the run was abandoned before the restores
* were written, and the next run found the same plan, for good. Only a
* listing that could have shown the copy may say it is gone, so otherwise
* the row waits; `needsFullListing` keeps that to a single run.
*
* The returned row is what the rest of the plan sees, so it has to match
* what the executor writes for the action added here.
*/
const reopen = (it: SyncItemState): SyncItemState => {
if (!isBackFromTrash(it, localById)) return it;
if (!input.remoteUnchanged && remoteById.has(it.remoteId)) {
add({ kind: 'local_restored', itemId: it.id });
return { ...it, state: 'synced', remoteMissingAt: null, remoteTrashedAt: null };
}
if (it.remoteTrashedAt === null) {
if (input.remoteUnchanged || remoteTruncated) return it;
foundGone += 1;
add({ kind: 'local_restored', itemId: it.id, missing: true });
return { ...it, state: 'remote_missing', remoteMissingAt: now, remoteTrashedAt: null };
}
// A pull-only binding cannot upload, and a gap in a truncated listing says
// nothing about the copy. The row waits for a run that can decide.
if (direction === 'pull' || remoteTruncated) return it;
add({ kind: 'detach', itemId: it.id });
return { ...it, state: 'pending', remoteId: null, remoteVersion: null, remoteTrashedAt: null };
};
// Upstream unchanged means there is no listing to follow anything into.
const followed = stableRemoteIds || input.remoteUnchanged ? input.items : followMoves(input.items);
const items = followed.map(reopen);
const itemsByRemote = new Map<string, SyncItemState>();
const itemsByFile = new Map<number, SyncItemState>();
for (const it of items) {
if (it.remoteId) itemsByRemote.set(it.remoteId, it);
if (it.fileId !== null) itemsByFile.set(it.fileId, it);
}
// Upstream is unchanged: only what happened in TREK can need doing, and the
// remote half of the plan would be reading a list the adapter did not fetch.
if (input.remoteUnchanged) {
for (const it of items) {
// A deletion already acted on stays acted on, as in the pair loop. A copy
// on record as gone is absent from a full listing too, so a full run
// plans nothing for it until it comes back; binning it here would also
// write the gap down as TREK's doing, and a restore would then upload
// what somebody else deleted.
if (!it.remoteId || it.state === 'local_deleted' || it.state === 'remote_missing') continue;
const l = it.fileId !== null ? localById.get(it.fileId) : undefined;
if (l?.deletedAt || lostItsFile(it)) add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
}
if (direction !== 'pull') {
for (const l of local) {
if (l.deletedAt) continue;
const it = itemsByFile.get(l.fileId);
if (it?.remoteId) continue;
add({ kind: 'push', local: l, itemId: it?.id ?? null, remoteId: null });
}
}
return { actions, massDeleteGuardTripped: false, missingCount: 0 };
}
// A listing that lost most of what it had last time is far more likely to be
// a broken mount, a revoked token or a moved folder than a real mass delete.
// A row `reopen` resumed is only `synced` here when its copy is listed, so a
// restore never reads as part of a listing that shrank.
const known = items.filter((i) => i.remoteId && i.state === 'synced');
const vanished = known.filter((i) => !remoteById.has(i.remoteId as string));
const guardTripped =
!remoteTruncated &&
known.length >= MASS_DELETE_MIN_ITEMS &&
vanished.length / known.length > MASS_DELETE_RATIO;
// ── Pairs that exist on both sides ────────────────────────────────────────
for (const it of items) {
if (!it.remoteId) continue;
const r = remoteById.get(it.remoteId);
if (!r) continue;
const l = it.fileId !== null ? localById.get(it.fileId) : undefined;
const remoteChanged = it.remoteVersion !== null && r.remoteVersion !== it.remoteVersion;
// The echo guard: bytes TREK itself pushed come back as a change on the
// provider side. They are only TREK's own write if the hash still matches
// what was pushed, so this compares content, never timestamps: a time
// window would misfire on every clock skew.
const isEcho =
remoteChanged &&
r.contentHash !== null &&
it.pushedSha256 !== null &&
r.contentHash === it.pushedSha256;
const localChanged = !!l && l.sha256 !== null && it.contentSha256 !== null && l.sha256 !== it.contentSha256;
/**
* A deletion is acted on once, under the policy in force at that moment.
*
* This used to be planned again on every run for as long as the file sat
* in TREK's trash. Under `unlink` nothing showed; switching the binding to
* `trash` later then binned every document ever deleted in TREK, months
* back included, and a binding on `trash` from the start asked the
* provider to bin the same copy on every run. A row still here is one
* `reopen` did not take back: its file is still in the trash or purged.
*/
if (it.state === 'local_deleted') {
if (it.remoteTrashedAt !== null) add({ kind: 'remote_restored', itemId: it.id });
continue;
}
if (l?.deletedAt || lostItsFile(it)) {
add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
continue;
}
/**
* Paired upstream, absent in TREK: the download never landed.
*
* A failed pull stores the remote's version alongside the error, so on the
* next run `remoteChanged` is false, there is no local file for the other
* branches to compare against, and the row fell through to `touch`, which
* refreshes it and changes nothing. Nothing anywhere planned a second
* attempt: the document never arrived, no retry ever happened, and pressing
* "Sync now" did not help either, because the plan was the same.
*
* A rejected type and an oversized file are left out on purpose. Those are
* not transport failures; they are answers, and they stay true until the
* document upstream changes, which the `remoteChanged` branch below picks
* up on its own.
*/
if (!l && it.fileId === null && direction !== 'push'
&& it.state !== 'rejected_type' && it.state !== 'too_large') {
add({ kind: 'pull_update', remote: r, itemId: it.id });
continue;
}
if (remoteChanged && !isEcho && localChanged) {
settle(it, r, l ?? null);
continue;
}
if (remoteChanged && !isEcho) {
if (direction === 'push') { add({ kind: 'touch', itemId: it.id, remote: r }); continue; }
add({ kind: 'pull_update', remote: r, itemId: it.id });
continue;
}
if (localChanged && l) {
if (direction === 'pull') { add({ kind: 'touch', itemId: it.id, remote: r }); continue; }
add({ kind: 'push_update', local: l, itemId: it.id, remoteId: it.remoteId });
continue;
}
// Same bytes, different name. Which side renamed is decided by comparing
// each against the name they last agreed on, not by direction. Reading
// `both` as "TREK always wins" silently renamed the provider's copy back
// every time somebody tidied up a folder, which is the opposite of a
// two-way sync.
//
// Compared through the same sanitiser the local name went through, or a
// provider name TREK had to clean up (a slash, a control character, a
// leading dot) reads as "TREK renamed this" on the very next run, and the
// provider's copy gets renamed to the cleaned version, unasked. For the
// same reason TREK takes a new name in its cleaned form, the one a download
// would have given it: a Paperless title with a slash in it used to land
// in the file manager as it was, and the next run read that as a rename
// made in TREK and sent it upstream again.
const incoming = sanitizeIncomingName(r.name);
if (l && incoming !== l.name) {
const agreed = it.remoteName === null ? null : sanitizeIncomingName(it.remoteName);
const providerRenamed = agreed !== null && incoming !== agreed;
const localRenamed = agreed !== null && l.name !== agreed;
if (providerRenamed && !localRenamed && direction !== 'push') {
add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
continue;
}
if (localRenamed && !providerRenamed && direction !== 'pull') {
add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
continue;
}
if (providerRenamed && localRenamed) {
// Both sides renamed. Nothing here can pick the right one on its own,
// and picking wrong loses a name somebody chose, so it follows the
// binding's answer, and goes to a person when there is none.
settleName(it, r, l, incoming);
continue;
}
// No stored name to arbitrate with (a row from before this column, or a
// freshly paired document): follow the binding's direction.
if (direction === 'pull') add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
else if (direction === 'push') add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
else add({ kind: 'touch', itemId: it.id, remote: r });
continue;
}
add({ kind: 'touch', itemId: it.id, remote: r });
}
// ── Upstream only ─────────────────────────────────────────────────────────
if (direction !== 'push') {
for (const r of remote) {
if (r.isDeleted) continue;
// A copy that was only renamed or moved is paired already (followMoves).
if (itemsByRemote.has(r.remoteId)) continue;
add({ kind: 'pull', remote: r, itemId: null });
}
}
// ── TREK only ─────────────────────────────────────────────────────────────
if (direction !== 'pull') {
for (const l of local) {
if (l.deletedAt) continue;
const it = itemsByFile.get(l.fileId);
if (it?.remoteId && remoteById.has(it.remoteId)) continue;
// A row that has a remoteId which is no longer listed is handled below as
// "missing", not re-uploaded. Otherwise a broken listing duplicates the
// entire trip upstream.
if (it?.remoteId) continue;
add({ kind: 'push', local: l, itemId: it?.id ?? null, remoteId: null });
}
}
// ── Vanished upstream ─────────────────────────────────────────────────────
if (!guardTripped && !remoteTruncated) {
for (const it of vanished) {
if (it.remoteMissingAt) continue;
add({ kind: 'mark_remote_missing', itemId: it.id });
}
}
return { actions, massDeleteGuardTripped: guardTripped, missingCount: vanished.length + foundGone };
}
/**
* Whether this run has to list the provider in full, whatever its cursor says.
*
* A file back out of TREK's trash whose copy TREK did not bin can only be
* settled by a listing that shows whether that copy is still there (see
* `reopen`). An unchanged cursor gives the planner nothing to read, and on a
* quiet binding the cursor stays unchanged until somebody touches the folder,
* so the restore would wait that long.
*/
export function needsFullListing(items: readonly SyncItemState[], local: readonly LocalDocument[]): boolean {
const localById = new Map(local.map((l) => [l.fileId, l]));
return items.some((it) => it.remoteTrashedAt === null && isBackFromTrash(it, localById));
}
/** A deletion TREK acted on whose file has since come back out of TREK's trash. */
function isBackFromTrash(it: SyncItemState, localById: ReadonlyMap<number, LocalDocument>): boolean {
if (it.state !== 'local_deleted' || !it.remoteId || it.fileId === null) return false;
const l = localById.get(it.fileId);
return !!l && !l.deletedAt;
}
/**
* Whether a listing entry is recognisably the copy a pairing had, judged by
* content rather than by id.
*
* A hash on both sides settles it either way. Without one (Synology offers
* none), size and modification time together: a rename or a move leaves both
* alone, and an edit does not.
*/
function sameCopy(it: SyncItemState, r: RemoteDocument): boolean {
if (r.contentHash !== null && it.contentSha256 !== null) return r.contentHash === it.contentSha256;
return r.size !== null && r.remoteModifiedAt !== null
&& r.size === it.remoteSize && r.remoteModifiedAt === it.remoteModifiedAt;
}
/** The copy is listed under the name both sides last agreed on: moved, not renamed. */
function keptName(it: SyncItemState, r: RemoteDocument): boolean {
return it.remoteName !== null && sanitizeIncomingName(it.remoteName) === sanitizeIncomingName(r.name);
}
/** The only candidate, or else the only one that kept its name; otherwise none. */
function soleMatch<T>(candidates: T[], keptItsName: (candidate: T) => boolean): T | null {
if (candidates.length === 1) return candidates[0];
const named = candidates.filter(keptItsName);
return named.length === 1 ? named[0] : null;
}
/**
* A file purged from TREK's trash before any run saw it deleted.
*
* `file_id` is ON DELETE SET NULL, so the row outlives the file with nothing
* but a NULL to show for it. Read as a download that never landed, which is
* what happened until now, the document came straight back and the delete
* policy never ran. A first download that failed has no file either, and is
* retried, so `error` and `pending` count only with an agreed hash: that is
* written by a pull or a push that landed and by nothing else, and a failed
* update keeps the file it had. A row shelved in `error` that the person then
* deleted and purged is a deletion, not a document to fetch again.
*/
function lostItsFile(it: SyncItemState): boolean {
if (it.fileId !== null) return false;
if (it.state === 'synced' || it.state === 'conflict' || it.state === 'remote_missing') return true;
return (it.state === 'error' || it.state === 'pending') && it.contentSha256 !== null;
}
/** Backoff lookup that saturates instead of running off the end of the curve. */
export function backoffSeconds(curve: readonly number[], failures: number): number {
if (failures <= 0) return curve[0];
return curve[Math.min(failures - 1, curve.length - 1)];
}
+114
View File
@@ -0,0 +1,114 @@
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
import { logError, logInfo } from '../audit/audit-log.logger';
import { ADDON_IDS } from '../../addons';
import { AddonsService } from '../addons/addons.service';
import { DatabaseService } from '../database/database.service';
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
import { DocSyncConfigService } from './doc-sync-config.service';
import { DocSyncService } from './doc-sync.service';
import {
DEFAULT_POLL_INTERVAL_SECONDS,
MAX_POLL_INTERVAL_SECONDS,
MIN_POLL_INTERVAL_SECONDS,
SETTING_POLL_INTERVAL,
SETTING_SYNC_ENABLED,
} from './doc-sync.constants';
/**
* The poll that carries document sync.
*
* It is the mechanism, not the fallback. Only Paperless lets TREK subscribe to
* changes on its own; Papra's webhook API is closed to API keys, Nextcloud's
* needs admin rights, OpenCloud has no registrable hook, and Synology has
* nothing at all. On top of that no provider here emits an event for every
* change that matters: Paperless has no deletion trigger, and a tag change in
* Papra moves no timestamp. So the run enumerates, and a webhook only ever
* makes it happen sooner.
*
* The addon gate and the interval are re-read per tick rather than at bootstrap
* so an admin toggling either takes effect without a restart. That differs from
* the Dawarich job on purpose: this one is driven by a per-trip binding an
* ordinary user creates, and telling them "restart your server" is not an
* answer.
*
* Which is why the cron itself runs every minute and the tick decides whether it
* is due: baking the interval into the cron expression at bootstrap (as this
* did until the claim above was checked against the code) meant a changed
* interval did nothing until a restart, quietly, while the admin screen said
* otherwise.
*/
@Injectable()
export class DocSyncJob implements OnApplicationBootstrap {
constructor(
private readonly db: DatabaseService,
private readonly sync: DocSyncService,
private readonly config: DocSyncConfigService,
private readonly addons: AddonsService,
private readonly registrar: CronRegistrarService,
) {}
/** When the last pass started; null until the first one, which is never skipped. */
private lastRunAt: number | null = null;
onApplicationBootstrap(): void {
if (!this.registrar.isEnabled()) return;
logInfo(`Document sync: polling every ${this.intervalSeconds()}s`);
this.registrar.register('docsync', '* * * * *', () => this.tick());
}
/**
* Whether enough time has passed for another pass.
*
* The minute the cron wakes up on is not the unit the setting is in, so a
* 90-second interval must not become one minute or two depending on where the
* boundaries fall. Comparing against the last run keeps the setting's own
* resolution; the cost of the extra wake-ups is one read of app_settings.
*/
private isDue(now: number): boolean {
// Null rather than 0: comparing against the epoch means "due" only once the
// clock has passed the interval since 1970, which is true in production and
// false for any test that picks a small timestamp, a difference that would
// have hidden here rather than in the behaviour it is supposed to describe.
if (this.lastRunAt === null) return true;
return now - this.lastRunAt >= this.intervalSeconds() * 1000;
}
private intervalSeconds(): number {
const raw = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', SETTING_POLL_INTERVAL)?.value;
const parsed = Number.parseInt(raw || '', 10);
if (!Number.isFinite(parsed)) return DEFAULT_POLL_INTERVAL_SECONDS;
return Math.min(MAX_POLL_INTERVAL_SECONDS, Math.max(MIN_POLL_INTERVAL_SECONDS, parsed));
}
async tick(): Promise<void> {
try {
if (!this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS)) return;
const killSwitch = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', SETTING_SYNC_ENABLED)?.value;
// Unrecognised values mean ON here because the setting is absent by
// default; only an explicit 'false' stops the sync.
if (killSwitch === 'false') return;
const now = Date.now();
if (!this.isDue(now)) return;
this.lastRunAt = now;
// Cheap, and it catches a binding whose owner left the trip through a
// path that has no hook to attach to: a transfer, a direct DB edit.
const orphaned = this.config.markOrphanedLinks();
if (orphaned > 0) logInfo(`Document sync: ${orphaned} link(s) orphaned, owner no longer on the trip`);
const links = this.sync.dueLinks();
for (const link of links) {
// One failing provider must not stop the others: an unreachable NAS on
// one trip is not a reason to skip a Paperless binding on another.
try {
await this.sync.syncLink(link);
} catch (err: unknown) {
logError(`Document sync: link ${link.id} failed: ${err instanceof Error ? err.message : err}`);
}
}
} catch (err: unknown) {
logError(`Document sync tick failed: ${err instanceof Error ? err.message : err}`);
}
}
}
+122
View File
@@ -0,0 +1,122 @@
import { z } from 'zod';
import {
McpController,
Tool,
type McpContext,
TOOL_ANNOTATIONS_READONLY,
TOOL_ANNOTATIONS_WRITE,
errorResult,
ok,
} from '../../nest-mcp';
import { ADDON_IDS } from '../../addons';
import { noAccess } from '../../mcp/tools/_shared';
import { addonGate } from '../addons/addon-gate';
import { AddonsService } from '../addons/addons.service';
import { FilesService } from '../files/files.service';
import { DocSyncConfigService } from './doc-sync-config.service';
import { DocSyncService } from './doc-sync.service';
import { PROVIDER_DISABLED } from './doc-sync.constants';
const documentsAddonOn = addonGate(ADDON_IDS.DOCUMENTS);
/**
* The MCP half of document sync, mirroring /api/trips/:tripId/docsync.
*
* Read and trigger only. There is no tool that creates a connection, and that
* is deliberate rather than unfinished: setting one up means handing over a
* credential that usually reaches a person's entire document archive, and an
* assistant is the wrong place to collect one. The same reason the REST side
* restricts those routes to the trip owner applies twice over here.
*
* `sync_trip_documents` is a write because it moves files in both directions,
* even though it creates nothing itself.
*/
@McpController()
export class DocSyncMcp {
constructor(
private readonly config: DocSyncConfigService,
private readonly sync: DocSyncService,
private readonly files: FilesService,
readonly addons: AddonsService,
) {}
@Tool({
name: 'get_trip_document_sync',
description:
'Show whether this trip\'s documents are synced with an external document store (Paperless-ngx, Papra, Nextcloud, OpenCloud or a Synology NAS), which folder or tag they are bound to, when the last run happened, and how many documents are waiting, in conflict or missing at the provider. Use this before telling someone where their documents live.',
inputSchema: {
tripId: z.number().int().positive(),
},
annotations: TOOL_ANNOTATIONS_READONLY,
access: { group: 'files', mode: 'read' },
when: documentsAddonOn,
})
getTripDocumentSync({ tripId }: { tripId: number }, ctx: McpContext) {
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
return ok(this.sync.status(tripId));
}
@Tool({
name: 'list_trip_document_sync_issues',
description:
'List the documents that need a person to look at them: conflicts where both copies changed, documents the provider refused because of their type or size, and documents that disappeared at the provider. Returns an empty list when everything is in step.',
inputSchema: {
tripId: z.number().int().positive(),
},
annotations: TOOL_ANNOTATIONS_READONLY,
access: { group: 'files', mode: 'read' },
when: documentsAddonOn,
})
listIssues({ tripId }: { tripId: number }, ctx: McpContext) {
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
const links = this.config.listLinks(tripId);
if (links.length === 0) return ok({ configured: false, issues: [] });
return ok({ configured: true, issues: this.sync.issues(tripId) });
}
@Tool({
name: 'sync_trip_documents',
description:
'Run the document sync for this trip now instead of waiting for the next scheduled check. Reports how many documents were pulled in from the provider, pushed out to it, and how many are in conflict. Safe to call repeatedly: a run that is already in progress is not started twice.',
inputSchema: {
tripId: z.number().int().positive(),
full: z
.boolean()
.optional()
.default(false)
.describe('Ignore the change marker from last time and compare both sides in full'),
},
annotations: TOOL_ANNOTATIONS_WRITE,
access: { group: 'files', mode: 'write' },
when: documentsAddonOn,
})
async syncNow({ tripId, full }: { tripId: number; full?: boolean }, ctx: McpContext) {
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
const links = this.config.listLinks(tripId);
if (links.length === 0) {
return errorResult('This trip is not connected to a document store. Connect one in the trip\'s file manager first.');
}
const results = [];
for (const link of links) {
// Refused as the REST route refuses it, before the shelved rows below are
// touched: a binding an admin switched off stays exactly as it was, so it
// resumes where it stopped once the provider is back on.
if (this.sync.isSwitchedOff(link)) {
results.push({ linkId: link.id, provider: link.provider_id, state: 'disabled', errorCode: PROVIDER_DISABLED });
continue;
}
// Asking for a run by hand means "try again", including the documents
// that were shelved after too many failures. The REST route does the
// same thing before its run; a tool that skipped it would answer "in
// sync" while leaving them shelved.
this.sync.retryShelvedItems(link.id);
results.push({ linkId: link.id, provider: link.provider_id, ...(await this.sync.syncLink(link, { full: full === true })) });
}
if (results.every((r) => r.errorCode === PROVIDER_DISABLED)) {
return errorResult(
`${PROVIDER_DISABLED}: an administrator has switched off the document store this trip is bound to, so nothing was synced. The binding and its documents are kept, and syncing resumes once the store is switched back on.`,
);
}
return ok({ runs: results });
}
}
@@ -0,0 +1,95 @@
import { Module } from '@nestjs/common';
import { AddonsModule } from '../addons/addons.module';
import { AuthModule } from '../auth/auth.module';
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
import { DatabaseModule } from '../database/database.module';
import { FilesModule } from '../files/files.module';
import { AllowedFileTypesModule } from '../files/allowed-file-types.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { RealtimeModule } from '../realtime/realtime.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { StorageModule } from '../storage/storage.module';
import { DocSyncConfigService } from './doc-sync-config.service';
import { DocSyncController } from './doc-sync.controller';
import { DocSyncWebhookController } from './doc-sync-webhook.controller';
import { DocSyncJob } from './doc-sync.job';
import { DocSyncMcp } from './doc-sync.mcp';
import { DocSyncService } from './doc-sync.service';
import { DOCUMENT_PROVIDERS, type DocumentProvider } from './document-provider';
import { DocumentProviderRegistry } from './document-provider.registry';
import { PaperlessDocumentProvider } from './providers/paperless.provider';
import { PapraDocumentProvider } from './providers/papra.provider';
import { SynologyDriveDocumentProvider } from './providers/synology-drive.provider';
import { NextcloudDocumentProvider, OpencloudDocumentProvider } from './providers/webdav.provider';
import { PaperlessClient } from './providers/paperless.client';
import { PapraClient } from './providers/papra.client';
import { SynologyDriveClient } from './providers/synology-drive.client';
import { WebdavClient } from './providers/webdav.client';
/**
* Document sync: one provider connection per trip, both directions.
*
* Its own domain rather than an extension of `files/`, because that domain's
* coverage ratchet sits at 97/95 and would now be measuring four HTTP clients
* and a cron job; and not under `integrations/`, whose ratchet is 78/68 and
* would be dragged down by new code landing there. The seam is clean anyway:
* this module consumes FilesService and StorageService and nothing consumes it.
*
* Nextcloud and OpenCloud are two registrations of one WebDAV adapter. They are
* separate ids because the admin toggles, the credential fields and the scope
* concept differ (a folder with an `oc:fileid` versus a space with a
* `driveId`), but the protocol underneath is the same and so is the code.
*/
@Module({
imports: [
DatabaseModule,
StorageModule,
FilesModule,
AllowedFileTypesModule,
PermissionsModule,
RealtimeModule,
SchedulingModule,
AddonsModule,
AuthModule,
McpSharedModule,
],
controllers: [DocSyncController, DocSyncWebhookController],
providers: [
DocSyncConfigService,
DocSyncService,
DocSyncJob,
DocSyncMcp,
DocumentProviderRegistry,
// The HTTP clients are providers of their own: each is the ONLY file that
// talks to its instance, which makes it the one thing a test has to replace
// to be sure a suite never opens a socket.
PaperlessClient,
PapraClient,
SynologyDriveClient,
WebdavClient,
PaperlessDocumentProvider,
PapraDocumentProvider,
NextcloudDocumentProvider,
OpencloudDocumentProvider,
SynologyDriveDocumentProvider,
{
provide: DOCUMENT_PROVIDERS,
useFactory: (
paperless: PaperlessDocumentProvider,
papra: PapraDocumentProvider,
nextcloud: NextcloudDocumentProvider,
opencloud: OpencloudDocumentProvider,
synology: SynologyDriveDocumentProvider,
): readonly DocumentProvider[] => [paperless, papra, nextcloud, opencloud, synology],
inject: [
PaperlessDocumentProvider,
PapraDocumentProvider,
NextcloudDocumentProvider,
OpencloudDocumentProvider,
SynologyDriveDocumentProvider,
],
},
],
exports: [DocSyncConfigService, DocSyncService],
})
export class DocSyncModule {}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,28 @@
import { Inject, Injectable } from '@nestjs/common';
import { DOCUMENT_PROVIDERS, type DocumentProvider } from './document-provider';
/**
* The one dispatch site for document providers.
*
* Same shape as PhotoProviderRegistry, and for the same reason: adding a
* backend is registering an adapter in the module, not editing a switch that
* exists in four places. Whether a registered provider is switched ON is a
* different question, answered by the `document_providers` table plus the
* `documents` addon. This only knows what the server can talk to at all.
*/
@Injectable()
export class DocumentProviderRegistry {
private readonly byId: Map<string, DocumentProvider>;
constructor(@Inject(DOCUMENT_PROVIDERS) providers: readonly DocumentProvider[]) {
this.byId = new Map(providers.map((p) => [p.id, p]));
}
get(id: string | null | undefined): DocumentProvider | undefined {
return id ? this.byId.get(id) : undefined;
}
ids(): string[] {
return [...this.byId.keys()];
}
}
@@ -0,0 +1,264 @@
import type { Readable } from 'node:stream';
import type { DocsyncErrorCode } from '@trek/shared';
/**
* The adapter seam for document providers.
*
* Shaped after `PhotoProvider` (memories/photo-provider.ts): interface, symbol
* token, registry. One difference is deliberate: that interface only covers
* handing out bytes, and everything else (settings, test, search, album list)
* is dispatched through hand-written `if (provider === 'immich')` branches in
* the service, the MCP tools and two separate controllers. With two providers
* that is untidy; with five it would be five parallel controllers and five
* branches per tool. So every operation the sync core needs lives here, and the
* core never learns which provider it is talking to.
*
* Capability differences are answered by data (`DocumentProviderCapabilities`),
* never by type-switching: the core asks what a provider can do and plans
* accordingly. Synology FileStation has no change feed and no stable file id,
* so the core must be correct with nothing but a full enumeration, and because
* the weakest provider sets the model, every provider is driven the same way.
* A webhook is only ever "look now", never a source of truth.
*/
/** Uniform result shape, mirroring `ServiceResult` in memories.helpers.ts. */
export type DocResult<T> =
| { success: true; data: T }
| { success: false; error: { code: DocsyncErrorCode; detail?: string; status?: number } };
export type DocFailure = { success: false; error: { code: DocsyncErrorCode; detail?: string; status?: number } };
export const docOk = <T>(data: T): DocResult<T> => ({ success: true, data });
export const docFail = <T>(
code: DocsyncErrorCode,
detail?: string,
status?: number,
): DocResult<T> => ({ success: false, error: { code, detail, status } });
/**
* Narrow a result to its failure half.
*
* `if (!res.success)` does NOT narrow here: the server workspace compiles with
* `strict: false`, and without `strictNullChecks` a boolean discriminant stops
* discriminating. Testing for the property is what the memories domain already
* does (`if ('error' in result)`), and wrapping it in a predicate keeps every
* call site reading as an intention rather than as a workaround.
*/
export function docFailed<T>(result: DocResult<T>): result is DocFailure {
return 'error' in result;
}
/**
* Whose credentials, against which instance.
*
* A ref object rather than positional arguments, for the reason written into
* `PhotoAssetRef`: two providers took the same three values in a different
* order, and swapping them compiled cleanly and served another user's data.
*/
export interface DocumentConnectionRef {
connectionId: number;
/**
* When the connection's row was created, or '' for form values that were
* never saved. The id alone does not name one row for good: a backup
* restored in place rolls the id sequence back, and the id comes round
* again on the next connection saved, on any trip. The two together do.
*/
createdAt: string;
/**
* The user whose credentials these are: the trip admin who set the binding
* up, not whoever is asking. The core passes both around separately and never
* derives one from the other.
*/
ownerId: number;
baseUrl: string;
/**
* Decrypted `secret = 1` fields, plus whatever the provider earned and kept
* through `saveSecret`. Never logged, never returned to a client.
*/
secrets: Readonly<Record<string, string>>;
/** Non-secret fields: username, organization_id, base_path, root_tag, … */
settings: Readonly<Record<string, string>>;
allowInsecureTls: boolean;
/**
* Keep a secret the provider earned itself (DSM's device token) with this
* connection, or drop it with null. `secrets` is the snapshot this ref was
* made from and does not change when this writes.
*
* Only a saved connection has one. A probe of form values may use what is
* stored but never changes it: the values on screen may be for another
* account, or never be saved at all.
*/
saveSecret?: (key: string, value: string | null) => void;
}
/** The container at the provider that one trip is bound to. */
export interface DocumentScopeRef {
linkId: number;
tripId: number;
/** Opaque, normalised, never empty. See `docsyncScopeKeySchema`. */
scopeKey: string;
remoteRootId: string | null;
remoteRootPath: string | null;
/**
* Whatever the adapter stored last run. A WebDAV adapter puts the root ETag
* here and can skip the whole walk when it is unchanged; Paperless stores the
* highest `modified` it saw. The core treats it as a black box and hands it
* back untouched.
*/
cursor: string | null;
}
export interface RemoteDocument {
/** Stable where the provider has one; a normalised path where it does not. */
remoteId: string;
name: string;
size: number | null;
mimeType: string | null;
/**
* Opaque version marker. Different means changed. That is all the core may
* conclude from it. It is not a content hash, not comparable across
* instances, and not ordered.
*/
remoteVersion: string;
/** sha256 hex when the listing carries one, otherwise null. */
contentHash: string | null;
remoteModifiedAt: string | null;
/** Provider-side tombstone, where one exists (Synology Drive `removed`). */
isDeleted: boolean;
}
export interface DocumentScopeOption {
scopeKey: string;
label: string;
remoteRootId: string | null;
remoteRootPath: string | null;
}
export interface DocumentProviderCapabilities {
/**
* `webhook-self-registered`: TREK can create the subscription itself
* (Paperless workflows). `webhook-manual`: the API exists but is closed to
* API keys, so a human has to paste a URL (Papra, and Nextcloud unless the
* account is an admin). `none`: polling only (Synology).
*/
push: 'webhook-self-registered' | 'webhook-manual' | 'none';
/** False means identity is the path, so rename looks like delete + create. */
stableId: boolean;
/** A delete lands in a recycle bin the user can recover from. */
remoteTrash: boolean;
/** A new revision can replace the bytes under the same remoteId. */
replaceInPlace: boolean;
/** The listing already carries a content hash, so no extra round trip. */
contentHashInListing: boolean;
maxUploadBytes: number | null;
/** null means anything; otherwise the provider's own MIME allowlist. */
acceptedMimeTypes: readonly string[] | null;
/** The adapter can create a new scope container, not just pick one. */
canCreateScope: boolean;
}
export interface PushRequest {
/** Bytes, streamed straight out of `StorageService.getStream('files', …)`. */
body: Readable;
fileName: string;
mimeType: string;
size: number;
/** sha256 of those bytes; adapters that can pin it upstream should. */
sha256: string;
/**
* Unix seconds. WebDAV sends `X-OC-MTime`, Synology multiplies by 1000.
* Without it every file TREK uploads comes back looking freshly changed,
* which is the classic sync loop.
*/
mtimeSeconds: number;
/** Present means replace that document rather than create a new one. */
remoteId?: string;
/** Optimistic concurrency (WebDAV `If-Match`); ignored where unsupported. */
expectedRemoteVersion?: string;
/**
* The anchors that survive a human renaming or re-tagging things upstream.
* Written into a custom field, a property or a tag where the provider has
* somewhere to put them.
*/
trekDocUid: string;
trekTripUid: string;
}
export interface PushResult {
remoteId: string;
remoteVersion: string;
remoteModifiedAt: string | null;
/**
* True when the provider matched existing bytes instead of storing new ones
* (Papra answers 409 on a duplicate hash, and restores a trashed twin under
* its ORIGINAL id). The core must not book that as a fresh document.
*/
deduplicated: boolean;
}
export interface FetchResult {
body: Readable;
size: number | null;
mimeType: string | null;
remoteVersion: string;
}
export interface DocumentProvider {
readonly id: string;
capabilities(conn: DocumentConnectionRef): DocumentProviderCapabilities;
/**
* Verify form values before anything is stored, and report what the account
* can actually do. Always resolves: a failed probe is data, not an
* exception, because the settings UI shows it inline (the photo providers
* pin that with a test marked CRITICAL in their e2e suite).
*/
probe(conn: DocumentConnectionRef): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>>;
/** Offer the containers a trip could be bound to: folders, tags, spaces. */
listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>>;
/** Create one, where the provider has the concept. */
createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>>;
/** Confirm the bound scope still exists, so `scope_lost` is detectable. */
resolveScope(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<DocumentScopeOption>>;
/**
* Enumerate everything currently in the scope, plus the cursor to hand back
* next time.
*
* Full enumeration rather than a delta, deliberately: a changed-since query
* cannot see a deletion, several of these providers bump no timestamp when a
* tag changes, and Paperless's bulk edit changes documents without touching
* `modified` at all. `cursorUnchanged` lets an adapter short-circuit the walk
* (the WebDAV root ETag) while keeping the core's model identical.
*/
list(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<{
documents: RemoteDocument[];
cursor: string | null;
cursorUnchanged: boolean;
/** True when the listing hit a page cap and is therefore not the whole truth. */
truncated: boolean;
}>>;
fetch(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): Promise<DocResult<FetchResult>>;
push(conn: DocumentConnectionRef, scope: DocumentScopeRef, req: PushRequest): Promise<DocResult<PushResult>>;
rename(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string, name: string): Promise<DocResult<{ remoteVersion: string }>>;
/**
* Move to the provider's recycle bin. Never a hard delete: Papra and
* Paperless only expose permanent deletion to a session, not to a token, and
* on the others an accidental propagation would be unrecoverable.
*/
trash(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): Promise<DocResult<void>>;
/** Subscribe to change notifications, where TREK is allowed to do it itself. */
registerWebhook?(conn: DocumentConnectionRef, scope: DocumentScopeRef, callbackUrl: string, secret: string): Promise<DocResult<{ subscriptionId: string }>>;
unregisterWebhook?(conn: DocumentConnectionRef, subscriptionId: string): Promise<DocResult<void>>;
}
export const DOCUMENT_PROVIDERS = Symbol('DOCUMENT_PROVIDERS');
@@ -0,0 +1,894 @@
import { randomUUID } from 'node:crypto';
import type { DocsyncErrorCode } from '@trek/shared';
import { readCappedJson, readCappedText, discardBody } from '../../../utils/cappedFetch';
import {
providerFetch,
statusErrorCode,
type TransportFailure,
type TransportFailureCode,
} from './provider-http';
import {
PROVIDER_JSON_MAX_BYTES,
PROVIDER_MAX_PAGES,
PROVIDER_TIMEOUT_MS,
PROVIDER_TRANSFER_TIMEOUT_MS,
} from '../doc-sync.constants';
/**
* Thin HTTP client for the Paperless-ngx REST API.
* This is the ONLY place that talks to a trip's Paperless instance.
*
* Measured against Paperless-ngx 3.1.3 / API version 10, not against its
* documentation. The two disagree in the places that matter here:
*
* - **A document carries no top-level `checksum` at 3.x.** The hash lives in
* `versions[]`, one entry per uploaded revision, and it is a **sha256** of
* the original file (verified: an uploaded file's sha256 came back verbatim).
* `versions` arrives newest first with the ORIGINAL marked `is_root: true`,
* so the entry describing the current bytes is the newest `added`, not the
* root one.
* - `GET /api/documents/{id}/download/` hands out the **archived**, OCR'd PDF,
* whose bytes are not the ones that were uploaded (398 B in, 8483 B out).
* Only `?original=true` round-trips, and only it matches the checksum.
* - **An upload answers with a task UUID, not a document.** The consume job is
* asynchronous (~3 s for a one-page PDF with OCR), and `GET /api/tasks/`
* answers a PAGINATED object at 3.x where older builds answered a bare
* array. `status` is lower case (`success`, `failure`) where older builds
* shouted it, so both spellings are compared case-insensitively.
* - **`POST /api/documents/{id}/update_version/` keeps the document id**, and
* with it the tags and custom fields, but the task it returns reports a
* DIFFERENT, internal document id (the version row), which answers 404 on
* `/api/documents/{id}/`. Callers must keep the id they updated.
* - **Duplicate bytes are not refused.** Uploading the same file twice yields
* two documents with the same checksum and an empty `duplicate_documents`,
* so there is no upstream deduplication to report. `?checksum__iexact=` does
* filter, which is what makes a lost upload recoverable instead of doubled.
* - **Unknown query parameters are ignored, not rejected.** A filter that does
* not exist silently widens the result, so every filter used here was proven
* with a matching and a non-matching value.
* - The MIME allow-list is enforced by the server: an unsupported type comes
* back as `400 {"document": ["File type … not supported"]}` before any task
* is created. Office formats pass only on an instance with Tika configured.
*
* The API version is pinned in the Accept header. Paperless moves its default
* with the server version, so an unpinned client changes shape under a routine
* `docker pull`. That is exactly how the `checksum` field moved out of the
* document serializer.
*/
/** The version this client's response handling was written against. */
const API_ACCEPT = 'application/json; version=10';
/** Tags, documents and tasks are small rows; a page of them is never megabytes. */
const MAX_JSON_BYTES = PROVIDER_JSON_MAX_BYTES;
/** Documents per page. Paperless accepts more, but the response grows with it. */
export const DOCUMENT_PAGE_SIZE = 100;
export const TAG_PAGE_SIZE = 100;
/** How often the consume task is polled, and for how long before giving up. */
export const TASK_POLL_INTERVAL_MS = 1500;
export const TASK_POLL_TIMEOUT_MS = PROVIDER_TRANSFER_TIMEOUT_MS;
export interface PaperlessCreds {
/** Instance origin. The client appends `/api` itself. */
baseUrl: string;
/** The API token from the user's Paperless profile. Never logged. */
token: string;
allowInsecureTls: boolean;
}
/**
* A failed Paperless call, carrying the reason as one of TREK's codes.
*
* The upstream text never reaches a user: it is English, it sometimes quotes
* the URL back, and behind an auth proxy it is an HTML login page. `detail`
* keeps it for the self-hoster reading their own log.
*/
export class PaperlessError extends Error {
readonly code: DocsyncErrorCode;
readonly status?: number;
readonly detail?: string;
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string) {
super(message);
this.name = 'PaperlessError';
this.code = code;
this.status = status;
this.detail = detail;
}
}
export interface PaperlessTag {
id: number;
name: string;
/** Absent on the POST response, present in listings. */
documentCount: number | null;
}
export interface PaperlessCustomField {
id: number;
name: string;
dataType: string | null;
}
/** One entry of a document's `versions`, i.e. one uploaded revision. */
export interface PaperlessVersion {
id: number;
/** sha256 hex of that revision's original file. */
checksum: string | null;
added: string | null;
isRoot: boolean;
}
export interface PaperlessDocument {
id: number;
title: string;
/** The name the file was uploaded under; null on documents created elsewhere. */
originalFileName: string | null;
mimeType: string | null;
/** The opaque version marker. Bumped by edits, uploads and bulk edits alike. */
modified: string | null;
added: string | null;
/** Non-null only in `/api/trash/`; a listed document is never trashed. */
deletedAt: string | null;
tagIds: number[];
versions: PaperlessVersion[];
}
export interface PaperlessTask {
taskId: string;
/** Lower case at 3.x, upper case on older builds. Compared case-insensitively. */
status: string;
documentIds: number[];
errorMessage: string | null;
}
export interface PaperlessProfile {
username: string;
version: string | null;
/** Django permission codenames the token's user holds, e.g. `add_workflow`. */
permissions: string[];
}
export interface PaperlessUploadFile {
fileName: string;
mimeType: string;
bytes: Buffer;
}
export interface PaperlessWorkflowSpec {
name: string;
/** Only documents carrying this tag fire the webhook. */
tagId: number;
callbackUrl: string;
/** Sent as a request header, so it never lands in an access log. */
secretHeader: { name: string; value: string };
/** Flat string map; Paperless sends it as the JSON body. */
params: Record<string, string>;
}
/**
* Paperless's own ceiling for a webhook URL (`max_length` on the field). A
* longer callback is rejected by the serializer with a 400 that reads like a
* generic validation failure, so it is worth catching before the round trip.
*/
export const WEBHOOK_URL_MAX_LENGTH = 256;
/**
* Workflow trigger types, as `OPTIONS /api/workflows/` enumerates them. There
* is deliberately no deletion trigger upstream: a document moved to the trash
* fires nothing, which is why a webhook can only ever mean "look now".
*/
export const TRIGGER_DOCUMENT_ADDED = 2;
export const TRIGGER_DOCUMENT_UPDATED = 3;
/** Action type 4 is "Webhook". */
export const ACTION_WEBHOOK = 4;
/**
* Consume folder, API upload, mail fetch, web UI: all four, or a document
* dropped into Paperless's own interface would never reach TREK.
*/
const TRIGGER_SOURCES = [1, 2, 3, 4];
/**
* `https://host/paperless/` → `https://host/paperless/api`, tolerating what
* people paste: a trailing slash, or the `/api` they copied out of the browser.
*/
export function apiBase(baseUrl: string): string {
const origin = baseUrl
.trim()
.replace(/(?<!\/)\/+$/, '')
.replace(/\/api$/i, '');
return `${origin}/api`;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function asString(value: unknown): string | null {
return typeof value === 'string' ? value : null;
}
function asNumber(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
/**
* The rows of a paginated answer, or of a bare array.
*
* Both shapes are live: `/api/tasks/` answers a bare array on older builds and
* a paginated object at 3.x, and reading only one of them would make the upload
* path fail on whichever half of the installed base it guessed wrong.
*/
function pageRows(data: unknown): { rows: unknown[]; hasNext: boolean; count: number | null } {
if (Array.isArray(data)) return { rows: data, hasNext: false, count: data.length };
if (isRecord(data) && Array.isArray(data.results)) {
return {
rows: data.results,
hasNext: typeof data.next === 'string' && data.next.length > 0,
count: asNumber(data.count),
};
}
throw new PaperlessError('provider_error', 'Paperless answered a listing in an unknown shape');
}
function parseTag(row: unknown): PaperlessTag | null {
if (!isRecord(row)) return null;
const id = asNumber(row.id);
const name = asString(row.name);
if (id === null || name === null) return null;
return { id, name, documentCount: asNumber(row.document_count) };
}
function parseCustomField(row: unknown): PaperlessCustomField | null {
if (!isRecord(row)) return null;
const id = asNumber(row.id);
const name = asString(row.name);
if (id === null || name === null) return null;
return { id, name, dataType: asString(row.data_type) };
}
function parseVersions(value: unknown): PaperlessVersion[] {
if (!Array.isArray(value)) return [];
const versions: PaperlessVersion[] = [];
for (const row of value) {
if (!isRecord(row)) continue;
const id = asNumber(row.id);
if (id === null) continue;
versions.push({
id,
checksum: asString(row.checksum),
added: asString(row.added),
isRoot: row.is_root === true,
});
}
return versions;
}
function parseDocument(row: unknown): PaperlessDocument | null {
if (!isRecord(row)) return null;
const id = asNumber(row.id);
if (id === null) return null;
const tagIds = Array.isArray(row.tags)
? row.tags.filter((tag): tag is number => typeof tag === 'number')
: [];
return {
id,
title: asString(row.title) ?? '',
originalFileName: asString(row.original_file_name),
mimeType: asString(row.mime_type),
modified: asString(row.modified),
added: asString(row.added),
deletedAt: asString(row.deleted_at),
tagIds,
versions: parseVersions(row.versions),
};
}
function parseTask(row: unknown): PaperlessTask | null {
if (!isRecord(row)) return null;
const taskId = asString(row.task_id);
if (taskId === null) return null;
const related = Array.isArray(row.related_document_ids)
? row.related_document_ids.filter((value): value is number => typeof value === 'number')
: [];
const fromResult = isRecord(row.result_data) ? asNumber(row.result_data.document_id) : null;
const documentIds = related.length > 0 ? related : fromResult === null ? [] : [fromResult];
return {
taskId,
status: asString(row.status) ?? '',
documentIds,
errorMessage: isRecord(row.result_data) ? asString(row.result_data.error_message) : null,
};
}
/**
* The revision whose bytes `?original=true` currently serves.
*
* `versions` arrives newest first at 3.1.3, but that is an ordering nobody
* promised, so the newest `added` decides and the array order is only the
* tie-breaker. The root entry is explicitly NOT it: after an in-place replace
* the root is the file that was replaced.
*/
export function currentVersion(doc: PaperlessDocument): PaperlessVersion | null {
let best: PaperlessVersion | null = null;
let bestAt = Number.NEGATIVE_INFINITY;
for (const version of doc.versions) {
const parsed = version.added === null ? Number.NaN : Date.parse(version.added);
const rank = Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
if (best === null || rank > bestAt) {
best = version;
bestAt = rank;
}
}
return best;
}
/** sha256 hex, or null. A hash of another shape must not be sold as one. */
export function sha256OrNull(value: string | null): string | null {
return value !== null && /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
}
const TRANSPORT_MESSAGES: Record<TransportFailureCode, string> = {
ssrf_blocked: 'The address is blocked',
timeout: 'Paperless did not answer in time',
tls_untrusted: 'The certificate of this Paperless instance is not trusted',
unreachable: 'Could not reach Paperless',
};
function transportError(failure: TransportFailure): PaperlessError {
return new PaperlessError(failure.code, TRANSPORT_MESSAGES[failure.code], undefined, failure.detail);
}
/**
* The code for an HTTP status, refined by the body where the status alone is
* ambiguous. Paperless answers 400 both for "I do not take this file type"
* (a per-document verdict the user can act on) and for a malformed request,
* and the difference is only in the text.
*/
function classify(status: number, body: string): DocsyncErrorCode {
if (status === 400) {
if (/not supported/i.test(body)) return 'unsupported_type';
if (/unique constraint|already exists/i.test(body)) return 'conflict';
return 'provider_error';
}
return statusErrorCode(status);
}
/** Trim an upstream message to something a log line can hold. */
function shorten(text: string): string {
const flat = text.replace(/\s+/g, ' ').trim();
return flat.length > 400 ? `${flat.slice(0, 400)}…` : flat;
}
interface RequestOptions {
method?: string;
query?: Record<string, string | undefined>;
json?: unknown;
body?: Buffer;
contentType?: string;
timeoutMs?: number;
/** Streaming responses are handed back unread; everything else is capped. */
raw?: boolean;
}
interface MultipartField {
name: string;
value: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Everything a header value may not contain, replaced rather than rejected. */
function headerSafe(value: string): string {
return value.replace(/[\r\n"\\]/g, '_');
}
/**
* A multipart/form-data body, built by hand.
*
* `FormData` plus `Blob` would be shorter but hands undici a body without a
* length; a Buffer keeps Content-Length honest, which is what lets a reverse
* proxy in front of Paperless enforce its own upload limit instead of buffering
* an unbounded stream. Field and file names are stripped of quotes and line
* breaks first: a document called `a".pdf` would otherwise end the
* Content-Disposition header early and rewrite the rest of the request.
*/
export function buildMultipart(
fields: MultipartField[],
file: PaperlessUploadFile,
): { body: Buffer; contentType: string } {
const boundary = `----TrekDocSync${randomUUID().replace(/-/g, '')}`;
const parts: Buffer[] = [];
for (const field of fields) {
parts.push(
Buffer.from(
`--${boundary}\r\nContent-Disposition: form-data; name="${headerSafe(field.name)}"\r\n\r\n${field.value}\r\n`,
'utf8',
),
);
}
parts.push(
Buffer.from(
`--${boundary}\r\nContent-Disposition: form-data; name="document"; filename="${headerSafe(file.fileName)}"\r\n` +
`Content-Type: ${headerSafe(file.mimeType)}\r\n\r\n`,
'utf8',
),
);
parts.push(file.bytes);
parts.push(Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8'));
return {
body: Buffer.concat(parts),
contentType: `multipart/form-data; boundary=${boundary}`,
};
}
export class PaperlessClient {
/**
* One authenticated request. Every call in this file goes through it, so the
* token header, the pinned API version, the timeout and the TLS decision are
* made in exactly one place.
*/
private async request(
creds: PaperlessCreds,
path: string,
options: RequestOptions = {},
): Promise<Response> {
const url = new URL(apiBase(creds.baseUrl) + path);
for (const [key, value] of Object.entries(options.query ?? {})) {
if (value !== undefined) url.searchParams.set(key, value);
}
const headers: Record<string, string> = {
Authorization: `Token ${creds.token}`,
Accept: options.raw === true ? '*/*' : API_ACCEPT,
};
let body: Buffer | string | undefined;
if (options.json !== undefined) {
body = JSON.stringify(options.json);
headers['Content-Type'] = 'application/json';
} else if (options.body !== undefined) {
body = options.body;
headers['Content-Type'] = options.contentType ?? 'application/octet-stream';
// Content-Length is deliberately NOT set: undici derives it from a buffer
// body and rejects the request outright (`UND_ERR_INVALID_ARG: invalid
// content-length header`) when the header is also present, which surfaces
// as a bare `TypeError: fetch failed` two layers up.
}
const response = await providerFetch(
url.toString(),
{ method: options.method ?? 'GET', headers, body },
{
timeoutMs: options.timeoutMs ?? PROVIDER_TIMEOUT_MS,
allowInsecureTls: creds.allowInsecureTls,
onTransportFailure: transportError,
},
);
if (!response.ok) {
// The body is what separates "wrong file type" from "wrong request", so it
// is read, through the cap, because a proxy's error page is HTML of
// unbounded length.
const { text } = await readCappedText(response, MAX_JSON_BYTES);
throw new PaperlessError(
classify(response.status, text),
`Paperless answered HTTP ${response.status}`,
response.status,
shorten(text),
);
}
return response;
}
private async requestJson<T>(
creds: PaperlessCreds,
path: string,
options: RequestOptions = {},
): Promise<T> {
const response = await this.request(creds, path, options);
const data = await readCappedJson<T>(response, MAX_JSON_BYTES);
if (data === undefined) {
throw new PaperlessError(
'provider_error',
'Paperless returned something that is not JSON',
response.status,
'Check that the URL points at Paperless itself and is reachable without a separate login.',
);
}
return data;
}
/**
* Who the token belongs to, which build answers, and what that user may do.
*
* `/api/ui_settings/` rather than `/api/profile/`: profile echoes the token
* back in its own body, and nothing here should carry a credential it does not
* need. ui_settings also carries the permission list, which is the only way to
* know in advance whether this token may create the webhook workflow.
*/
async probe(creds: PaperlessCreds): Promise<PaperlessProfile> {
const data = await this.requestJson<unknown>(creds, '/ui_settings/');
if (!isRecord(data)) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown profile shape');
}
const user = isRecord(data.user) ? data.user : {};
const settings = isRecord(data.settings) ? data.settings : {};
return {
username: asString(user.username) ?? '',
version: asString(settings.version),
permissions: Array.isArray(data.permissions)
? data.permissions.filter((permission): permission is string => typeof permission === 'string')
: [],
};
}
/** Every tag, or those whose name contains `query`. */
async listTags(creds: PaperlessCreds, query?: string): Promise<PaperlessTag[]> {
const tags: PaperlessTag[] = [];
for (let page = 1; page <= PROVIDER_MAX_PAGES; page++) {
const data = await this.requestJson<unknown>(creds, '/tags/', {
query: {
page: String(page),
page_size: String(TAG_PAGE_SIZE),
ordering: 'name',
name__icontains: query !== undefined && query.trim().length > 0 ? query.trim() : undefined,
},
});
const { rows, hasNext } = pageRows(data);
for (const row of rows) {
const tag = parseTag(row);
if (tag !== null) tags.push(tag);
}
if (!hasNext) break;
}
return tags;
}
async getTag(creds: PaperlessCreds, tagId: number): Promise<PaperlessTag> {
const tag = parseTag(await this.requestJson<unknown>(creds, `/tags/${tagId}/`));
if (tag === null) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown tag shape');
}
return tag;
}
async createTag(creds: PaperlessCreds, name: string): Promise<PaperlessTag> {
const tag = parseTag(
await this.requestJson<unknown>(creds, '/tags/', { method: 'POST', json: { name } }),
);
if (tag === null) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown tag shape');
}
return tag;
}
/**
* Every document carrying the tag, oldest id first.
*
* `tags__id__all` rather than `tags__id__in`: with a single tag the two agree,
* and `__all` keeps meaning "carries this tag" if a caller ever passes more.
* Ordering by id keeps the pages stable while documents are being added:
* ordering by `modified`, the field that changes under us, would let a
* document move between pages and never be seen at all.
*
* Pages are walked by number instead of by following `next`: Paperless builds
* that URL from the request's own Host header, so behind a reverse proxy it
* can name an origin this connection was never allowed to talk to.
*/
async listDocuments(
creds: PaperlessCreds,
tagId: number,
): Promise<{ documents: PaperlessDocument[]; truncated: boolean; count: number | null }> {
const documents: PaperlessDocument[] = [];
let truncated = false;
let count: number | null = null;
for (let page = 1; page <= PROVIDER_MAX_PAGES; page++) {
const data = await this.requestJson<unknown>(creds, '/documents/', {
query: {
tags__id__all: String(tagId),
ordering: 'id',
page: String(page),
page_size: String(DOCUMENT_PAGE_SIZE),
},
});
const { rows, hasNext, count: total } = pageRows(data);
count ??= total;
for (const row of rows) {
const doc = parseDocument(row);
if (doc !== null) documents.push(doc);
}
if (!hasNext) break;
if (page === PROVIDER_MAX_PAGES) truncated = true;
}
return { documents, truncated, count };
}
async getDocument(creds: PaperlessCreds, documentId: number): Promise<PaperlessDocument> {
const doc = parseDocument(await this.requestJson<unknown>(creds, `/documents/${documentId}/`));
if (doc === null) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown document shape');
}
return doc;
}
/**
* Documents whose original file has exactly these bytes.
*
* The filter is server-side and exact. It exists here for one case: an upload
* whose consume task was never observed, where the document may or may not
* have landed and a blind retry would leave two: Paperless does not refuse
* duplicate bytes.
*/
async findByChecksum(creds: PaperlessCreds, sha256: string): Promise<PaperlessDocument[]> {
const data = await this.requestJson<unknown>(creds, '/documents/', {
query: { checksum__iexact: sha256, ordering: '-id', page_size: String(DOCUMENT_PAGE_SIZE) },
});
const { rows } = pageRows(data);
const documents: PaperlessDocument[] = [];
for (const row of rows) {
const doc = parseDocument(row);
if (doc !== null) documents.push(doc);
}
return documents;
}
async patchDocument(
creds: PaperlessCreds,
documentId: number,
patch: { title?: string },
): Promise<PaperlessDocument> {
const doc = parseDocument(
await this.requestJson<unknown>(creds, `/documents/${documentId}/`, {
method: 'PATCH',
json: patch,
}),
);
if (doc === null) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown document shape');
}
return doc;
}
/** Moves the document to Paperless's trash. There is no hard delete here. */
async deleteDocument(creds: PaperlessCreds, documentId: number): Promise<void> {
discardBody(await this.request(creds, `/documents/${documentId}/`, { method: 'DELETE' }));
}
/**
* The ORIGINAL bytes, streaming. The caller owns the body from here.
*
* Without `original=true` Paperless serves the archived PDF it produced
* itself, which is a different file with a different length and a different
* hash. TREK would store something the user never uploaded, and every
* comparison against the checksum would fail.
*/
async downloadOriginal(creds: PaperlessCreds, documentId: number): Promise<Response> {
return this.request(creds, `/documents/${documentId}/download/`, {
query: { original: 'true' },
timeoutMs: PROVIDER_TRANSFER_TIMEOUT_MS,
raw: true,
});
}
/** Upload a new document. Answers the consume task's id, not a document id. */
async postDocument(
creds: PaperlessCreds,
file: PaperlessUploadFile,
fields: { title?: string; tagId: number; customFields?: Record<string, string> },
): Promise<string> {
const parts: MultipartField[] = [{ name: 'tags', value: String(fields.tagId) }];
if (fields.title !== undefined && fields.title.length > 0) {
parts.push({ name: 'title', value: fields.title });
}
if (fields.customFields !== undefined && Object.keys(fields.customFields).length > 0) {
// Paperless expects the custom fields as ONE JSON object in a single form
// field, keyed by field id. Repeating `custom_fields=<id>` instead
// attaches the fields with no value, which loses the anchor silently.
parts.push({ name: 'custom_fields', value: JSON.stringify(fields.customFields) });
}
return this.postMultipart(creds, '/documents/post_document/', parts, file);
}
/**
* Replace the bytes of an existing document, keeping its id, tags and custom
* fields. Answers the consume task's id.
*/
async updateVersion(
creds: PaperlessCreds,
documentId: number,
file: PaperlessUploadFile,
): Promise<string> {
return this.postMultipart(creds, `/documents/${documentId}/update_version/`, [], file);
}
private async postMultipart(
creds: PaperlessCreds,
path: string,
fields: MultipartField[],
file: PaperlessUploadFile,
): Promise<string> {
const { body, contentType } = buildMultipart(fields, file);
const data = await this.requestJson<unknown>(creds, path, {
method: 'POST',
body,
contentType,
timeoutMs: PROVIDER_TRANSFER_TIMEOUT_MS,
});
// The whole answer is the task UUID as a bare JSON string.
const taskId = asString(data);
if (taskId === null || taskId.length === 0) {
throw new PaperlessError('provider_error', 'Paperless accepted the upload without naming a task');
}
return taskId;
}
async getTask(creds: PaperlessCreds, taskId: string): Promise<PaperlessTask | null> {
const data = await this.requestJson<unknown>(creds, '/tasks/', { query: { task_id: taskId } });
const { rows } = pageRows(data);
for (const row of rows) {
const task = parseTask(row);
// `task_id` is compared rather than trusted: the filter is a query
// parameter, and Paperless ignores parameters it does not know instead of
// rejecting them, so an older build would answer with every task there is.
if (task !== null && task.taskId === taskId) return task;
}
return null;
}
/**
* Wait for a consume task to finish and answer the document it produced.
*
* Bounded twice: each poll carries its own request timeout, and the loop as a
* whole gives up at `timeoutMs`. A large scan with OCR can outlast any budget
* worth holding a worker for, and the caller has a cheaper way to find out
* whether the upload landed (the checksum filter), so this reports `timeout`
* rather than waiting longer.
*/
async awaitConsume(
creds: PaperlessCreds,
taskId: string,
options: { intervalMs?: number; timeoutMs?: number } = {},
): Promise<{ documentId: number }> {
const interval = options.intervalMs ?? TASK_POLL_INTERVAL_MS;
const deadline = Date.now() + (options.timeoutMs ?? TASK_POLL_TIMEOUT_MS);
for (;;) {
const task = await this.getTask(creds, taskId);
const status = task?.status.toLowerCase() ?? '';
if (status === 'failure' || status === 'revoked') {
// A consume that fails on the file itself (a broken PDF, an encrypted
// one) is a per-document verdict, not a transport fault.
const detail = task?.errorMessage ?? null;
throw new PaperlessError(
detail !== null && /duplicate/i.test(detail) ? 'conflict' : 'provider_error',
'Paperless could not process the uploaded file',
undefined,
detail === null ? undefined : shorten(detail),
);
}
if (status === 'success' && task !== null) {
const documentId = task.documentIds[0];
if (documentId === undefined) {
throw new PaperlessError(
'provider_error',
'Paperless finished the upload without naming a document',
);
}
return { documentId };
}
if (Date.now() + interval >= deadline) {
throw new PaperlessError(
'timeout',
'Paperless is still processing the uploaded file',
undefined,
`task ${taskId} was still ${status === '' ? 'unknown' : status}`,
);
}
await sleep(interval);
}
}
/** The custom field with this exact name, or null. */
async findCustomField(creds: PaperlessCreds, name: string): Promise<PaperlessCustomField | null> {
const data = await this.requestJson<unknown>(creds, '/custom_fields/', {
query: { name__iexact: name, page_size: String(TAG_PAGE_SIZE) },
});
const { rows } = pageRows(data);
for (const row of rows) {
const field = parseCustomField(row);
// The filter is server-side, but an unknown query parameter is IGNORED by
// Paperless rather than rejected, so a build without it would hand back
// every field there is. Comparing the name here makes that harmless.
if (field !== null && field.name.toLowerCase() === name.toLowerCase()) return field;
}
return null;
}
async createCustomField(creds: PaperlessCreds, name: string): Promise<PaperlessCustomField> {
const field = parseCustomField(
await this.requestJson<unknown>(creds, '/custom_fields/', {
method: 'POST',
json: { name, data_type: 'string' },
}),
);
if (field === null) {
throw new PaperlessError('provider_error', 'Paperless answered an unknown custom field shape');
}
return field;
}
/**
* Subscribe Paperless to its own changes for one tag.
*
* Two triggers rather than one: "document added" misses every later edit, and
* "document updated" misses the arrival. Both are filtered on the tag, so the
* rest of a household's papers never cause a callback.
*
* The payload is deliberately thin. Placeholders such as `{doc_id}` are NOT
* substituted in a webhook body at 3.1.3 (measured: they arrive literally),
* and `as_json` wraps a body STRING into a JSON string rather than sending an
* object, so the parameters, which do arrive as a real JSON object, carry the
* link, and the callback means nothing more than "look now".
*/
async createWorkflow(creds: PaperlessCreds, spec: PaperlessWorkflowSpec): Promise<number> {
if (spec.callbackUrl.length > WEBHOOK_URL_MAX_LENGTH) {
throw new PaperlessError(
'provider_error',
'The callback URL is longer than Paperless accepts',
undefined,
`${spec.callbackUrl.length} characters, limit ${WEBHOOK_URL_MAX_LENGTH}`,
);
}
const trigger = (type: number) => ({
type,
sources: TRIGGER_SOURCES,
filter_has_tags: [spec.tagId],
matching_algorithm: 0,
match: '',
is_insensitive: true,
});
const data = await this.requestJson<unknown>(creds, '/workflows/', {
method: 'POST',
json: {
name: spec.name,
order: 1,
enabled: true,
triggers: [trigger(TRIGGER_DOCUMENT_ADDED), trigger(TRIGGER_DOCUMENT_UPDATED)],
actions: [
{
type: ACTION_WEBHOOK,
webhook: {
url: spec.callbackUrl,
use_params: true,
as_json: true,
params: spec.params,
headers: { [spec.secretHeader.name]: spec.secretHeader.value },
include_document: false,
},
},
],
},
});
const id = isRecord(data) ? asNumber(data.id) : null;
if (id === null) {
throw new PaperlessError('provider_error', 'Paperless created the workflow without naming it');
}
return id;
}
/** Removing a subscription that is already gone is success, not a failure. */
async deleteWorkflow(creds: PaperlessCreds, workflowId: number): Promise<void> {
try {
discardBody(await this.request(creds, `/workflows/${workflowId}/`, { method: 'DELETE' }));
} catch (err: unknown) {
if (err instanceof PaperlessError && err.code === 'not_found') return;
throw err;
}
}
}
@@ -0,0 +1,723 @@
import { Injectable } from '@nestjs/common';
import { Readable } from 'node:stream';
import {
docFail,
docOk,
type DocResult,
type DocumentConnectionRef,
type DocumentProvider,
type DocumentProviderCapabilities,
type DocumentScopeOption,
type DocumentScopeRef,
type FetchResult,
type PushRequest,
type PushResult,
type RemoteDocument,
} from '../document-provider';
import {
PaperlessClient,
PaperlessError,
currentVersion,
sha256OrNull,
type PaperlessCreds,
type PaperlessDocument,
type PaperlessTag,
} from './paperless.client';
import { DOWNLOAD_MAX_BYTES, guardDownload } from './provider-http';
/**
* Paperless-ngx as a document scope.
*
* The scope is a TAG, because that is the only container Paperless has: there
* are no folders, and a document belongs to as many tags as someone gives it.
* A tag is also the one thing the instance's owner can see and reason about in
* their own interface, which matters: this is their archive, and TREK is a
* guest in it.
*
* The tag is paired with a custom field (`trek_trip_id`) carrying the trip's
* uid. A tag can be removed by a human in two clicks and the binding would be
* silently empty; the custom field survives that and says which trip a document
* came from even after someone re-tags it. Nothing here treats the field as
* authority: it is a breadcrumb for the person cleaning up, and for a support
* question that starts with "where did this come from".
*
* Deletion propagates only as far as Paperless's own trash. `DELETE` on a
* document is a soft delete there (`deleted_at` is set, the document leaves
* every listing and `/api/documents/{id}/` answers 404), and the retention is
* the instance's `trash_delay`. There is no hard delete on the API at all.
*/
/** `tag:<id>`, the only scope key shape this adapter reads or writes. */
const SCOPE_PREFIX = 'tag:';
/** The custom field the trip uid is written into, created on demand. */
export const TRIP_UID_FIELD_NAME = 'trek_trip_id';
/** The header the self-registered workflow sends its shared secret in. */
export const PAPERLESS_WEBHOOK_SECRET_HEADER = 'X-Trek-Docsync-Secret';
/**
* What one push may buffer when the core did not say how large the file is.
*
* The body has to become a Buffer (Paperless wants a sized multipart request),
* so an unbounded read would be a memory bomb driven by a remote file. The
* declared size wins where there is one; this is the fallback ceiling.
*/
const PUSH_FALLBACK_MAX_BYTES = 64 * 1024 * 1024;
/**
* What Paperless accepts without Tika. Measured against 3.1.3: an unsupported
* type is refused with `400 … not supported` before a task is created.
*
* Note that Paperless decides on the SNIFFED type of the bytes, not on the
* Content-Type of the multipart part: a .csv arrives as `text/plain` and is
* stored as such. The list is therefore a pre-filter that saves a round trip,
* never the last word; the 400 is still mapped to the same code.
*/
const BASE_MIME_TYPES = [
'application/pdf',
'image/jpeg',
'image/png',
'image/tiff',
'image/gif',
'image/bmp',
'image/webp',
'image/heic',
'image/heif',
'text/plain',
'text/csv',
'message/rfc822',
] as const;
/**
* Accepted only on an instance that runs Tika and Gotenberg. They stay in the
* list: an instance WITH Tika is a supported setup, and refusing a .docx here
* would drop a file the provider would have taken. Without Tika the upload
* comes back as `unsupported_type` from Paperless itself, which is the same
* verdict one round trip later.
*/
const TIKA_MIME_TYPES = [
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.ms-powerpoint',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/vnd.oasis.opendocument.text',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.oasis.opendocument.presentation',
'application/rtf',
] as const;
const ACCEPTED_MIME_TYPES: readonly string[] = [...BASE_MIME_TYPES, ...TIKA_MIME_TYPES];
/**
* The extension a Paperless document should carry in TREK.
*
* Paperless keeps a `title` without an extension and the uploaded file name
* separately, while TREK's file list is a list of file names. The extension is
* taken from the original name where there is one and derived from the MIME
* type otherwise, so a document pulled twice keeps the same name.
*/
const EXTENSION_BY_MIME: Readonly<Record<string, string>> = {
'application/pdf': '.pdf',
'image/jpeg': '.jpg',
'image/png': '.png',
'image/tiff': '.tif',
'image/gif': '.gif',
'image/bmp': '.bmp',
'image/webp': '.webp',
'image/heic': '.heic',
'image/heif': '.heif',
'text/plain': '.txt',
'text/csv': '.csv',
'message/rfc822': '.eml',
'application/msword': '.doc',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
'application/vnd.ms-excel': '.xls',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
'application/vnd.ms-powerpoint': '.ppt',
'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
'application/vnd.oasis.opendocument.text': '.odt',
'application/vnd.oasis.opendocument.spreadsheet': '.ods',
'application/vnd.oasis.opendocument.presentation': '.odp',
'application/rtf': '.rtf',
};
const KNOWN_EXTENSIONS = new Set(Object.values(EXTENSION_BY_MIME));
/**
* The bare type, without the parameters a stored Content-Type may carry.
* `text/plain; charset=utf-8` is text/plain, and comparing it verbatim against
* the allow-list would refuse a file the provider takes happily.
*/
export function normalizeMime(mimeType: string): string {
const [type = ''] = mimeType.split(';');
return type.trim().toLowerCase();
}
function extensionOf(name: string): string | null {
const match = /\.[A-Za-z0-9]{1,5}$/.exec(name);
return match === null ? null : match[0].toLowerCase();
}
/**
* The file name a document gets in TREK: the title, plus the extension it lost
* on the way into Paperless.
*/
export function documentFileName(doc: PaperlessDocument): string {
const base = doc.title.trim().length > 0
? doc.title.trim()
: (doc.originalFileName ?? `document-${doc.id}`);
const fromOriginal = doc.originalFileName === null ? null : extensionOf(doc.originalFileName);
const fromMime = doc.mimeType === null ? null : (EXTENSION_BY_MIME[doc.mimeType] ?? null);
const extension = fromOriginal ?? fromMime;
if (extension === null) return base;
return base.toLowerCase().endsWith(extension) ? base : `${base}${extension}`;
}
/**
* The title for a file name. Only an extension this adapter would have ADDED is
* removed, so a document genuinely called `Vertrag 2026.2` keeps its name.
*/
export function titleFromFileName(name: string): string {
const trimmed = name.trim();
const extension = extensionOf(trimmed);
if (extension === null || !KNOWN_EXTENSIONS.has(extension)) return trimmed;
const base = trimmed.slice(0, trimmed.length - extension.length).trim();
return base.length > 0 ? base : trimmed;
}
export function formatScopeKey(tagId: number): string {
return `${SCOPE_PREFIX}${tagId}`;
}
/** The tag id in a scope key, or null when it is not one of ours. */
export function parseScopeKey(scopeKey: string): number | null {
if (!scopeKey.startsWith(SCOPE_PREFIX)) return null;
const raw = scopeKey.slice(SCOPE_PREFIX.length);
if (!/^\d+$/.test(raw)) return null;
const id = Number.parseInt(raw, 10);
return Number.isSafeInteger(id) && id > 0 ? id : null;
}
function scopeOption(tag: PaperlessTag): DocumentScopeOption {
return {
scopeKey: formatScopeKey(tag.id),
label: tag.name,
remoteRootId: String(tag.id),
// The filter URL of the instance's own document list: paste it after the
// base URL and you are looking at exactly what TREK syncs.
remoteRootPath: `/documents?tags__id__all=${tag.id}`,
};
}
function toRemoteDocument(doc: PaperlessDocument): RemoteDocument {
const version = currentVersion(doc);
return {
remoteId: String(doc.id),
name: documentFileName(doc),
// Paperless publishes no byte count anywhere in the document serializer;
// the only way to learn it is to download, which is not worth a round trip
// per document per run.
size: null,
mimeType: doc.mimeType,
// `modified` moves on every edit, including the bulk edits that older
// Paperless builds left untouched (3.1.3 does bump it, measured).
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
contentHash: sha256OrNull(version?.checksum ?? null),
remoteModifiedAt: doc.modified,
isDeleted: doc.deletedAt !== null,
};
}
/**
* Cursor shape. The count travels with the timestamp because Paperless has no
* change feed: a document leaving the tag moves no timestamp at all, and only
* the count notices it.
*/
function buildCursor(count: number | null, documents: PaperlessDocument[]): string {
let newest = '';
for (const doc of documents) {
const modified = doc.modified ?? '';
if (modified > newest) newest = modified;
}
return `v1:${count ?? documents.length}:${newest}`;
}
/** Read a stream into memory, refusing to grow past the cap. */
async function readStreamCapped(
body: Readable,
maxBytes: number,
): Promise<{ bytes: Buffer; truncated: boolean }> {
const chunks: Buffer[] = [];
let received = 0;
for await (const chunk of body as AsyncIterable<Buffer | Uint8Array | string>) {
const buffer = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : Buffer.from(chunk);
received += buffer.length;
if (received > maxBytes) {
body.destroy();
return { bytes: Buffer.concat(chunks), truncated: true };
}
chunks.push(buffer);
}
return { bytes: Buffer.concat(chunks), truncated: false };
}
@Injectable()
export class PaperlessDocumentProvider implements DocumentProvider {
readonly id = 'paperless';
constructor(private readonly client: PaperlessClient) {}
capabilities(_conn: DocumentConnectionRef): DocumentProviderCapabilities {
return {
// Paperless can subscribe itself: a workflow with a webhook action is
// creatable through the same token, provided its user holds
// `add_workflow`. probe() reports the downgrade when it does not.
push: 'webhook-self-registered',
stableId: true,
remoteTrash: true,
// `update_version` replaces the bytes under the same document id, which
// is what keeps the mapping and the document's history intact.
replaceInPlace: true,
// Every listed document carries the sha256 of its current revision in
// `versions[]`, so no extra round trip is needed to compare content.
contentHashInListing: true,
// Paperless imposes no limit of its own; what a large upload runs into is
// the reverse proxy in front of it, which the adapter cannot know.
maxUploadBytes: null,
acceptedMimeTypes: ACCEPTED_MIME_TYPES,
canCreateScope: true,
};
}
async probe(
conn: DocumentConnectionRef,
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
try {
const profile = await this.client.probe(creds);
const capabilities = this.capabilities(conn);
// A token whose user may not create workflows can still sync; it just
// cannot subscribe itself, and the operator has to be told that here
// rather than after the first silent hour of polling.
const canSubscribe = profile.permissions.length === 0
|| profile.permissions.includes('add_workflow');
return docOk({
account: profile.username.length > 0 ? profile.username : 'paperless',
capabilities: canSubscribe ? capabilities : { ...capabilities, push: 'webhook-manual' },
});
} catch (err: unknown) {
return failure(err);
}
}
async listScopes(
conn: DocumentConnectionRef,
query?: string,
): Promise<DocResult<DocumentScopeOption[]>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
try {
const tags = await this.client.listTags(creds, query);
return docOk(tags.map(scopeOption));
} catch (err: unknown) {
return failure(err);
}
}
async createScope(
conn: DocumentConnectionRef,
name: string,
): Promise<DocResult<DocumentScopeOption>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
try {
return docOk(scopeOption(await this.client.createTag(creds, name.trim())));
} catch (err: unknown) {
// A name that is already taken comes back as a 400 about the owner/name
// unique constraint. It is not adopted silently: an existing tag may be
// someone else's, and binding a trip to it would publish their documents
// to everyone on that trip.
return failure(err);
}
}
async resolveScope(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<DocumentScopeOption>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
const tagId = parseScopeKey(scope.scopeKey);
if (tagId === null) return badScope(scope.scopeKey);
try {
return docOk(scopeOption(await this.client.getTag(creds, tagId)));
} catch (err: unknown) {
// A tag that is gone is `scope_lost` for the link, not a generic 404:
// someone deleted or renamed it upstream and a human has to choose again.
if (err instanceof PaperlessError && err.code === 'not_found') {
return docFail('scope_missing', `tag ${tagId} no longer exists`, err.status);
}
return failure(err);
}
}
async list(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<{
documents: RemoteDocument[];
cursor: string | null;
cursorUnchanged: boolean;
truncated: boolean;
}>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
const tagId = parseScopeKey(scope.scopeKey);
if (tagId === null) return badScope(scope.scopeKey);
try {
const { documents, truncated, count } = await this.client.listDocuments(creds, tagId);
const cursor = buildCursor(count, documents);
return docOk({
documents: documents.map(toRemoteDocument),
cursor,
// A truncated walk never claims the scope is unchanged: the pages it
// did not read could hold anything.
cursorUnchanged: !truncated && scope.cursor !== null && scope.cursor === cursor,
truncated,
});
} catch (err: unknown) {
// An empty tag and a deleted tag look alike in a document listing, so the
// tag is only ever confirmed by resolveScope. A 404 here is the documents
// endpoint itself, which is a different problem.
return failure(err);
}
}
async fetch(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<FetchResult>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
const documentId = parseRemoteId(remoteId);
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
try {
// The metadata first: it carries the version marker the core stores
// alongside the bytes, and it fails cheaply when the document is gone.
const doc = await this.client.getDocument(creds, documentId);
const response = await this.client.downloadOriginal(creds, documentId);
const download = guardDownload(response, {
maxBytes: DOWNLOAD_MAX_BYTES,
tooLarge: (declared) =>
new PaperlessError(
'too_large',
'The document is larger than TREK will transfer',
undefined,
`content_length=${declared}`,
),
noBody: () => new PaperlessError('provider_error', 'Paperless sent no body for the document'),
});
return docOk({
body: download.body,
size: download.size,
mimeType: response.headers.get('content-type') ?? doc.mimeType,
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
});
} catch (err: unknown) {
return failure(err);
}
}
async push(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
req: PushRequest,
): Promise<DocResult<PushResult>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
const tagId = parseScopeKey(scope.scopeKey);
if (tagId === null) return badScope(scope.scopeKey);
const mimeType = normalizeMime(req.mimeType);
if (!ACCEPTED_MIME_TYPES.includes(mimeType)) {
return docFail('unsupported_type', `Paperless does not accept ${req.mimeType}`);
}
const cap = req.size > 0 ? req.size : PUSH_FALLBACK_MAX_BYTES;
let bytes: Buffer;
try {
const read = await readStreamCapped(req.body, cap);
if (read.truncated) {
return docFail('too_large', `the file is larger than the declared ${cap} bytes`);
}
bytes = read.bytes;
} catch (err: unknown) {
return docFail(
'provider_error',
err instanceof Error ? err.message : 'could not read the local file',
);
}
const file = { fileName: req.fileName, mimeType, bytes };
const replacing = req.remoteId !== undefined;
try {
const documentId = replacing
? await this.replace(creds, file, req)
: await this.create(creds, tagId, file, req);
const doc = await this.client.getDocument(creds, documentId);
// Paperless stores the sha256 of what it received. Comparing it here
// turns a truncated upload into a reported mismatch instead of a
// document that silently differs from the one in the trip.
const stored = sha256OrNull(currentVersion(doc)?.checksum ?? null);
if (stored !== null && stored !== req.sha256.toLowerCase()) {
// A document this call created and nobody has seen is litter, so it
// goes to the trash where the owner can still get at it. A REPLACE is
// left alone: that document existed before and its earlier revisions
// are still attached to it.
if (!replacing) await this.discard(creds, doc.id);
return docFail(
'checksum_mismatch',
`Paperless stored ${stored} for document ${doc.id}`,
);
}
return docOk({
remoteId: String(doc.id),
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
remoteModifiedAt: doc.modified,
// Paperless does not deduplicate: the same bytes uploaded twice become
// two documents with the same checksum and an empty
// `duplicate_documents`. Nothing here may claim otherwise.
deduplicated: false,
});
} catch (err: unknown) {
return failure(err);
}
}
/**
* Trash a document this push created but is not going to report.
*
* Best effort on purpose: the caller is already returning a failure, and a
* second one on top of it would replace a precise diagnosis with the fallout
* of the cleanup.
*/
private async discard(creds: PaperlessCreds, documentId: number): Promise<void> {
try {
await this.client.deleteDocument(creds, documentId);
} catch {
// Nothing to do: the document stays in the archive and the mismatch is
// still what the caller hears about.
}
}
/**
* A new document, with both anchors set in the same request: the tag, which
* is what the listing filters on, and the trip uid in the custom field, which
* outlives someone re-tagging the document by hand.
*
* The field id is resolved per upload rather than cached. It is one small GET
* against a local instance, and a cached id goes stale exactly when it hurts
* most: after a human deleted the field, when every later upload would fail
* with a validation error nobody can read.
*/
private async create(
creds: PaperlessCreds,
tagId: number,
file: { fileName: string; mimeType: string; bytes: Buffer },
req: PushRequest,
): Promise<number> {
const field = (await this.client.findCustomField(creds, TRIP_UID_FIELD_NAME))
?? (await this.client.createCustomField(creds, TRIP_UID_FIELD_NAME));
const taskId = await this.client.postDocument(creds, file, {
title: titleFromFileName(req.fileName),
tagId,
customFields: { [String(field.id)]: req.trekTripUid },
});
try {
return (await this.client.awaitConsume(creds, taskId)).documentId;
} catch (err: unknown) {
// The consume outlived the poll budget. The document may well be there
// (OCR on a long scan takes minutes), and retrying blind would leave two,
// because Paperless accepts duplicate bytes. So the checksum decides.
if (err instanceof PaperlessError && err.code === 'timeout') {
const existing = await this.client.findByChecksum(creds, req.sha256);
// Only a document carrying this scope's tag can be the one this push
// made: the tag goes on at upload. Anything else with the same bytes
// belongs to someone else's filing, and adopting it would put a
// stranger's document in the trip, and delete it on the next
// delete-through.
const mine = existing.find((doc) => doc.tagIds.includes(tagId));
if (mine !== undefined) return mine.id;
}
throw err;
}
}
/**
* Replace the bytes of a document TREK already knows.
*
* The document id is the one that was passed in, never the one the consume
* task reports: `update_version` files the new revision under a fresh
* internal id that answers 404 on its own, and adopting it would break the
* mapping on the first replace.
*/
private async replace(
creds: PaperlessCreds,
file: { fileName: string; mimeType: string; bytes: Buffer },
req: PushRequest,
): Promise<number> {
const documentId = parseRemoteId(req.remoteId ?? '');
if (documentId === null) {
throw new PaperlessError('not_found', `unreadable document id ${req.remoteId ?? ''}`);
}
const taskId = await this.client.updateVersion(creds, documentId, file);
try {
await this.client.awaitConsume(creds, taskId);
} catch (err: unknown) {
if (!(err instanceof PaperlessError) || err.code !== 'timeout') throw err;
// Same reasoning as create(), one step shorter: the document keeps its
// id, so whether the new revision landed is visible on the document.
const doc = await this.client.getDocument(creds, documentId);
if (sha256OrNull(currentVersion(doc)?.checksum ?? null) !== req.sha256.toLowerCase()) {
throw err;
}
}
return documentId;
}
async rename(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
name: string,
): Promise<DocResult<{ remoteVersion: string }>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
const documentId = parseRemoteId(remoteId);
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
try {
// Only the title moves. The uploaded file name is immutable in Paperless,
// and the archived file it serves is named after the title anyway.
const doc = await this.client.patchDocument(creds, documentId, {
title: titleFromFileName(name),
});
return docOk({ remoteVersion: doc.modified ?? doc.added ?? String(doc.id) });
} catch (err: unknown) {
return failure(err);
}
}
async trash(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<void>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
const documentId = parseRemoteId(remoteId);
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
try {
await this.client.deleteDocument(creds, documentId);
return docOk(undefined);
} catch (err: unknown) {
// Already in the trash is the state that was asked for.
if (err instanceof PaperlessError && err.code === 'not_found') return docOk(undefined);
return failure(err);
}
}
async registerWebhook(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
callbackUrl: string,
secret: string,
): Promise<DocResult<{ subscriptionId: string }>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
const tagId = parseScopeKey(scope.scopeKey);
if (tagId === null) return badScope(scope.scopeKey);
try {
const workflowId = await this.client.createWorkflow(creds, {
name: `TREK document sync (link ${scope.linkId})`,
tagId,
callbackUrl,
secretHeader: { name: PAPERLESS_WEBHOOK_SECRET_HEADER, value: secret },
// Whatever Paperless sends is untrusted and only means "look now", so
// the payload carries the link and nothing about the document. It could
// not carry more: placeholders are not substituted in a webhook payload.
params: { linkId: String(scope.linkId), tripId: String(scope.tripId) },
});
return docOk({ subscriptionId: String(workflowId) });
} catch (err: unknown) {
return failure(err);
}
}
async unregisterWebhook(
conn: DocumentConnectionRef,
subscriptionId: string,
): Promise<DocResult<void>> {
const creds = credsOf(conn);
if (creds === null) return missingToken();
const workflowId = parseRemoteId(subscriptionId);
if (workflowId === null) {
return docFail('not_found', `unreadable subscription id ${subscriptionId}`);
}
try {
await this.client.deleteWorkflow(creds, workflowId);
return docOk(undefined);
} catch (err: unknown) {
return failure(err);
}
}
}
function credsOf(conn: DocumentConnectionRef): PaperlessCreds | null {
const token = conn.secrets.api_token ?? '';
if (token.trim().length === 0) return null;
return {
baseUrl: conn.baseUrl,
token: token.trim(),
allowInsecureTls: conn.allowInsecureTls,
};
}
function parseRemoteId(remoteId: string): number | null {
if (!/^\d+$/.test(remoteId.trim())) return null;
const id = Number.parseInt(remoteId.trim(), 10);
return Number.isSafeInteger(id) && id > 0 ? id : null;
}
function missingToken<T>(): DocResult<T> {
return docFail<T>('unauthorized', 'this connection has no Paperless API token');
}
function badScope<T>(scopeKey: string): DocResult<T> {
return docFail<T>('scope_missing', `not a Paperless scope key: ${scopeKey}`);
}
/**
* Every failure leaves through here, so a stray upstream exception cannot
* escape as an `unknown` with an English sentence in it.
*/
function failure<T>(err: unknown): DocResult<T> {
if (err instanceof PaperlessError) {
return docFail<T>(err.code, err.detail ?? err.message, err.status);
}
return docFail<T>('unknown', err instanceof Error ? err.message : undefined);
}
@@ -0,0 +1,704 @@
import { readCappedJson } from '../../../utils/cappedFetch';
import type { DocsyncErrorCode } from '@trek/shared';
import {
guardDownload,
providerFetch,
statusErrorCode,
type TransportFailure,
type TransportFailureCode,
} from './provider-http';
import crypto from 'node:crypto';
import { Readable } from 'node:stream';
/**
* Thin HTTP client for the Papra REST API (github.com/papra-hq/papra).
* This is the ONLY place that talks to a Papra instance.
*
* Verified against a running 26.6.2 instance on 2026-09-18, because Papra's
* prose documentation is silent on most of what a two-way sync depends on:
*
* - **Auth** is `Authorization: Bearer ppapi_…`. A key carries per-verb scopes
* (`documents:read`, `tags:create`, …) and a missing scope answers **401
* `auth.unauthorized`**, exactly like a wrong key: there is no 403 to tell
* the two apart, so a failed call names the route it failed on instead.
* - **No timestamp is usable for change detection.** `updatedAt` did not move
* when a tag was added, when the document was renamed, or when it was
* trashed; only `createdAt` and `deletedAt` are ever written. Hence
* `snapshotVersion()` (a hash over a fixed field order) rather than an
* ETag or a modified time.
* - **Uploads deduplicate on the sha256 of the bytes**, not on the name. An
* identical file answers 409 `document.already_exists`, and if the twin is
* in the trash it is **restored** and returned with its ORIGINAL id under a
* plain 200. Both are `deduplicated` to the sync core; see `uploadDocument`.
* - **The search grammar needs quoting**: `tag:"Trek Reise 2026"` matches,
* `tag:Trek Reise 2026` matches nothing. It is an exact, case-insensitive
* match on the tag name (`tag:trek-trip-4` does not find `trek-trip-42`),
* and it cannot search a sha256 at all.
* - **`custom-properties`, `webhooks` and the restore and hard-delete routes
* are session-only.** With a valid API key they answer 401; with a browser
* session the same request succeeds. So there is no anchor property to write
* and no subscription TREK can register for itself.
* - `pageSize` caps at 100 (101 is a 400), `pageIndex` is 0-based, and
* `documentsCount` is the total rather than the length of the page. The
* trash listing takes those two parameters and refuses every other one.
*/
const TIMEOUT_MS = 20000;
/** Uploads and downloads move real files, so they get their own budget. */
const TRANSFER_TIMEOUT_MS = 180000;
/** Papra's JSON answers are documents metadata; a page of 100 is tens of kilobytes. */
const MAX_JSON_BYTES = 8 * 1024 * 1024;
/**
* A runaway guard on a download, not the sync core's file-size policy. That
* one lives in the core, which knows what it is willing to store. This only has
* to stop an instance that answers a document request with an endless body.
*/
const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024;
/** Papra refuses anything above this outright. */
export const PAPRA_PAGE_SIZE = 100;
/**
* How many pages one enumeration walks before reporting itself incomplete.
*
* Papra has no change feed, so a sync is a full walk of the tag every time. A
* cap rather than a loop: a caller told the answer is partial can keep the
* previous state instead of reading absence as deletion.
*/
export const PAPRA_MAX_PAGES = 50;
/** Papra requires a colour on every tag, so scope creation has to pick one. */
export const PAPRA_DEFAULT_TAG_COLOR = '#4F46E5';
/** `org_` + 24 lowercase alphanumerics. Papra 400s on anything else. */
const ORG_ID_PATTERN = /^org_[a-z0-9]{24}$/;
export interface PapraCreds {
/** Instance origin. The client appends `/api` itself. */
baseUrl: string;
apiKey: string;
organizationId: string;
allowInsecureTls: boolean;
}
export interface PapraTag {
id: string;
name: string;
color: string | null;
description: string | null;
documentsCount: number | null;
}
export interface PapraDocument {
id: string;
name: string;
originalName: string | null;
originalSize: number | null;
/** Real sha256 of the stored bytes, and the key Papra deduplicates on. */
originalSha256Hash: string | null;
mimeType: string | null;
createdAt: string | null;
updatedAt: string | null;
deletedAt: string | null;
isDeleted: boolean;
/**
* Absent from the upload and rename answers, present in every listing. Null
* therefore means "not stated here", never "no tags".
*/
tags: PapraTag[] | null;
}
export interface PapraPage {
documents: PapraDocument[];
/** Total across all pages, not the length of this one. */
total: number;
}
export interface PapraUploadResult {
document: PapraDocument;
/**
* True when Papra answered with a document it already held: a trashed twin it
* restored under its original id, or, on the 409 path, the live twin this
* client then looked up.
*/
deduplicated: boolean;
}
export interface PapraDownload {
body: Readable;
/** From `content-length`, which Papra does send on the file route. */
size: number | null;
}
/**
* A failed Papra call, carrying the reason as a sync-core error code.
*
* The upstream message is English and occasionally an HTML error page from a
* reverse proxy, so it travels in `detail` for the self-hoster's log while the
* code is what the user sees through i18n. `papraCode` is kept separately
* because several branches turn on it: `document.already_exists` is a dedup,
* not a failure, and `tags.not_found` is a lost scope rather than a bad id.
*/
export class PapraError extends Error {
readonly code: DocsyncErrorCode;
readonly status?: number;
readonly detail?: string;
readonly papraCode?: string;
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string, papraCode?: string) {
super(message);
this.name = 'PapraError';
this.code = code;
this.status = status;
this.detail = detail;
this.papraCode = papraCode;
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function str(value: unknown): string | null {
return typeof value === 'string' ? value : null;
}
function num(value: unknown): number | null {
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function parseTag(value: unknown): PapraTag | null {
if (!isRecord(value)) return null;
const id = str(value.id);
const name = str(value.name);
if (!id || name === null) return null;
return {
id,
name,
color: str(value.color),
description: str(value.description),
documentsCount: num(value.documentsCount),
};
}
function parseDocument(value: unknown): PapraDocument | null {
if (!isRecord(value)) return null;
const id = str(value.id);
if (!id) return null;
const rawTags = value.tags;
let tags: PapraTag[] | null = null;
if (Array.isArray(rawTags)) {
tags = [];
for (const entry of rawTags) {
const tag = parseTag(entry);
if (tag) tags.push(tag);
}
}
return {
id,
// Papra has always sent a name; falling back to the original one keeps a
// stripped-down answer usable instead of dropping the whole document.
name: str(value.name) ?? str(value.originalName) ?? id,
originalName: str(value.originalName),
originalSize: num(value.originalSize),
originalSha256Hash: str(value.originalSha256Hash),
mimeType: str(value.mimeType),
createdAt: str(value.createdAt),
updatedAt: str(value.updatedAt),
deletedAt: str(value.deletedAt),
isDeleted: value.isDeleted === true,
tags,
};
}
/** The `{ error: { message, code } }` envelope Papra answers every 4xx with. */
function parseErrorBody(body: unknown): { message: string | null; code: string | null } {
if (!isRecord(body) || !isRecord(body.error)) return { message: null, code: null };
return { message: str(body.error.message), code: str(body.error.code) };
}
/**
* `https://host/papra/api/` → `https://host/papra/api`, tolerating what people
* paste. Without it a pasted `…/api` becomes `…/api/api` and every call 404s.
*/
export function papraApiBase(baseUrl: string): string {
const origin = baseUrl
.trim()
.replace(/(?<!\/)\/+$/, '')
.replace(/\/api$/i, '');
return `${origin}/api`;
}
/**
* Stable snapshot hash over a FIXED field order, standing in for the version
* marker Papra does not have. `updatedAt` is in it although it never moves: it
* costs nothing and it is the field that would start carrying the signal if a
* later Papra ever began writing it.
*/
export function snapshotVersion(doc: PapraDocument): string {
const snapshot = [doc.id, doc.name, doc.originalSha256Hash ?? '', doc.updatedAt ?? ''];
return crypto.createHash('sha256').update(JSON.stringify(snapshot)).digest('hex');
}
/**
* A tag name as the search grammar wants it: quoted, because an unquoted name
* with a space matches nothing at all.
*
* Returns null for a name the grammar cannot express (Papra accepts `"` in a
* tag name and offers no escape for it), so the caller can fall back to
* enumerating the organisation and filtering by tag id.
*/
export function tagSearchQuery(tagName: string): string | null {
if (tagName.includes('"') || tagName.includes('\\')) return null;
return `tag:"${tagName}"`;
}
/**
* Papra's status codes mapped onto the sync core's vocabulary.
*
* 400 on a path parameter is a `not_found` rather than a provider fault: the
* ids are format-checked upstream, so a remote id Papra refuses to parse is one
* that cannot exist. The organisation id is validated here before the call, so
* it can never be the parameter Papra is complaining about.
*/
function classify(status: number, papraCode: string | null): DocsyncErrorCode {
if (papraCode === 'tags.not_found') return 'scope_missing';
if (status === 400) return papraCode === 'server.invalid_request.params' ? 'not_found' : 'provider_error';
return statusErrorCode(status);
}
const TRANSPORT_MESSAGES: Record<TransportFailureCode, string> = {
ssrf_blocked: 'The Papra URL is not allowed',
timeout: 'Papra did not answer in time',
tls_untrusted: 'Papra presented a certificate that is not trusted',
unreachable: 'Could not reach Papra',
};
/** Why a request never produced a response at all. */
function transportError(failure: TransportFailure): PapraError {
return new PapraError(failure.code, TRANSPORT_MESSAGES[failure.code], undefined, failure.detail);
}
/** RFC 7578 as undici's own FormData writes it: UTF-8 bytes, three characters escaped. */
function escapeFormValue(value: string): string {
return value.replace(/\r/g, '%0D').replace(/\n/g, '%0A').replace(/"/g, '%22');
}
/**
* A media type that is safe to put in a header.
*
* The value comes from `trip_files.mime_type`, which an upload can influence,
* and it went into the part header untouched while the filename beside it was
* escaped. A CR or LF in it ends the header early and lets whatever follows be
* read as headers of its own. Anything that is not a plain media type is
* replaced rather than repaired: a mangled type is not worth guessing at, and
* the generic one is what an unknown file gets anyway.
*/
function safeMimeType(raw: string): string {
const TOKEN = /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/;
return TOKEN.test(raw) ? raw : 'application/octet-stream';
}
/** The bytes before the file in a one-field multipart body. Exported so the tests can read them. */
export function multipartHeader(boundary: string, fileName: string, mimeType: string): Buffer {
return Buffer.from(
`--${boundary}\r\n` +
`Content-Disposition: form-data; name="file"; filename="${escapeFormValue(fileName)}"\r\n` +
`Content-Type: ${safeMimeType(mimeType)}\r\n\r\n`,
'utf8',
);
}
export function multipartFooter(boundary: string): Buffer {
return Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8');
}
export class PapraClient {
/**
* One authenticated JSON call.
*
* `expectedEmpty` covers the 204s (tagging, some deletes): those have no body
* to parse and asking for one would turn a success into a parse failure.
*/
private async call(
creds: PapraCreds,
method: string,
path: string,
options: { query?: Record<string, string>; json?: unknown; expectEmpty?: boolean } = {},
): Promise<unknown> {
const url = new URL(papraApiBase(creds.baseUrl) + path);
for (const [key, value] of Object.entries(options.query ?? {})) url.searchParams.set(key, value);
const headers: Record<string, string> = {
Authorization: `Bearer ${creds.apiKey}`,
Accept: 'application/json',
};
if (options.json !== undefined) headers['Content-Type'] = 'application/json';
const response = await providerFetch(
url.toString(),
{ method, headers, body: options.json === undefined ? undefined : JSON.stringify(options.json) },
{ timeoutMs: TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
);
return this.readBody(response, `${method} ${path}`, options.expectEmpty === true);
}
/** Shared tail of every JSON call: cap the body, parse it, turn a non-2xx into a PapraError. */
private async readBody(response: Response, route: string, expectEmpty: boolean): Promise<unknown> {
const body = response.status === 204 ? null : await readCappedJson<unknown>(response, MAX_JSON_BYTES);
if (!response.ok) {
const { message, code } = parseErrorBody(body);
throw new PapraError(
classify(response.status, code),
`Papra answered HTTP ${response.status} on ${route}`,
response.status,
message ?? undefined,
code ?? undefined,
);
}
if (expectEmpty) return null;
if (body === undefined) {
throw new PapraError(
'provider_error',
'Papra returned something that is not JSON',
response.status,
`on ${route}; check the URL points at Papra itself and not at a login page in front of it`,
);
}
return body;
}
/**
* The organisation the connection names, checked before it is put in a URL.
*
* Papra answers a malformed organisation id with the same 400 shape it uses
* for a malformed document id, and those two mean very different things to a
* caller. Settling it here keeps the wire-level mapping unambiguous.
*/
private orgPath(creds: PapraCreds): string {
if (!ORG_ID_PATTERN.test(creds.organizationId)) {
throw new PapraError(
'provider_error',
'The organisation id is not in Papra’s format',
undefined,
`expected org_ followed by 24 lowercase alphanumerics, got "${creds.organizationId}"`,
);
}
return `/organizations/${creds.organizationId}`;
}
/**
* The organisations the key can reach, for the connection's display name.
*
* Separate from the reachability test on purpose: this needs the
* `organizations:read` scope, which a key limited to documents and tags does
* not carry, and a connection that syncs perfectly well should not fail its
* probe over a missing label.
*/
async listOrganizations(creds: PapraCreds): Promise<Array<{ id: string; name: string }>> {
const body = await this.call(creds, 'GET', '/organizations');
const raw = isRecord(body) && Array.isArray(body.organizations) ? body.organizations : [];
const out: Array<{ id: string; name: string }> = [];
for (const entry of raw) {
if (!isRecord(entry)) continue;
const id = str(entry.id);
if (!id) continue;
out.push({ id, name: str(entry.name) ?? id });
}
return out;
}
async listTags(creds: PapraCreds): Promise<PapraTag[]> {
const body = await this.call(creds, 'GET', `${this.orgPath(creds)}/tags`);
const raw = isRecord(body) && Array.isArray(body.tags) ? body.tags : [];
const tags: PapraTag[] = [];
for (const entry of raw) {
const tag = parseTag(entry);
if (tag) tags.push(tag);
}
return tags;
}
async createTag(creds: PapraCreds, name: string, color: string): Promise<PapraTag> {
const body = await this.call(creds, 'POST', `${this.orgPath(creds)}/tags`, { json: { name, color } });
const tag = isRecord(body) ? parseTag(body.tag) : null;
if (!tag) throw new PapraError('provider_error', 'Papra created a tag but did not describe it');
return tag;
}
/**
* One page of documents.
*
* Sorted by `createdAt` ascending rather than descending: a document uploaded
* while the walk is in flight then lands at the END of the last page, where
* missing it costs one sync cycle. Descending puts it at the front and pushes
* every later page down by one, which silently skips an existing document.
*
* The trash route takes `pageIndex` and `pageSize` and NOTHING else (it
* rejects a sort or a search with 400 `Invalid key: Expected never` rather
* than ignoring it), so the sort and the query are dropped there.
*/
async listDocuments(
creds: PapraCreds,
options: { searchQuery?: string; pageIndex: number; deleted?: boolean },
): Promise<PapraPage> {
const trash = options.deleted === true;
const query: Record<string, string> = {
pageIndex: String(options.pageIndex),
pageSize: String(PAPRA_PAGE_SIZE),
};
if (!trash) {
query.sortField = 'createdAt';
query.sortOrder = 'asc';
if (options.searchQuery) query.searchQuery = options.searchQuery;
}
const path = `${this.orgPath(creds)}/documents${trash ? '/deleted' : ''}`;
const body = await this.call(creds, 'GET', path, { query });
const raw = isRecord(body) && Array.isArray(body.documents) ? body.documents : [];
const documents: PapraDocument[] = [];
for (const entry of raw) {
const doc = parseDocument(entry);
if (doc) documents.push(doc);
}
const total = isRecord(body) ? num(body.documentsCount) : null;
return { documents, total: total ?? documents.length };
}
/**
* Every page, up to the cap.
*
* `truncated` is the honest half of the answer: the sync core reads absence
* from a listing as a possible deletion, so a walk that stopped early has to
* say so or a capped enumeration would look like a mass delete.
*/
async listAllDocuments(
creds: PapraCreds,
options: { searchQuery?: string; deleted?: boolean } = {},
): Promise<{ documents: PapraDocument[]; truncated: boolean }> {
const documents: PapraDocument[] = [];
let truncated = false;
for (let pageIndex = 0; pageIndex < PAPRA_MAX_PAGES; pageIndex++) {
const page = await this.listDocuments(creds, { ...options, pageIndex });
documents.push(...page.documents);
if (page.documents.length < PAPRA_PAGE_SIZE) break;
if (documents.length >= page.total) break;
if (pageIndex === PAPRA_MAX_PAGES - 1) truncated = true;
}
return { documents, truncated };
}
async getDocument(creds: PapraCreds, documentId: string): Promise<PapraDocument> {
const body = await this.call(creds, 'GET', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`);
const doc = isRecord(body) ? parseDocument(body.document) : null;
if (!doc) throw new PapraError('provider_error', 'Papra returned a document in an unexpected shape');
return doc;
}
/**
* The stored bytes.
*
* The response's own `content-type` is always `application/octet-stream`, so
* the caller takes the MIME type from the document record instead of from the
* transfer.
*/
async downloadDocument(creds: PapraCreds, documentId: string): Promise<PapraDownload> {
const url = `${papraApiBase(creds.baseUrl) + this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}/file`;
const response = await providerFetch(
url,
{ method: 'GET', headers: { Authorization: `Bearer ${creds.apiKey}` } },
{ timeoutMs: TRANSFER_TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
);
if (!response.ok) {
const body = await readCappedJson<unknown>(response, MAX_JSON_BYTES);
const { message, code } = parseErrorBody(body);
throw new PapraError(
classify(response.status, code),
`Papra answered HTTP ${response.status} on the document file`,
response.status,
message ?? undefined,
code ?? undefined,
);
}
return guardDownload(response, {
maxBytes: MAX_DOWNLOAD_BYTES,
tooLarge: () =>
new PapraError('too_large', 'The Papra document is larger than TREK will transfer', response.status),
noBody: () => new PapraError('provider_error', 'Papra answered the document file with no body', response.status),
});
}
/**
* Store bytes as a new document.
*
* Streamed rather than buffered: the sync core hands over whatever is in trip
* storage, and holding a document in memory to build a multipart body is the
* one thing a file sync must not do. The body is assembled by hand because
* `FormData` would want the bytes first.
*
* Two answers are not what they look like. A 200 carrying a `createdAt` from
* before this request is a trashed twin Papra restored under its original id,
* and a 409 `document.already_exists` is a live twin. Neither is a new
* document, and the caller resolves the second one through `findByHash`.
*/
async uploadDocument(
creds: PapraCreds,
file: { body: Readable; fileName: string; mimeType: string },
): Promise<PapraUploadResult> {
const boundary = `----trek${crypto.randomBytes(16).toString('hex')}`;
const header = multipartHeader(boundary, file.fileName, file.mimeType);
const footer = multipartFooter(boundary);
async function* frame(): AsyncGenerator<Buffer> {
yield header;
for await (const chunk of file.body) {
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
}
yield footer;
}
const sentAt = Date.now();
const response = await providerFetch(
`${papraApiBase(creds.baseUrl) + this.orgPath(creds)}/documents`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${creds.apiKey}`,
Accept: 'application/json',
'Content-Type': `multipart/form-data; boundary=${boundary}`,
},
body: Readable.toWeb(Readable.from(frame())),
// Required by undici for a streamed request body; the platform types
// carry the field, the DOM lib this project does not load does not.
duplex: 'half',
},
{ timeoutMs: TRANSFER_TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
);
const elapsedMs = Date.now() - sentAt;
const body = await this.readBody(response, 'POST /documents', false);
const document = isRecord(body) ? parseDocument(body.document) : null;
if (!document)
throw new PapraError('provider_error', 'Papra accepted the upload but did not describe the document');
return {
document,
deduplicated: isRestoredTwin(document, response.headers.get('date'), elapsedMs),
};
}
/**
* The document holding these bytes, for the 409 path.
*
* Three widening passes because Papra cannot search a hash: the name the
* upload used, then every live document, then the trash. The name pass is
* almost always enough and costs one request; the others exist because a twin
* can have been stored under a different name entirely, which is exactly the
* case where guessing would pair the wrong document.
*/
async findByHash(creds: PapraCreds, sha256: string, nameHint?: string): Promise<PapraDocument | null> {
const wanted = sha256.toLowerCase();
const match = (docs: PapraDocument[]): PapraDocument | null =>
docs.find((doc) => doc.originalSha256Hash?.toLowerCase() === wanted) ?? null;
if (nameHint) {
const byName = await this.listAllDocuments(creds, { searchQuery: nameHint });
const hit = match(byName.documents);
if (hit) return hit;
}
const live = await this.listAllDocuments(creds);
const hit = match(live.documents);
if (hit) return hit;
const trashed = await this.listAllDocuments(creds, { deleted: true });
return match(trashed.documents);
}
/**
* Put a tag on a document.
*
* An already-present tag answers 409 `documents.already_has_tag`, which is the
* state the caller wanted, so it is swallowed here rather than left for every
* call site to recognise.
*/
async addTag(creds: PapraCreds, documentId: string, tagId: string): Promise<void> {
try {
await this.call(creds, 'POST', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}/tags`, {
json: { tagId },
expectEmpty: true,
});
} catch (err: unknown) {
if (err instanceof PapraError && err.papraCode === 'documents.already_has_tag') return;
throw err;
}
}
async renameDocument(creds: PapraCreds, documentId: string, name: string): Promise<PapraDocument> {
const body = await this.call(creds, 'PATCH', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`, {
json: { name },
});
const doc = isRecord(body) ? parseDocument(body.document) : null;
if (!doc) throw new PapraError('provider_error', 'Papra renamed the document but did not describe it');
return doc;
}
/**
* Move a document to Papra's trash.
*
* This is the only destructive call the API offers a key: permanent deletion
* and restoring are session-only, and `?permanent=true` is accepted and then
* ignored, which is worse than being refused. Papra keeps trashed documents
* for its configured retention window, so a wrong propagation stays
* recoverable through its web interface.
*/
async trashDocument(creds: PapraCreds, documentId: string): Promise<void> {
await this.call(creds, 'DELETE', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`, {
expectEmpty: true,
});
}
}
/**
* Whether a 200 from the upload route describes a document Papra already held:
* a trashed twin it silently restored under its original id.
*
* Papra says nothing about which of the two happened, so this has to be read
* off `createdAt`: a restored document was created before this request started.
* That start is not a time this process knows in Papra's clock, so it is
* reconstructed from the answer as `Date` minus how long the call took.
* Comparing a duration instead of two clocks is what makes the test survive a
* self-hosted instance whose clock is minutes out.
*
* `Date` carries whole seconds, so the reconstruction always lands at or BEFORE
* the true start. That asymmetry is chosen rather than tolerated: a document
* this request really did create can never be mistaken for a restore, and the
* price is blindness to a twin trashed and re-pushed within roughly a second of
* being created. Nothing else in the answer closes that second: the name is
* not a tiebreaker, because a restore overwrites the stored name with the
* filename of the upload that triggered it.
*
* Without a `Date` header at all there is nothing to anchor to and the local
* clock has to stand in, which is why that branch needs a tolerance.
*/
function isRestoredTwin(doc: PapraDocument, dateHeader: string | null, elapsedMs: number): boolean {
if (!doc.createdAt) return false;
const created = Date.parse(doc.createdAt);
if (!Number.isFinite(created)) return false;
const answeredAt = dateHeader ? Date.parse(dateHeader) : Number.NaN;
if (Number.isFinite(answeredAt)) return created < answeredAt - elapsedMs;
return created < Date.now() - elapsedMs - UNANCHORED_CLOCK_TOLERANCE_MS;
}
/** How far a headerless instance's clock may sit from this one before dedup detection misfires. */
const UNANCHORED_CLOCK_TOLERANCE_MS = 30000;
@@ -0,0 +1,395 @@
import {
docFail,
docOk,
type DocResult,
type DocumentConnectionRef,
type DocumentProvider,
type DocumentProviderCapabilities,
type DocumentScopeOption,
type DocumentScopeRef,
type FetchResult,
type PushRequest,
type PushResult,
type RemoteDocument,
} from '../document-provider';
import {
PapraClient,
PapraError,
PAPRA_DEFAULT_TAG_COLOR,
snapshotVersion,
tagSearchQuery,
type PapraCreds,
type PapraDocument,
type PapraTag,
} from './papra.client';
import { Injectable } from '@nestjs/common';
import crypto from 'node:crypto';
/**
* Papra as a document scope: one organisation, one tag per trip.
*
* Papra has no folders, so the container a trip binds to is a tag, which is
* why the scope key carries both halves (`org:…/tag:…`). The organisation is in
* there although it is also on the connection: an API key reaches every
* organisation its owner belongs to, so a binding that only named the tag would
* silently follow the connection if its organisation were ever re-pointed.
*
* Two capabilities are `false`/`manual` because of what Papra withholds from an
* API key rather than what it cannot do. Webhook CRUD and the restore and
* hard-delete routes answer 401 to a key and 200 to a browser session, so TREK
* cannot subscribe itself, and there is no endpoint that replaces the bytes of
* an existing document at all.
*/
@Injectable()
export class PapraDocumentProvider implements DocumentProvider {
readonly id = 'papra';
private readonly client = new PapraClient();
capabilities(_conn: DocumentConnectionRef): DocumentProviderCapabilities {
return {
// The API exists, but only a browser session may call it.
push: 'webhook-manual',
stableId: true,
remoteTrash: true,
// No endpoint replaces a document's bytes. A new revision is a new
// document, and an identical one is deduplicated into the old.
replaceInPlace: false,
// `originalSha256Hash` is a real sha256 of the stored bytes.
contentHashInListing: true,
// The ceiling is DOCUMENT_STORAGE_MAX_UPLOAD_SIZE on the instance and is
// not exposed anywhere in the API, so claiming Papra's 10 MB default here
// would refuse uploads a tuned instance accepts. An oversized one comes
// back as 413 `too_large` instead.
maxUploadBytes: null,
acceptedMimeTypes: null,
canCreateScope: true,
};
}
async probe(
conn: DocumentConnectionRef,
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
return this.withCreds(conn, async (creds) => {
// Tags rather than `/organizations`: it is the narrowest call that proves
// all three things at once: the instance answers, the key is valid, and
// the key's owner is in this organisation.
await this.client.listTags(creds);
return { account: await this.accountLabel(creds), capabilities: this.capabilities(conn) };
});
}
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
return this.withCreds(conn, async (creds) => {
const tags = await this.client.listTags(creds);
const needle = query?.trim().toLowerCase();
return tags
.filter((tag) => !needle || tag.name.toLowerCase().includes(needle))
.map((tag) => toScopeOption(creds.organizationId, tag));
});
}
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
return this.withCreds(conn, async (creds) => {
// A name collision is reported rather than resolved to the existing tag:
// that tag may be someone else's filing, and the picker already lists it
// for an admin who meant to bind to it.
const tag = await this.client.createTag(creds, name, PAPRA_DEFAULT_TAG_COLOR);
return toScopeOption(creds.organizationId, tag);
});
}
async resolveScope(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<DocumentScopeOption>> {
return this.withCreds(conn, async (creds) => {
const tag = await this.requireTag(creds, scope);
return toScopeOption(creds.organizationId, tag);
});
}
async list(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<
DocResult<{ documents: RemoteDocument[]; cursor: string | null; cursorUnchanged: boolean; truncated: boolean }>
> {
return this.withCreds(conn, async (creds) => {
const tag = await this.requireTag(creds, scope);
const searchQuery = tagSearchQuery(tag.name);
const walked = await this.client.listAllDocuments(creds, searchQuery === null ? {} : { searchQuery });
const documents = walked.documents.filter((doc) => inScope(doc, tag.id, searchQuery !== null)).map(toRemote);
// A capped walk saw part of the tag, and the core reads absence as a
// possible deletion. Reporting no cursor keeps the incomplete picture out
// of the agreed state instead of pinning it as the new truth.
if (walked.truncated) {
return { documents, cursor: null, cursorUnchanged: false, truncated: true };
}
const cursor = enumerationDigest(documents);
return { documents, cursor, cursorUnchanged: scope.cursor === cursor, truncated: false };
});
}
/**
* The scope is not re-checked here. Every route is organisation-scoped
* upstream, so a document from elsewhere cannot be reached, and a document
* that has lost its tag is exactly the one the core is fetching to resolve
* the drift.
*/
async fetch(
conn: DocumentConnectionRef,
_scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<FetchResult>> {
return this.withCreds(conn, async (creds) => {
// The document first: the file route answers
// `content-type: application/octet-stream` for everything, so the real
// MIME type and the version marker only exist on the record.
const doc = await this.client.getDocument(creds, remoteId);
const download = await this.client.downloadDocument(creds, remoteId);
return {
body: download.body,
size: download.size ?? doc.originalSize,
mimeType: doc.mimeType,
remoteVersion: snapshotVersion(doc),
};
});
}
/**
* Store a trip file as a Papra document and tag it into the trip's scope.
*
* `trekDocUid`/`trekTripUid` are not written anywhere: Papra's custom
* properties are session-only, so a key has nowhere to put an anchor. The tag
* is the whole binding, and a human who removes it upstream detaches the
* document, which is `scope_drift` in the core, not a silent loss.
*
* The tag goes on in a second call because the upload route ignores every
* multipart field except the file. That leaves a window in which the document
* exists untagged; it closes on the next call, and a concurrent listing that
* misses it picks it up on the following run.
*/
async push(conn: DocumentConnectionRef, scope: DocumentScopeRef, req: PushRequest): Promise<DocResult<PushResult>> {
return this.withCreds(conn, async (creds) => {
if (req.remoteId !== undefined) {
throw new PapraError(
'provider_error',
'Papra has no endpoint that replaces the bytes of a document; push a new one instead (replaceInPlace is false)',
);
}
const tag = await this.requireTag(creds, scope);
let document: PapraDocument;
let deduplicated: boolean;
try {
const uploaded = await this.client.uploadDocument(creds, {
body: req.body,
fileName: req.fileName,
mimeType: req.mimeType,
});
document = uploaded.document;
deduplicated = uploaded.deduplicated;
} catch (err: unknown) {
if (!(err instanceof PapraError) || err.papraCode !== 'document.already_exists') throw err;
// Papra refused the bytes because it already holds them. The twin is
// the document this push is about, so it has to be found before the tag
// can go on, and it is found by hash, because a twin stored under
// another name would otherwise pair the wrong document.
const twin = await this.client.findByHash(creds, req.sha256, req.fileName);
if (!twin) {
throw new PapraError(
'conflict',
'Papra holds these bytes already but did not return the document that has them',
err.status,
err.detail,
err.papraCode,
);
}
document = twin;
deduplicated = true;
}
await this.client.addTag(creds, document.id, tag.id);
return {
remoteId: document.id,
remoteVersion: snapshotVersion(document),
remoteModifiedAt: document.updatedAt ?? document.createdAt,
deduplicated,
};
});
}
async rename(
conn: DocumentConnectionRef,
_scope: DocumentScopeRef,
remoteId: string,
name: string,
): Promise<DocResult<{ remoteVersion: string }>> {
return this.withCreds(conn, async (creds) => {
const doc = await this.client.renameDocument(creds, remoteId, name);
return { remoteVersion: snapshotVersion(doc) };
});
}
async trash(conn: DocumentConnectionRef, _scope: DocumentScopeRef, remoteId: string): Promise<DocResult<void>> {
return this.withCreds(conn, async (creds) => {
await this.client.trashDocument(creds, remoteId);
});
}
/** The bound tag, or a `scope_missing` a caller can surface as `scope_lost`. */
private async requireTag(creds: PapraCreds, scope: DocumentScopeRef): Promise<PapraTag> {
const parsed = parseScopeKey(scope.scopeKey);
if (!parsed) {
throw new PapraError('scope_missing', 'The stored scope key is not a Papra scope', undefined, scope.scopeKey);
}
if (parsed.organizationId !== creds.organizationId) {
throw new PapraError(
'scope_missing',
'The bound tag belongs to a different Papra organisation than this connection',
undefined,
`scope ${parsed.organizationId}, connection ${creds.organizationId}`,
);
}
// Papra has no single-tag route, so existence is answered by the list, the
// same call that supplies the name the document search needs.
const tags = await this.client.listTags(creds);
const tag = tags.find((candidate) => candidate.id === parsed.tagId);
if (!tag) {
throw new PapraError('scope_missing', 'The tag this trip is bound to no longer exists', undefined, parsed.tagId);
}
return tag;
}
/** The organisation's name, if the key is allowed to read it. */
private async accountLabel(creds: PapraCreds): Promise<string> {
try {
const organizations = await this.client.listOrganizations(creds);
const mine = organizations.find((org) => org.id === creds.organizationId);
if (mine) return mine.name;
} catch {
// `organizations:read` is a separate scope a working sync key need not
// hold, so its absence costs a nicer label and nothing else.
}
return creds.organizationId;
}
/**
* Resolve the connection's credentials and run one operation on them.
*
* Every operation answers with a result rather than a throw, because the
* settings screen shows a failed probe inline. Reading the credentials inside
* the same guard is what keeps a half-filled connection form on that path
* too: it is reported exactly like a provider fault, and nothing reaches the
* network to find out.
*/
private async withCreds<T>(
conn: DocumentConnectionRef,
run: (creds: PapraCreds) => Promise<T>,
): Promise<DocResult<T>> {
try {
return docOk(await run(readCreds(conn)));
} catch (err: unknown) {
if (err instanceof PapraError) return docFail<T>(err.code, describe(err), err.status);
return docFail<T>('unknown', err instanceof Error ? err.message : String(err));
}
}
}
function toScopeOption(organizationId: string, tag: PapraTag): DocumentScopeOption {
return {
scopeKey: buildScopeKey(organizationId, tag.id),
label: tag.name,
remoteRootId: tag.id,
remoteRootPath: `#${tag.name}`,
};
}
function toRemote(doc: PapraDocument): RemoteDocument {
return {
remoteId: doc.id,
name: doc.name,
size: doc.originalSize,
mimeType: doc.mimeType,
remoteVersion: snapshotVersion(doc),
contentHash: doc.originalSha256Hash,
// In practice the creation time: Papra writes `updatedAt` once and never
// touches it again, not on a rename, a tag change or a delete.
remoteModifiedAt: doc.updatedAt ?? doc.createdAt,
isDeleted: doc.isDeleted,
};
}
/**
* Whether a listed document really carries the bound tag.
*
* The search already filters server-side, so this is belt and braces, except
* for the one case that matters: a tag name the grammar cannot express makes
* the client enumerate the whole organisation, and then this is the only filter
* there is. A document that states no tags at all is kept when the server did
* the filtering, because reading a missing field as "no tags" would empty the
* listing, and an empty listing is how a mass delete looks to the core.
*/
function inScope(doc: PapraDocument, tagId: string, serverFiltered: boolean): boolean {
if (doc.tags === null) return serverFiltered;
return doc.tags.some((tag) => tag.id === tagId);
}
/**
* A fingerprint of the whole enumeration, used as the cursor.
*
* Papra offers nothing to short-circuit a walk with, so this cannot save the
* enumeration. It only lets the core skip diffing when the tag is provably
* unchanged. It covers deletions as well as edits because it is built from the
* complete set rather than from the newest entry.
*/
function enumerationDigest(documents: RemoteDocument[]): string {
const pairs = documents
.map((doc) => `${doc.remoteId}:${doc.remoteVersion}`)
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
return crypto.createHash('sha256').update(pairs.join('\n')).digest('hex');
}
export function buildScopeKey(organizationId: string, tagId: string): string {
return `org:${organizationId}/tag:${tagId}`;
}
export function parseScopeKey(scopeKey: string): { organizationId: string; tagId: string } | null {
const match = /^org:([A-Za-z0-9_-]{1,64})\/tag:([A-Za-z0-9_-]{1,64})$/.exec(scopeKey.trim());
if (!match) return null;
return { organizationId: match[1], tagId: match[2] };
}
/**
* Connection fields to client credentials.
*
* A missing field is answered here rather than as a 401 from Papra, because the
* two look identical to the admin staring at the form and only one of them is
* fixed by issuing a new key.
*/
function readCreds(conn: DocumentConnectionRef): PapraCreds {
const apiKey = conn.secrets.api_key?.trim();
if (!apiKey) throw new PapraError('unauthorized', 'The connection has no Papra API key');
const organizationId = conn.settings.organization_id?.trim();
if (!organizationId) throw new PapraError('provider_error', 'The connection names no Papra organisation');
const baseUrl = conn.baseUrl.trim();
if (!baseUrl) throw new PapraError('provider_error', 'The connection has no Papra URL');
return { baseUrl, apiKey, organizationId, allowInsecureTls: conn.allowInsecureTls };
}
/** Upstream text for the self-hoster's log, never for the user's file list. */
function describe(err: PapraError): string {
const parts = [err.message];
if (err.papraCode) parts.push(`papra=${err.papraCode}`);
if (err.detail) parts.push(err.detail);
return parts.join(' | ');
}
@@ -0,0 +1,225 @@
import { pipeline, Readable, Transform } from 'node:stream';
import type { DocsyncErrorCode } from '@trek/shared';
import { safeFetch, SsrfBlockedError } from '../../../utils/ssrfGuard';
import { discardBody } from '../../../utils/cappedFetch';
/**
* The transport the four document clients share: the SSRF-guarded fetch with
* its timeout and certificate switch, the reading of a request that never got
* an answer, the base status table, and the guard on a streamed download.
*
* What stays in the clients is what differs on purpose: how each one
* authenticates, the shape of its requests (Papra's JSON, WebDAV's methods and
* Depth header, DSM's JSON-encoded form parameters), how it reads an error
* body, and which statuses mean something narrower on that product. Each
* client also keeps its own error class and wording, so the `instanceof`
* checks in the adapters see exactly the errors they always saw.
*/
/** The four ways a request can fail without an HTTP answer. */
export type TransportFailureCode = Extract<
DocsyncErrorCode,
'ssrf_blocked' | 'timeout' | 'tls_untrusted' | 'unreachable'
>;
export interface TransportFailure {
code: TransportFailureCode;
/** Every message in the cause chain, for the self-hoster's log. */
detail: string;
}
export interface ProviderFetchOptions {
timeoutMs: number;
/** The connection's self-signed switch. It relaxes the certificate check only; the SSRF guard still applies. */
allowInsecureTls: boolean;
/** The client's own error for a request that never got an answer. */
onTransportFailure: (failure: TransportFailure) => Error;
}
export interface DownloadGuard {
maxBytes: number;
/** The client's error for a download that announces more than `maxBytes`. */
tooLarge: (declared: number) => Error;
/** And for one that arrives without a body to stream. */
noBody: () => Error;
}
export interface GuardedDownload {
body: Readable;
/** The announced length, or null when the answer did not state one. */
size: number | null;
}
/**
* The ceiling for one download where an adapter has no number of its own.
*
* A runaway guard, not the install's file-size policy: the sync core applies
* that to every byte it pulls, and this sits far above it so that the core's
* verdict is the one a user sees.
*/
export const DOWNLOAD_MAX_BYTES = 2 * 1024 * 1024 * 1024;
/**
* The abort `AbortSignal.timeout` fires, and undici's header and body timers:
* a host that took the request and then went quiet. undici's connect timer is
* left out on purpose. A host that never completed the handshake (switched
* off, or behind a firewall that drops the SYN) did not answer at all, so it
* reads as unreachable.
*/
const TIMEOUT_NAME = /^(TimeoutError|AbortError)$/;
const TIMEOUT_CODE = /ABORT_ERR|HEADERS_TIMEOUT|BODY_TIMEOUT/i;
/**
* Codes that are a verdict on the certificate rather than on the connection,
* which is what lets the settings screen offer the self-signed switch instead
* of a dead end. Most carry CERT or TLS in the name; the verify codes listed
* here do not, and the switch gets past each of them all the same.
*
* ERR_SSL_* is deliberately not among them: a handshake that fails on the
* protocol (https against a plain http port, say) is not something the switch
* can fix, and `tls_untrusted` tells the user it is.
*/
const CERTIFICATE_CODES = new Set([
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
'HOSTNAME_MISMATCH',
'INVALID_PURPOSE',
'INVALID_CA',
'PATH_LENGTH_EXCEEDED',
]);
const CERTIFICATE_CODE = /CERT|SELF_SIGNED|ERR_TLS/i;
/**
* The codes every product answers alike. A client layers its own meaning on
* top through `overrides`, and `fallback` answers whatever neither table knows.
*/
const STATUS_CODES: Readonly<Record<number, DocsyncErrorCode>> = {
401: 'unauthorized',
403: 'forbidden',
404: 'not_found',
409: 'conflict',
413: 'too_large',
415: 'unsupported_type',
429: 'rate_limited',
507: 'quota_exceeded',
};
/** The error and each `cause` below it. undici hides the real reason one or two levels down. */
function causeChain(err: unknown): unknown[] {
const chain: unknown[] = [];
let current: unknown = err;
for (let depth = 0; depth < 5 && current !== undefined && current !== null; depth++) {
chain.push(current);
current = current instanceof Error ? current.cause : undefined;
}
return chain;
}
function codeOf(entry: unknown): string {
return typeof entry === 'object' && entry !== null && 'code' in entry
? String((entry as { code: unknown }).code)
: '';
}
/** Why a request never produced a response. */
export function classifyTransportFailure(err: unknown): TransportFailure {
const chain = causeChain(err);
const detail = chain
.map((entry) => (entry instanceof Error ? entry.message : String(entry)))
.filter((message) => message.length > 0)
.join(': ');
if (err instanceof SsrfBlockedError) return { code: 'ssrf_blocked', detail };
const codes = chain.map(codeOf);
const names = chain.map((entry) => (entry instanceof Error ? entry.name : ''));
if (names.some((name) => TIMEOUT_NAME.test(name)) || codes.some((code) => TIMEOUT_CODE.test(code))) {
return { code: 'timeout', detail };
}
if (codes.some((code) => CERTIFICATE_CODES.has(code) || CERTIFICATE_CODE.test(code))) {
return { code: 'tls_untrusted', detail };
}
return { code: 'unreachable', detail };
}
/**
* One request to a user's instance: through the SSRF guard, bounded by
* `timeoutMs`, with the certificate check relaxed only when the connection
* asks for it. The timeout covers the body as well, so a streamed transfer is
* bounded end to end rather than only until the headers arrive.
*/
export async function providerFetch(url: string, init: RequestInit, options: ProviderFetchOptions): Promise<Response> {
try {
return await safeFetch(
url,
{ ...init, signal: AbortSignal.timeout(options.timeoutMs) as AbortSignal },
{ rejectUnauthorized: !options.allowInsecureTls },
);
} catch (err: unknown) {
throw options.onTransportFailure(classifyTransportFailure(err));
}
}
/** The docsync code for an HTTP status that is not a product-specific answer. */
export function statusErrorCode(
status: number,
overrides: Readonly<Record<number, DocsyncErrorCode>> = {},
fallback: DocsyncErrorCode = 'provider_error',
): DocsyncErrorCode {
return overrides[status] ?? STATUS_CODES[status] ?? fallback;
}
/**
* The byte count a response announced, or null.
*
* An absent header must not become 0: `Number('')` is 0, and a size of zero
* reads as "an empty file" to everything downstream. Compressed and chunked
* answers routinely leave it out.
*/
export function declaredLength(header: string | null): number | null {
if (header === null || header.trim().length === 0) return null;
const value = Number(header);
return Number.isFinite(value) && value >= 0 ? value : null;
}
/**
* Count the bytes as they pass. A declared length is only half a cap: a
* chunked answer declares nothing, and nothing stops a server from sending
* more than it announced.
*/
function limitStream(source: Readable, maxBytes: number): Readable {
let seen = 0;
const limiter = new Transform({
transform(chunk: Buffer, _encoding, done) {
seen += chunk.length;
if (seen > maxBytes) {
done(new Error(`The download exceeded ${maxBytes} bytes`));
return;
}
done(null, chunk);
},
});
// pipeline rather than pipe: a limiter that gives up, or a consumer that
// stops reading, has to take the source down with it. undici keeps the
// socket reserved until the body is released.
pipeline(source, limiter, () => {});
return limiter;
}
/**
* A download handed on as a stream, never buffered: the caller pipes it into
* storage. An announced length over the limit is refused before a byte is
* read, and the bytes are counted on the way through for the answers that
* announce nothing.
*/
export function guardDownload(response: Response, guard: DownloadGuard): GuardedDownload {
const size = declaredLength(response.headers.get('content-length'));
if (size !== null && size > guard.maxBytes) {
discardBody(response);
throw guard.tooLarge(size);
}
if (!response.body) throw guard.noBody();
return {
body: limitStream(Readable.fromWeb(response.body as Parameters<typeof Readable.fromWeb>[0]), guard.maxBytes),
size,
};
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,631 @@
import { Injectable } from '@nestjs/common';
import { createHash, type Hash } from 'node:crypto';
import { Readable } from 'node:stream';
import {
docFail,
docOk,
type DocResult,
type DocumentConnectionRef,
type DocumentProvider,
type DocumentProviderCapabilities,
type DocumentScopeOption,
type DocumentScopeRef,
type FetchResult,
type PushRequest,
type PushResult,
type RemoteDocument,
} from '../document-provider';
import { PROVIDER_MAX_PAGES } from '../doc-sync.constants';
import { contentTypeFor } from '../../storage/content-type';
import {
isValidSynoName,
isWithinScope,
joinSynoPath,
MAX_SCOPE_PATH_LENGTH,
normalizeSynoPath,
snapshotVersion,
SynologyDriveClient,
SynologyDriveError,
type SynoEntry,
type SynologyDriveCreds,
} from './synology-drive.client';
/**
* Synology Drive over the FileStation Web API.
*
* The scope is a folder and identity is its path (FileStation has no file id,
* no change feed and no webhook), so this adapter is the one the sync core's
* "full enumeration, nothing assumed" model was shaped around. Everything it
* cannot honestly promise it declares as absent in `capabilities()` rather than
* faking: no push, no stable id, no hash in the listing.
*
* All HTTP lives in `synology-drive.client.ts`; this file is scope arithmetic,
* the mapping into the sync core's vocabulary, and the boundary checks DSM does
* not do for us.
*/
/** Matches `DOCUMENT_PROVIDER_IDS` in shared/src/docsync/docsync.schema.ts. */
const PROVIDER_ID = 'synologydrive';
/** What the connection form offers as a placeholder, and the fallback here. */
const DEFAULT_BASE_PATH = '/trek';
/** Scope keys are `path:` plus the normalised path. See `docsyncScopeKeySchema`. */
const SCOPE_PREFIX = 'path:';
/**
* DSM exposes no recycle-bin switch on the API's delete, so a propagated delete
* moves the file here instead. It is inside the scope so it inherits the scope's
* permissions, and it is skipped by every listing so a trashed document does not
* come back as a new one on the next run.
*/
const TRASH_FOLDER = '.trek-trash';
/**
* The adapter's own ceiling for one transfer. Not a policy: the install's limit
* is the upload limit (`MAX_FILE_SIZE`), enforced by the core before and during
* every transfer. This is just a bound so a wrong path or a truncated header
* cannot start an unbounded read.
*/
const MAX_TRANSFER_BYTES = 2 * 1024 * 1024 * 1024;
/**
* Up to this size an upload is verified against the NAS's own digest.
*
* SYNO.FileStation.MD5 is a task that reads the whole file, so it is priced per
* megabyte on the NAS's disks. Below the threshold that is a fraction of the
* upload that just happened; above it, the verification would cost more than the
* transfer and the core's own sha256 of the local bytes is the better check.
*/
const MD5_VERIFY_MAX_BYTES = 64 * 1024 * 1024;
/** How many scope options one picker call may answer with. */
const MAX_SCOPE_OPTIONS = 200;
/**
* Where the connection keeps DSM's device token among its secrets. No form
* field has this key, which is what keeps it out of every response and out of
* the form's reach; see `saveEarnedSecret` in the config service.
*/
const DEVICE_TOKEN_SECRET = 'device_token';
const CAPABILITIES: DocumentProviderCapabilities = {
// FileStation has no subscription API at all. DSM's own notifications are
// mail and mobile push, neither reachable from here.
push: 'none',
// The path IS the id: a rename upstream reads as delete + create.
stableId: false,
// Through the move into TRASH_FOLDER below, not through DSM's own recycle bin:
// the API's delete is permanent and offers no flag to route it elsewhere.
remoteTrash: true,
// An upload with overwrite=true replaces the bytes under the same path.
replaceInPlace: true,
// FileStation computes MD5 only, as a task per file, and MD5 is not the
// sha256 the core compares against. `contentHash` therefore stays null.
contentHashInListing: false,
maxUploadBytes: MAX_TRANSFER_BYTES,
// DSM stores whatever it is given; there is no server-side type allow-list.
acceptedMimeTypes: null,
canCreateScope: true,
};
/** Hash the bytes on their way past, without buffering them. */
async function* hashing(source: Readable, hash: Hash): AsyncGenerator<Buffer> {
for await (const chunk of source) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
hash.update(buffer);
yield buffer;
}
}
function baseName(path: string): string {
const cut = path.lastIndexOf('/');
return cut === -1 ? path : path.slice(cut + 1);
}
function parentPath(path: string): string {
const cut = path.lastIndexOf('/');
return cut <= 0 ? '/' : path.slice(0, cut);
}
function isoFromSeconds(seconds: number | null): string | null {
return seconds === null ? null : new Date(seconds * 1000).toISOString();
}
function toRemoteDocument(entry: SynoEntry): RemoteDocument {
return {
remoteId: entry.path,
name: entry.name,
size: entry.size,
// `additional.type` is the extension in capitals, not a media type, so the
// shared mapping is the honest source for what the bytes are.
mimeType: contentTypeFor(entry.name),
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
contentHash: null,
remoteModifiedAt: isoFromSeconds(entry.mtimeSeconds),
// FileStation keeps no tombstone: a file that is gone is simply absent.
isDeleted: false,
};
}
function scopeOption(path: string, label?: string): DocumentScopeOption {
return {
scopeKey: `${SCOPE_PREFIX}${path}`,
label: label ?? baseName(path),
// There is no stable folder id on a Synology share: the path is all there
// is, and pretending otherwise would put a fiction in the binding.
remoteRootId: null,
remoteRootPath: path,
};
}
@Injectable()
export class SynologyDriveDocumentProvider implements DocumentProvider {
readonly id = PROVIDER_ID;
/**
* Injected, like the Paperless and WebDAV clients: it holds the SID cache,
* the lockouts and the device token slots, so there has to be exactly one of
* it per process, and the module's singleton is that one.
*/
constructor(private readonly client: SynologyDriveClient) {}
capabilities(): DocumentProviderCapabilities {
return CAPABILITIES;
}
async probe(
conn: DocumentConnectionRef,
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
try {
const probe = await this.client.probe(creds);
if (!probe.availableApis.includes('SYNO.FileStation.List')) {
return docFail(
'provider_error',
'The account logged in, but this DSM does not expose File Station. Install the File Station package and allow this account to use it.',
);
}
return docOk({ account: creds.username, capabilities: CAPABILITIES });
} catch (error: unknown) {
return this.failure(error);
}
}
/**
* The shares this account can see, plus the folders under the configured base
* path, the two levels an operator actually picks from. Shares are offered
* too because a small NAS often has a share per purpose and no sub-folders.
*/
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
const basePath = this.basePath(conn);
try {
const options = new Map<string, DocumentScopeOption>();
for (const share of await this.client.listShares(creds)) {
const path = normalizeSynoPath(share.path, MAX_SCOPE_PATH_LENGTH);
if (path && share.isdir) options.set(path, scopeOption(path, share.name));
}
// The base path is a convenience, not a requirement: an install that has
// not created it yet should still see its shares rather than an error.
try {
const listing = await this.client.listFolder(creds, basePath);
for (const entry of listing.entries) {
if (!entry.isdir || entry.name === TRASH_FOLDER) continue;
const path = normalizeSynoPath(entry.path, MAX_SCOPE_PATH_LENGTH);
if (path) options.set(path, scopeOption(path, entry.name));
}
} catch (error: unknown) {
if (!(error instanceof SynologyDriveError) || error.code !== 'not_found') throw error;
}
const needle = query?.trim().toLowerCase();
const filtered = [...options.values()]
.filter((option) => !needle || option.label.toLowerCase().includes(needle) || option.remoteRootPath!.toLowerCase().includes(needle))
.sort((a, b) => a.remoteRootPath!.localeCompare(b.remoteRootPath!))
.slice(0, MAX_SCOPE_OPTIONS);
return docOk(filtered);
} catch (error: unknown) {
return this.failure(error);
}
}
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
const trimmed = name.trim();
if (!isValidSynoName(trimmed)) {
return docFail('provider_error', 'DSM will not accept that folder name');
}
const basePath = this.basePath(conn);
const path = normalizeSynoPath(joinSynoPath(basePath, trimmed), MAX_SCOPE_PATH_LENGTH);
if (!path) return docFail('provider_error', 'The resulting folder path is not one DSM accepts');
try {
const folder = await this.client.createFolder(creds, basePath, trimmed);
const created = normalizeSynoPath(folder.path, MAX_SCOPE_PATH_LENGTH) ?? path;
return docOk(scopeOption(created, folder.name));
} catch (error: unknown) {
// With force_parent DSM creates missing folders, but never a missing
// share: that is a storage operation, not a file one. Saying "the base
// path does not exist" is the actionable version of its 408.
if (error instanceof SynologyDriveError && error.code === 'not_found') {
return docFail('scope_missing', `The base path ${basePath} does not exist on the NAS`);
}
return this.failure(error);
}
}
async resolveScope(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<DocumentScopeOption>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
const scopePath = this.scopePath(scope);
if (!scopePath) return docFail('scope_missing', 'The stored scope key is not a usable path');
try {
const entry = await this.client.getInfo(creds, scopePath);
if (!entry.isdir) {
return docFail('scope_missing', 'The bound path is a file on the NAS, not a folder');
}
return docOk(scopeOption(scopePath, entry.name));
} catch (error: unknown) {
if (error instanceof SynologyDriveError && error.code === 'not_found') {
return docFail('scope_missing', `The NAS no longer has ${scopePath}`);
}
return this.failure(error);
}
}
/**
* Every file under the scope, folders walked breadth-first.
*
* Recursive because FileStation's `list` is not, and a sub-folder someone made
* upstream would otherwise be an invisible hole in the trip's documents. One
* request budget covers the whole walk, so a deep tree runs out of budget and
* says `truncated` instead of holding a worker.
*
* The cursor is a digest of the result, not a change token: DSM offers nothing
* to ask "what changed", and a folder's own mtime does not move when a file
* inside it is edited in place. It cannot save the walk, so it is only used to
* tell the core that the walk found exactly what it found last time.
*/
async list(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<{ documents: RemoteDocument[]; cursor: string | null; cursorUnchanged: boolean; truncated: boolean }>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
const scopePath = this.scopePath(scope);
if (!scopePath) return docFail('scope_missing', 'The stored scope key is not a usable path');
const documents: RemoteDocument[] = [];
const folders = [scopePath];
const seen = new Set<string>([scopePath]);
let budget = PROVIDER_MAX_PAGES;
let truncated = false;
try {
while (folders.length > 0) {
if (budget <= 0) {
truncated = true;
break;
}
const folder = folders.shift()!;
const listing = await this.client.listFolder(creds, folder, { maxPages: budget });
budget -= listing.pages;
if (listing.truncated) truncated = true;
for (const entry of listing.entries) {
const path = normalizeSynoPath(entry.path);
// A path the adapter cannot spell is one it could not fetch, rename or
// trash either, so it is left out rather than offered to the core.
if (!path || !isWithinScope(scopePath, path)) continue;
if (entry.isdir) {
if (entry.name === TRASH_FOLDER || seen.has(path)) continue;
seen.add(path);
folders.push(path);
continue;
}
documents.push(toRemoteDocument({ ...entry, path }));
}
}
} catch (error: unknown) {
if (error instanceof SynologyDriveError && error.code === 'not_found') {
return docFail('scope_missing', `The NAS no longer has ${scopePath}`);
}
return this.failure(error);
}
const digest = createHash('sha256');
for (const document of [...documents].sort((a, b) => a.remoteId.localeCompare(b.remoteId))) {
digest.update(`${document.remoteId}\t${document.remoteVersion}\n`);
}
const cursor = `v1:${digest.digest('hex')}`;
return docOk({
documents,
cursor,
// A partial walk saw a partial tree, so its digest says nothing about
// whether anything changed.
cursorUnchanged: !truncated && scope.cursor === cursor,
truncated,
});
}
async fetch(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<FetchResult>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
try {
const path = this.requireInScope(scope, remoteId);
// The version is read before the bytes so it describes the file that is
// about to be streamed. Reading it after would describe whatever the file
// became while it was being read.
const entry = await this.client.getInfo(creds, path);
if (entry.isdir) return docFail('not_found', 'That path is a folder on the NAS');
const download = await this.client.download(creds, path, MAX_TRANSFER_BYTES);
return docOk({
body: download.body,
size: download.size ?? entry.size,
mimeType: contentTypeFor(entry.name),
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
});
} catch (error: unknown) {
return this.failure(error);
}
}
/**
* Upload, then confirm what landed.
*
* The confirmation is not ceremony: FileStation answers `success: true` as
* soon as it has written the part, and the version the core stores has to
* describe the file as the NAS now sees it (including the mtime DSM rounded
* to whole seconds), or the next listing reads TREK's own upload as a change
* made upstream.
*/
async push(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
req: PushRequest,
): Promise<DocResult<PushResult>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
if (req.size > MAX_TRANSFER_BYTES) {
return docFail('too_large', `This adapter refuses transfers over ${MAX_TRANSFER_BYTES} bytes`);
}
const md5 = createHash('md5');
try {
const scopePath = this.requireScopePath(scope);
let folderPath = scopePath;
let fileName = req.fileName;
if (req.remoteId !== undefined) {
const replacing = this.requireInScope(scope, req.remoteId);
folderPath = parentPath(replacing);
// The name follows the path being replaced, not the request: changing
// both the bytes and the name in one upload would leave the old path
// behind as a second document.
fileName = baseName(replacing);
}
if (!isValidSynoName(fileName)) {
return docFail('provider_error', 'DSM will not accept that file name');
}
const targetPath = joinSynoPath(folderPath, fileName);
if (normalizeSynoPath(targetPath) !== targetPath || !isWithinScope(scopePath, targetPath)) {
return docFail('provider_error', 'The resulting file path is not one DSM accepts');
}
await this.client.upload(creds, {
folderPath,
fileName,
body: Readable.from(hashing(req.body, md5)),
size: req.size,
mimeType: req.mimeType,
mtimeSeconds: req.mtimeSeconds,
// Replacing is only ever what the caller asked for. Without a remoteId a
// name collision is a real conflict: the file at that path is somebody
// else's, and overwriting it would destroy it silently.
overwrite: req.remoteId !== undefined,
createParents: true,
});
const entry = await this.client.getInfo(creds, targetPath);
if (entry.size !== null && entry.size !== req.size) {
return docFail(
'checksum_mismatch',
`The NAS stored ${entry.size} bytes of an ${req.size}-byte upload`,
);
}
if (req.size <= MD5_VERIFY_MAX_BYTES) {
const remote = await this.client.md5(creds, targetPath);
if (remote !== md5.digest('hex')) {
return docFail('checksum_mismatch', 'The NAS computed a different digest for the uploaded bytes');
}
}
return docOk({
remoteId: entry.path,
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
remoteModifiedAt: isoFromSeconds(entry.mtimeSeconds),
// FileStation stores what it is given; there is no content-addressed
// store behind it that could answer with an existing file.
deduplicated: false,
});
} catch (error: unknown) {
return this.failure(error);
}
}
/**
* Rename in place.
*
* The caller's `remoteId` is stale afterwards, because the path is the id.
* That is why `capabilities().stableId` is false: the core is expected to
* re-derive the id from the folder and the new name, or to find it on the next
* enumeration.
*/
async rename(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
name: string,
): Promise<DocResult<{ remoteVersion: string }>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
const trimmed = name.trim();
if (!isValidSynoName(trimmed)) {
return docFail('provider_error', 'DSM will not accept that file name');
}
try {
const entry = await this.client.rename(creds, this.requireInScope(scope, remoteId), trimmed);
return docOk({ remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds) });
} catch (error: unknown) {
return this.failure(error);
}
}
/**
* Move into the scope's own bin.
*
* DSM's API delete is permanent and takes no "to the recycle bin" flag, and a
* recycle bin is not even enabled on every share. A document that four people
* are travelling on must not be one API call away from gone, so a propagated
* delete becomes a move that any of them can undo in File Station.
*/
async trash(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<void>> {
const creds = this.credentials(conn);
if (!creds) return docFail('unauthorized', 'username and password are required');
try {
const scopePath = this.requireScopePath(scope);
const path = this.requireInScope(scope, remoteId);
const trashPath = joinSynoPath(scopePath, TRASH_FOLDER);
await this.client.createFolder(creds, scopePath, TRASH_FOLDER);
try {
await this.client.move(creds, path, trashPath, { overwrite: false });
} catch (error: unknown) {
if (!(error instanceof SynologyDriveError) || error.code !== 'conflict') throw error;
// A file of that name is already in the bin. Overwriting it would throw
// away the older copy, which is the one thing a bin exists to prevent,
// so the older copy steps aside under a timestamped name.
const occupant = joinSynoPath(trashPath, baseName(path));
await this.client.rename(creds, occupant, this.archivedName(baseName(path)));
await this.client.move(creds, path, trashPath, { overwrite: false });
}
return docOk(undefined);
} catch (error: unknown) {
return this.failure(error);
}
}
/** `boarding.pdf` → `boarding.1789745537.pdf`, keeping the extension readable. */
private archivedName(name: string): string {
const stamp = Math.floor(Date.now() / 1000);
const dot = name.lastIndexOf('.');
return dot <= 0 ? `${name}.${stamp}` : `${name.slice(0, dot)}.${stamp}${name.slice(dot)}`;
}
private credentials(conn: DocumentConnectionRef): SynologyDriveCreds | null {
const username = (conn.settings.username ?? '').trim();
const password = conn.secrets.password ?? '';
if (!username || !password || !conn.baseUrl.trim()) return null;
const otpCode = (conn.secrets.otp_code ?? '').trim();
const save = conn.saveSecret;
return {
connectionId: conn.connectionId,
connectionCreatedAt: conn.createdAt,
baseUrl: conn.baseUrl,
username,
password,
otpCode: otpCode === '' ? undefined : otpCode,
storedDeviceToken: conn.secrets[DEVICE_TOKEN_SECRET],
saveDeviceToken: save ? (stored) => save(DEVICE_TOKEN_SECRET, stored) : undefined,
allowInsecureTls: conn.allowInsecureTls,
};
}
private basePath(conn: DocumentConnectionRef): string {
const configured = (conn.settings.base_path ?? '').trim();
if (!configured) return DEFAULT_BASE_PATH;
return normalizeSynoPath(configured, MAX_SCOPE_PATH_LENGTH) ?? DEFAULT_BASE_PATH;
}
/**
* The folder this trip is bound to, from the scope key.
*
* `remoteRootPath` is only a fallback: the key is what a unique index protects
* and what a restore keeps, so a row whose two columns disagree is read the
* way the constraint reads it.
*/
private scopePath(scope: DocumentScopeRef): string | null {
const raw = scope.scopeKey.startsWith(SCOPE_PREFIX)
? scope.scopeKey.slice(SCOPE_PREFIX.length)
: scope.remoteRootPath;
if (!raw) return null;
const path = normalizeSynoPath(raw, MAX_SCOPE_PATH_LENGTH);
// The FileStation root is not a scope: it holds the shares, and binding a
// trip to it would put every share on the NAS inside one trip.
return path === null || path === '/' ? null : path;
}
/**
* A remote id turned into a path this call is allowed to touch.
*
* DSM checks DSM permissions and nothing else: the account TREK logs in with
* can reach every share it owns, so `/trips/japan/../../payroll/2026.pdf` is a
* perfectly legal request that another trip's binding must never be able to
* make. The trip boundary exists only here.
*
* Throws rather than returning a result, so a caller cannot forget to check
* it. Every caller already runs inside the try that turns a
* `SynologyDriveError` into a `DocResult`.
*/
private requireInScope(scope: DocumentScopeRef, remoteId: string): string {
const scopePath = this.requireScopePath(scope);
const path = normalizeSynoPath(remoteId);
if (!path || path === scopePath || !isWithinScope(scopePath, path)) {
throw new SynologyDriveError('not_found', 'That path is outside the folder this trip is bound to', {
detail: 'That path is outside the folder this trip is bound to',
});
}
if (path.split('/').includes(TRASH_FOLDER)) {
throw new SynologyDriveError('not_found', 'That path is inside the sync bin', {
detail: 'That path is inside the sync bin',
});
}
return path;
}
private requireScopePath(scope: DocumentScopeRef): string {
const scopePath = this.scopePath(scope);
if (!scopePath) {
throw new SynologyDriveError('scope_missing', 'The stored scope key is not a usable path', {
detail: 'The stored scope key is not a usable path',
});
}
return scopePath;
}
private failure<T>(error: unknown): DocResult<T> {
if (error instanceof SynologyDriveError) {
return docFail<T>(error.code, error.detail, error.status);
}
return docFail<T>('unknown', error instanceof Error ? error.message : undefined);
}
}
@@ -0,0 +1,782 @@
import { Injectable } from '@nestjs/common';
import { Readable } from 'node:stream';
import { XMLParser } from 'fast-xml-parser';
import type { DocsyncErrorCode } from '@trek/shared';
import { readCappedJson, readCappedText } from '../../../utils/cappedFetch';
import {
PROVIDER_JSON_MAX_BYTES,
PROVIDER_TIMEOUT_MS,
PROVIDER_TRANSFER_TIMEOUT_MS,
} from '../doc-sync.constants';
import { DOWNLOAD_MAX_BYTES, guardDownload, providerFetch, statusErrorCode } from './provider-http';
/**
* HTTP client for Nextcloud and OpenCloud. This is the ONLY place that talks to
* either instance; the adapter above it holds no URLs and no transport logic.
*
* Measured against Nextcloud 31.0.14 and OpenCloud 8.0.1 on 2026-09-18, not
* against their prose documentation, which is silent or wrong on most of this:
*
* - **`OC-Checksum` is a Nextcloud-only header here.** OpenCloud answers HTTP
* 400 to `SHA256:<hex>`: it knows SHA1, MD5 and ADLER32, and treats an
* unknown algorithm exactly like a mismatch, so sending the sha256 TREK
* already has would fail every upload into a space. Nextcloud accepts SHA256
* but does NOT verify it: `SHA256:0000` is stored verbatim and served back
* through `oc:checksums`. A checksum from a listing is therefore a claim,
* never a proof, and the core still has to hash the bytes it downloads.
* - **OpenCloud computes SHA1/MD5/ADLER32 itself** for every file, whether or
* not one was sent, and never SHA256. That is why `contentHashInListing` is
* false for both products rather than true for one.
* - **`X-OC-MTime` is honoured by both**, answered with `X-OC-MTime: accepted`.
* Without it every file TREK uploads comes back on the next run looking
* freshly changed upstream, which is the classic sync loop.
* - **Nextcloud's `OC-FileId` response header is not its `oc:fileid`**: the PUT
* that creates `oc:fileid` 62 answers `OC-FileId: 00000062oc9fy1g5e2cj`
* (zero-padded id plus instance id). Storing the header form would make every
* file TREK uploaded look like a stranger on the next listing, so it is
* normalised back to the numeric id. OpenCloud's two spellings agree.
* - **PROPFIND has no pagination.** `Paginate: true` plus `Nc-Paginate-*` is
* advertised in the `DAV:` header (`nc-paginate`) and ignored on PROPFIND, so
* one Depth:1 request is the whole folder. Hence the capped read and the
* repair of a cut-off body into the responses that did arrive, with
* `truncated` set, rather than failing a big folder outright.
* - **Webhook registration is form-encoded only.** A JSON body reaches the
* controller with every argument null and the instance answers HTTP 500.
*/
/** Which product a connection points at. Decided by the connection, not by the class. */
export type WebdavFlavor = 'nextcloud' | 'opencloud';
export interface WebdavCreds {
/** Instance origin, already normalised: no trailing slash, no DAV suffix. */
origin: string;
username: string;
password: string;
allowInsecureTls: boolean;
flavor: WebdavFlavor;
}
/**
* A failed call, carrying the reason as a code.
*
* The code is what reaches the user through i18n; `detail` keeps the upstream
* text, which is English, sometimes an HTML login page from a reverse proxy, and
* in Sabre's case a stack-shaped XML document.
*/
export class WebdavError extends Error {
readonly code: DocsyncErrorCode;
readonly status?: number;
readonly detail?: string;
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string) {
super(message);
this.name = 'WebdavError';
this.code = code;
this.status = status;
this.detail = detail;
}
}
/** One `<d:response>` of a PROPFIND, reduced to what the sync core reads. */
export interface WebdavEntry {
/** Path portion of the `d:href`, still percent-encoded, as sent. */
href: string;
/** The same path decoded, for comparing against a requested path. */
path: string;
name: string;
isCollection: boolean;
/** Quoted ETag exactly as the server spelled it, or '' when absent. */
etag: string;
fileId: string | null;
size: number | null;
mimeType: string | null;
lastModifiedIso: string | null;
/** Only when the instance carries a SHA256; see the header note above. */
sha256: string | null;
}
export interface WebdavPropfindResult {
/** The requested resource itself. */
self: WebdavEntry;
children: WebdavEntry[];
/** The body hit the size cap, so `children` is a prefix of the folder. */
truncated: boolean;
}
/** An OpenCloud space, as `/graph/v1.0/…/drives` describes it. */
export interface WebdavDrive {
id: string;
name: string;
driveType: string;
/**
* Path of `root.webDavUrl`, never its origin: the instance answers with its
* own configured `OC_URL` (`https://localhost:8804` for a host reached as
* 127.0.0.1), and following a host that an untrusted response chose is how an
* SSRF guard gets walked around. Only the path survives, re-based on the
* connection's own origin.
*/
webDavPath: string;
}
export interface WebdavPutOptions {
size: number;
mimeType: string;
/** Unix seconds, sent as `X-OC-MTime`. */
mtimeSeconds: number;
sha256: string;
/** Optimistic concurrency. Both products answer a mismatch with 412. */
ifMatch?: string;
/** `*` refuses to create over an existing resource (RFC 9110 §13.1.2). */
ifNoneMatch?: string;
}
export interface WebdavPutResult {
fileId: string | null;
etag: string | null;
/** 201 rather than 204: a new resource rather than a replaced one. */
created: boolean;
}
export interface WebdavDownload {
body: Readable;
size: number | null;
mimeType: string | null;
etag: string;
}
const PROPFIND_BODY =
'<?xml version="1.0" encoding="UTF-8"?>' +
'<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop>' +
'<d:getetag/><d:getlastmodified/><d:getcontentlength/><d:getcontenttype/><d:resourcetype/>' +
'<oc:fileid/><oc:checksums/>' +
'</d:prop></d:propfind>';
/**
* Namespace for the dead properties TREK writes. A URI, not an address: it is
* never fetched, it only has to be unmistakably ours so a second tool writing
* `doc-uid` on the same file cannot collide with it.
*/
export const TREK_PROP_NS = 'https://github.com/liketrek/TREK/ns/docsync';
/** Property names under that namespace, matching the anchors in `PushRequest`. */
export const TREK_PROP_DOC_UID = 'doc-uid';
export const TREK_PROP_TRIP_UID = 'trip-uid';
const OCS_WEBHOOKS = '/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks';
/** OCS refuses any request without the first header and answers XML without the second. */
const OCS_HEADERS: Record<string, string> = {
'OCS-APIRequest': 'true',
Accept: 'application/json',
};
/**
* `removeNSPrefix` because the prefix is the server's choice rather than the
* protocol's (`d:` on both of these, `D:` on other Sabre builds), and
* `parseTagValue: false` because an ETag, a file id and a checksum are opaque
* strings that have to survive a server spelling them as digits.
*/
const propfindParser = new XMLParser({
removeNSPrefix: true,
parseTagValue: false,
ignoreAttributes: true,
isArray: (name) => name === 'response' || name === 'propstat' || name === 'checksum',
});
/** Nextcloud's `OC-FileId` header carries the instance id after the number. */
export function normalizeFileId(raw: string | null, flavor: WebdavFlavor): string | null {
const value = raw?.trim();
if (!value) return null;
if (flavor !== 'nextcloud') return value;
const digits = /^0*(\d+)/.exec(value);
return digits ? digits[1] : value;
}
/**
* What a user pastes into the URL field, reduced to an origin.
*
* People paste the address bar of their own file list, so the DAV and Graph
* prefixes come off rather than turning into `/remote.php/dav/remote.php/dav`
* 404s, which read like a wrong password.
*/
export function normalizeOrigin(baseUrl: string): string {
return baseUrl
.trim()
.replace(/[?#].*$/, '')
.replace(/\/+$/, '')
.replace(/\/(?:remote|index)\.php(?:\/.*)?$/i, '')
.replace(/\/dav\/spaces(?:\/.*)?$/i, '')
.replace(/\/graph(?:\/.*)?$/i, '')
.replace(/\/+$/, '');
}
/** Percent-encode each segment of a decoded path, keeping the separators. */
export function encodePath(path: string): string {
return path
.split('/')
.map((segment) => encodeURIComponent(segment))
.join('/');
}
/** `/a/b/` and `a//b` both mean `/a/b`. Every comparison here depends on one spelling. */
export function normalizePath(path: string): string {
const collapsed = `/${path}`.replace(/\/+/g, '/').replace(/\/+$/, '');
return collapsed === '' ? '/' : collapsed;
}
/** The path portion of a stored root, which is a URL on OpenCloud and a path on Nextcloud. */
export function davPathOf(rootPathOrUrl: string): string | null {
const value = rootPathOrUrl.trim();
if (!value) return null;
if (/^https?:\/\//i.test(value)) {
try {
return normalizePath(new URL(value).pathname);
} catch {
return null;
}
}
return value.startsWith('/') ? normalizePath(value) : null;
}
function hrefPathname(href: string): string {
try {
return new URL(href).pathname;
} catch {
return href;
}
}
/** The still-encoded path of an href, which is what the next request is built from. */
function hrefPath(href: string): string {
return /^https?:\/\//i.test(href) ? hrefPathname(href) : href;
}
function decodeHrefPath(href: string): string {
const raw = hrefPath(href);
try {
return normalizePath(decodeURIComponent(raw));
} catch {
return normalizePath(raw);
}
}
/**
* 405 is MKCOL on an existing collection, 412 a failed If-Match, 423 a locked
* node: with 409, four ways of saying the caller's picture of the remote is
* stale.
*/
const STALE_STATUSES: Readonly<Record<number, DocsyncErrorCode>> = {
405: 'conflict',
412: 'conflict',
423: 'conflict',
};
function classifyStatus(status: number): DocsyncErrorCode {
return statusErrorCode(status, STALE_STATUSES, status >= 500 ? 'provider_error' : 'unknown');
}
/**
* A body cut off at the cap is still a run of complete `<d:response>` blocks
* followed by a fragment. Dropping the fragment and closing the document keeps
* the first few thousand files of a huge folder usable, which is what
* `truncated` is for. The alternative is telling someone with one big folder
* that nothing works at all.
*/
function repairTruncatedMultistatus(xml: string): string | null {
const closing = /<\/([a-z0-9]+:)?response\s*>/gi;
let cut = -1;
for (let match = closing.exec(xml); match !== null; match = closing.exec(xml)) {
cut = match.index + match[0].length;
}
if (cut < 0) return null;
const open = /<([a-z0-9]+:)?multistatus[^>]*>/i.exec(xml);
if (!open) return null;
return `${xml.slice(0, cut)}</${open[1] ?? ''}multistatus>`;
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: null;
}
function asArray(value: unknown): unknown[] {
if (Array.isArray(value)) return value;
return value === undefined || value === null ? [] : [value];
}
function textOf(value: unknown): string | null {
if (typeof value === 'string') return value;
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
return null;
}
/**
* Props out of the propstat blocks that reported 2xx only. A PROPFIND answers
* one block per status, and reading the 404 block too turns "this folder has no
* content type" into an empty string that looks like a real value.
*/
function propsOf(response: Record<string, unknown>): Record<string, unknown> {
const props: Record<string, unknown> = {};
for (const stat of asArray(response.propstat)) {
const record = asRecord(stat);
if (!record) continue;
const status = textOf(record.status) ?? '';
if (!/\s2\d\d(\s|$)/.test(status)) continue;
const prop = asRecord(record.prop);
if (prop) Object.assign(props, prop);
}
return props;
}
/**
* `SHA256:<hex>` out of either spelling: Nextcloud sends one element carrying
* one algorithm, OpenCloud sends one element carrying all three separated by
* spaces.
*/
function sha256Of(checksums: unknown): string | null {
const record = asRecord(checksums);
if (!record) return null;
for (const entry of asArray(record.checksum)) {
const text = textOf(entry);
if (!text) continue;
for (const token of text.split(/\s+/)) {
const match = /^SHA256:([0-9a-f]{64})$/i.exec(token);
if (match) return match[1].toLowerCase();
}
}
return null;
}
function entryOf(response: Record<string, unknown>, flavor: WebdavFlavor): WebdavEntry | null {
const href = textOf(response.href);
if (!href) return null;
const props = propsOf(response);
const path = decodeHrefPath(href);
const segments = path.split('/');
const length = textOf(props.getcontentlength);
const modified = textOf(props.getlastmodified);
const parsedModified = modified ? new Date(modified) : null;
const mimeType = textOf(props.getcontenttype);
const resourceType = asRecord(props.resourcetype);
return {
href: hrefPath(href),
path,
name: segments[segments.length - 1] ?? '',
isCollection: resourceType !== null && 'collection' in resourceType,
etag: textOf(props.getetag) ?? '',
fileId: normalizeFileId(textOf(props.fileid), flavor),
size: length !== null && length !== '' && Number.isFinite(Number(length)) ? Number(length) : null,
mimeType: mimeType ? mimeType.split(';')[0].trim() : null,
lastModifiedIso:
parsedModified && !Number.isNaN(parsedModified.getTime()) ? parsedModified.toISOString() : null,
sha256: sha256Of(props.checksums),
};
}
function toDrive(value: unknown): WebdavDrive | null {
const record = asRecord(value);
const id = textOf(record?.id);
if (!record || !id) return null;
const webDavUrl = textOf(asRecord(record.root)?.webDavUrl);
return {
id,
name: textOf(record.name) ?? id,
driveType: textOf(record.driveType) ?? '',
webDavPath: (webDavUrl ? davPathOf(webDavUrl) : null) ?? `/dav/spaces/${encodeURIComponent(id)}`,
};
}
/** Text content for the PROPPATCH body, which is built as a string rather than parsed back. */
function escapeXmlText(value: string): string {
return value
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;');
}
@Injectable()
export class WebdavClient {
/**
* One authenticated request. Throws for anything that never reached the
* application, and hands the caller the response otherwise: a 404 on DELETE
* and a 405 on MKCOL are answers, not failures.
*/
private async send(
creds: WebdavCreds,
path: string,
init: RequestInit,
timeoutMs = PROVIDER_TIMEOUT_MS,
): Promise<Response> {
const auth = Buffer.from(`${creds.username}:${creds.password}`, 'utf8').toString('base64');
const headers = new Headers(init.headers);
headers.set('Authorization', `Basic ${auth}`);
return providerFetch(`${creds.origin}${path}`, { ...init, headers }, {
timeoutMs,
allowInsecureTls: creds.allowInsecureTls,
onTransportFailure: ({ code, detail }) =>
new WebdavError(code, 'Could not reach the WebDAV instance', undefined, detail),
});
}
/** The error for a response nobody wanted, with a bounded slice of its body as detail. */
private async failed(response: Response, what: string): Promise<WebdavError> {
const { text } = await readCappedText(response, 8 * 1024);
return new WebdavError(
classifyStatus(response.status),
`${what} answered HTTP ${response.status}`,
response.status,
text.slice(0, 500),
);
}
/**
* PROPFIND with the props the sync core reads.
*
* Depth 0 is the cursor probe: a write anywhere below a collection changes
* that collection's ETag on both products, up to the root, so an unchanged
* root ETag means the whole walk can be skipped. Depth 1 is the walk.
*/
async propfind(creds: WebdavCreds, path: string, depth: 0 | 1): Promise<WebdavPropfindResult> {
const response = await this.send(creds, path, {
method: 'PROPFIND',
headers: { Depth: String(depth), 'Content-Type': 'application/xml; charset=utf-8' },
body: PROPFIND_BODY,
});
if (response.status !== 207) {
throw await this.failed(response, 'PROPFIND');
}
const { text, truncated } = await readCappedText(response, PROVIDER_JSON_MAX_BYTES);
const xml = truncated ? repairTruncatedMultistatus(text) : text;
if (xml === null) {
throw new WebdavError('too_large', 'The folder listing exceeded the size limit', response.status);
}
const multistatus = asRecord(asRecord(propfindParser.parse(xml))?.multistatus);
if (!multistatus) {
throw new WebdavError(
'provider_error',
'The instance answered something that is not a WebDAV multistatus',
response.status,
text.slice(0, 200),
);
}
const entries: WebdavEntry[] = [];
for (const element of asArray(multistatus.response)) {
const record = asRecord(element);
const entry = record ? entryOf(record, creds.flavor) : null;
if (entry) entries.push(entry);
}
if (entries.length === 0) {
throw new WebdavError('provider_error', 'The instance answered an empty multistatus', response.status);
}
// RFC 4918 puts the requested resource first, but it is matched by path
// rather than trusted by position: the root must never end up in the
// document list, and one misordered server would put it there.
const wanted = decodeHrefPath(path);
const selfIndex = Math.max(
entries.findIndex((entry) => entry.path === wanted),
0,
);
return {
self: entries[selfIndex],
children: entries.filter((_, index) => index !== selfIndex),
truncated,
};
}
/**
* Write dead properties, which both products persist (Nextcloud in
* `oc_properties`, OpenCloud in the space's metadata) and hand back through a
* PROPFIND that asks for them. That is where the TREK anchors go: they are the
* only thing that still identifies a document after a human has renamed it and
* moved it somewhere else in the folder.
*
* Answers whether the server accepted them. A 207 is not enough on its own:
* a refusal arrives as a non-2xx status inside the multistatus.
*/
async proppatch(creds: WebdavCreds, path: string, props: Record<string, string>): Promise<boolean> {
const body =
'<?xml version="1.0" encoding="UTF-8"?>' +
`<d:propertyupdate xmlns:d="DAV:" xmlns:trek="${TREK_PROP_NS}"><d:set><d:prop>` +
Object.entries(props)
.map(([name, value]) => `<trek:${name}>${escapeXmlText(value)}</trek:${name}>`)
.join('') +
'</d:prop></d:set></d:propertyupdate>';
const response = await this.send(creds, path, {
method: 'PROPPATCH',
headers: { 'Content-Type': 'application/xml; charset=utf-8' },
body,
});
if (response.status !== 207) {
throw await this.failed(response, 'PROPPATCH');
}
const { text } = await readCappedText(response, PROVIDER_JSON_MAX_BYTES);
const multistatus = asRecord(asRecord(propfindParser.parse(text))?.multistatus);
for (const element of asArray(multistatus?.response)) {
const record = asRecord(element);
if (!record) continue;
const accepted = propsOf(record);
if (Object.keys(props).every((name) => name in accepted)) return true;
}
return false;
}
/** MKCOL. `exists` rather than an error, because 405 is how both say "already there". */
async mkcol(creds: WebdavCreds, path: string): Promise<'created' | 'exists'> {
const response = await this.send(creds, path, { method: 'MKCOL' });
void response.body?.cancel().catch(() => {});
if (response.ok) return 'created';
if (response.status === 405) return 'exists';
throw await this.failed(response, 'MKCOL');
}
async put(
creds: WebdavCreds,
path: string,
body: Readable,
options: WebdavPutOptions,
): Promise<WebdavPutResult> {
const headers: Record<string, string> = {
'Content-Type': options.mimeType,
// Explicit, so the upload does not go out chunked: a reverse proxy in
// front of a self-hosted instance is the common case and several refuse
// a chunked PUT outright.
'Content-Length': String(options.size),
'X-OC-MTime': String(options.mtimeSeconds),
};
if (creds.flavor === 'nextcloud' && /^[0-9a-f]{64}$/i.test(options.sha256)) {
headers['OC-Checksum'] = `SHA256:${options.sha256.toLowerCase()}`;
}
if (options.ifNoneMatch) {
headers['If-None-Match'] = options.ifNoneMatch;
}
if (options.ifMatch) {
headers['If-Match'] = options.ifMatch;
}
const response = await this.send(
creds,
path,
{ method: 'PUT', headers, body, duplex: 'half' },
PROVIDER_TRANSFER_TIMEOUT_MS,
);
if (!response.ok) {
throw await this.failed(response, 'PUT');
}
void response.body?.cancel().catch(() => {});
return {
fileId: normalizeFileId(response.headers.get('oc-fileid'), creds.flavor),
etag: response.headers.get('oc-etag') ?? response.headers.get('etag'),
created: response.status === 201,
};
}
/**
* GET, handed back as a stream.
*
* Never buffered: the caller pipes it into storage, and holding a trip's
* scanned passport in memory to count its bytes is the opposite of what a
* cap is for. The guard counts them on the way through instead.
*/
async get(creds: WebdavCreds, path: string): Promise<WebdavDownload> {
const response = await this.send(creds, path, { method: 'GET' }, PROVIDER_TRANSFER_TIMEOUT_MS);
if (!response.ok) {
throw await this.failed(response, 'GET');
}
const { body, size } = guardDownload(response, {
maxBytes: DOWNLOAD_MAX_BYTES,
tooLarge: (declared) =>
new WebdavError(
'too_large',
'The document is larger than TREK will transfer',
response.status,
`content_length=${declared}`,
),
noBody: () =>
new WebdavError('provider_error', 'The instance answered a download without a body', response.status),
});
const mimeType = response.headers.get('content-type');
return {
body,
size,
mimeType: mimeType ? mimeType.split(';')[0].trim() : null,
etag: response.headers.get('oc-etag') ?? response.headers.get('etag') ?? '',
};
}
/**
* MOVE within one instance. `Overwrite: F` on purpose: a rename that silently
* replaced a namesake would destroy a document nobody asked about, and the
* 412 it produces instead is a conflict the core can show someone.
*/
async move(creds: WebdavCreds, fromPath: string, toPath: string): Promise<{ etag: string | null }> {
const response = await this.send(creds, fromPath, {
method: 'MOVE',
headers: { Destination: `${creds.origin}${toPath}`, Overwrite: 'F' },
});
if (!response.ok) {
throw await this.failed(response, 'MOVE');
}
void response.body?.cancel().catch(() => {});
return { etag: response.headers.get('oc-etag') ?? response.headers.get('etag') };
}
/** DELETE, which lands in the owner's trash on Nextcloud and the space trash on OpenCloud. */
async delete(creds: WebdavCreds, path: string): Promise<void> {
const response = await this.send(creds, path, { method: 'DELETE' });
void response.body?.cancel().catch(() => {});
// A document that is already gone is the state the caller asked for.
if (response.ok || response.status === 404) return;
throw await this.failed(response, 'DELETE');
}
private async graph(
creds: WebdavCreds,
method: 'GET' | 'POST',
path: string,
body?: unknown,
): Promise<unknown> {
const response = await this.send(creds, path, {
method,
headers:
body === undefined
? { Accept: 'application/json' }
: { Accept: 'application/json', 'Content-Type': 'application/json' },
body: body === undefined ? undefined : JSON.stringify(body),
});
if (!response.ok) {
throw await this.failed(response, 'The Graph API');
}
const parsed = await readCappedJson<unknown>(response, PROVIDER_JSON_MAX_BYTES);
if (parsed === undefined) {
throw new WebdavError(
'provider_error',
'The Graph API answered something that is not JSON. Check that the URL points at OpenCloud itself.',
response.status,
);
}
return parsed;
}
async listDrives(creds: WebdavCreds): Promise<WebdavDrive[]> {
const parsed = asRecord(await this.graph(creds, 'GET', '/graph/v1.0/me/drives'));
const drives: WebdavDrive[] = [];
for (const value of asArray(parsed?.value)) {
const drive = toDrive(value);
if (drive) drives.push(drive);
}
return drives;
}
/** One space, or null when it is gone, which is what makes `scope_missing` detectable. */
async getDrive(creds: WebdavCreds, driveId: string): Promise<WebdavDrive | null> {
try {
return toDrive(await this.graph(creds, 'GET', `/graph/v1.0/drives/${encodeURIComponent(driveId)}`));
} catch (err: unknown) {
if (err instanceof WebdavError && err.status === 404) return null;
throw err;
}
}
async createDrive(creds: WebdavCreds, name: string): Promise<WebdavDrive> {
const drive = toDrive(
await this.graph(creds, 'POST', '/graph/v1.0/drives', { name, driveType: 'project' }),
);
if (!drive) {
throw new WebdavError('provider_error', 'OpenCloud created a space it then described incompletely');
}
return drive;
}
/** The account's own name, for the line the settings screen shows after a test. */
async whoAmI(creds: WebdavCreds): Promise<string> {
const me = asRecord(await this.graph(creds, 'GET', '/graph/v1.0/me'));
return (
textOf(me?.onPremisesSamAccountName) ??
textOf(me?.displayName) ??
textOf(me?.mail) ??
creds.username
);
}
/**
* Whether this account may manage webhooks at all.
*
* The endpoint is admin-only and a read is harmless, so asking is cheaper and
* more honest than registering four subscriptions to find out. A non-admin
* app password gets 403 here, which is exactly the case `webhook-manual`
* exists for.
*/
async canRegisterWebhooks(creds: WebdavCreds): Promise<boolean> {
const response = await this.send(creds, OCS_WEBHOOKS, { method: 'GET', headers: OCS_HEADERS });
void response.body?.cancel().catch(() => {});
return response.ok;
}
/**
* Register one event, unfiltered.
*
* Form-encoded, because a JSON body arrives at the controller with every
* argument null and the instance answers HTTP 500.
*
* No `eventFilter`, and that is measured rather than lazy. The filter is
* evaluated by the app's own bundled PHPMongoQuery, which implements
* `$all $e $in $lt $lte $gt $gte $ne $nin $exists $mod` and nothing else, so a
* `$regex` prefix match on the changed path throws "Operator $regex is
* unknown" inside the event listener. The damage does not stop at this
* subscription: the throw aborts the listener for that event, so one filtered
* webhook silently stops delivery for every other webhook on the same event,
* including ones belonging to other applications. An unfiltered subscription
* that occasionally says "look now" about an unrelated folder is the cheap
* mistake; the filter is the expensive one.
*/
async createWebhook(
creds: WebdavCreds,
options: { uri: string; event: string; secretHeader: string; secret: string },
): Promise<number> {
const form = new URLSearchParams();
form.set('httpMethod', 'POST');
form.set('uri', options.uri);
form.set('event', options.event);
form.set('authMethod', 'header');
form.set(`authData[${options.secretHeader}]`, options.secret);
const response = await this.send(creds, OCS_WEBHOOKS, {
method: 'POST',
headers: { ...OCS_HEADERS, 'Content-Type': 'application/x-www-form-urlencoded' },
body: form.toString(),
});
if (!response.ok) {
throw await this.failed(response, 'The webhook API');
}
const parsed = await readCappedJson<unknown>(response, PROVIDER_JSON_MAX_BYTES);
const id = Number(textOf(asRecord(asRecord(asRecord(parsed)?.ocs)?.data)?.id));
if (!Number.isInteger(id) || id <= 0) {
throw new WebdavError('provider_error', 'The webhook API answered without an id', response.status);
}
return id;
}
async deleteWebhook(creds: WebdavCreds, id: number): Promise<void> {
const response = await this.send(creds, `${OCS_WEBHOOKS}/${id}`, {
method: 'DELETE',
headers: OCS_HEADERS,
});
void response.body?.cancel().catch(() => {});
if (!response.ok && response.status !== 404) {
throw new WebdavError(
classifyStatus(response.status),
`Removing the webhook answered HTTP ${response.status}`,
response.status,
);
}
}
}
@@ -0,0 +1,862 @@
import { Injectable } from '@nestjs/common';
import type {
DocResult,
DocumentConnectionRef,
DocumentProvider,
DocumentProviderCapabilities,
DocumentScopeOption,
DocumentScopeRef,
FetchResult,
PushRequest,
PushResult,
RemoteDocument,
} from '../document-provider';
import { docFail, docFailed, docOk } from '../document-provider';
import {
TREK_PROP_DOC_UID,
TREK_PROP_TRIP_UID,
WebdavClient,
WebdavError,
davPathOf,
encodePath,
normalizeOrigin,
normalizePath,
type WebdavCreds,
type WebdavEntry,
type WebdavFlavor,
} from './webdav.client';
/**
* The document adapter for Nextcloud and OpenCloud.
*
* One implementation serves both, and which one it is talking to is read off
* the connection: Nextcloud brings a login name, an app password and a folder,
* OpenCloud a username, an app token and a space. Nothing below branches on the
* registered `id`; the two subclasses at the end of this file exist only because
* the registry keys providers by `id` and Nest keys them by class, so two
* product ids need two tokens. Everything a product does differently lives in
* `flavorOf()` and the handful of branches it feeds.
*
* The scope is a folder (`fileid:<oc:fileid>`) or a space (`drive:<driveId>`).
* Both ids survive a rename and a move within the same storage, which is what
* lets `stableId` be true and a trip binding survive someone tidying up.
*
* Deliberately NOT here: a change feed. `sync-collection` (RFC 6578) is not
* implemented by either product (Nextcloud answers HTTP 415 with
* `ReportNotSupported`), so the listing is a full enumeration, short-circuited
* by the root ETag, exactly as `DocumentProvider.list` describes.
*/
/** The four Nextcloud events that mean "something in this folder moved". */
const WEBHOOK_EVENTS = [
'OCP\\Files\\Events\\Node\\NodeCreatedEvent',
'OCP\\Files\\Events\\Node\\NodeWrittenEvent',
'OCP\\Files\\Events\\Node\\NodeDeletedEvent',
'OCP\\Files\\Events\\Node\\NodeRenamedEvent',
] as const;
/**
* Header the subscription carries the shared secret in.
*
* It has to be the one the endpoint reads: `doc-sync-webhook.controller.ts`
* looks for `x-trek-docsync-secret`. It used to say `X-TREK-Docsync-Signature`
* here, so every webhook Nextcloud actually sent arrived without a secret the
* controller could find and was silently dropped: the subscription existed, the
* calls arrived, and nothing ever came of them.
*/
const WEBHOOK_SECRET_HEADER = 'x-trek-docsync-secret';
/** Where a Nextcloud connection looks for trip folders when nobody said otherwise. */
const DEFAULT_BASE_PATH = '/TREK';
/** Enough scopes for a busy install, and a ceiling so the cache cannot grow forever. */
const MAX_CACHED_SCOPES = 200;
/**
* A document whose entry carried no `oc:fileid`. Both products always send one,
* but a reverse proxy stripping the `oc:` namespace would otherwise drop the
* file from the trip silently; a path-shaped id keeps it visible at the cost of
* looking renamed if someone renames it.
*/
const PATH_ID_PREFIX = 'path:';
/** Everything the adapter needs to address one bound scope. */
interface ResolvedScope {
creds: WebdavCreds;
/** DAV path of the scope root, percent-encoded, on the connection's own origin. */
rootPath: string;
/** The same path decoded, for deriving a document's path inside the scope. */
rootDecoded: string;
/** The id inside the scope key: an `oc:fileid` or a `driveId`. */
scopeId: string;
}
function textSetting(values: Readonly<Record<string, string>>, key: string): string {
const value = values[key];
return typeof value === 'string' ? value.trim() : '';
}
/**
* Which product this connection describes.
*
* Read off the field set the admin filled in rather than off the provider id,
* so a connection cannot end up being addressed as the product it is not
* because two rows in `document_providers` were swapped.
*/
export function flavorOf(conn: DocumentConnectionRef): WebdavFlavor | null {
if (textSetting(conn.settings, 'login_name') || textSetting(conn.secrets, 'app_password')) {
return 'nextcloud';
}
if (textSetting(conn.settings, 'username') || textSetting(conn.secrets, 'app_token')) {
return 'opencloud';
}
return null;
}
function credsOf(conn: DocumentConnectionRef): WebdavCreds | null {
const flavor = flavorOf(conn);
const origin = normalizeOrigin(conn.baseUrl);
if (!flavor || !/^https?:\/\/[^/]+$/i.test(origin)) return null;
const username =
flavor === 'nextcloud'
? textSetting(conn.settings, 'login_name')
: textSetting(conn.settings, 'username');
const password =
flavor === 'nextcloud'
? textSetting(conn.secrets, 'app_password')
: textSetting(conn.secrets, 'app_token');
if (!username || !password) return null;
return { origin, username, password, allowInsecureTls: conn.allowInsecureTls, flavor };
}
/** `/remote.php/dav/files/<login>`: the per-account root of Nextcloud's file DAV. */
function filesRoot(creds: WebdavCreds): string {
return `/remote.php/dav/files/${encodeURIComponent(creds.username)}`;
}
function basePathOf(conn: DocumentConnectionRef): string {
const configured = textSetting(conn.settings, 'base_path');
return normalizePath(configured || DEFAULT_BASE_PATH);
}
/** `fileid:437` / `drive:storage$space` → the id, for the kind this flavor uses. */
function scopeIdOf(scopeKey: string, flavor: WebdavFlavor): string | null {
const expected = flavor === 'nextcloud' ? 'fileid:' : 'drive:';
if (!scopeKey.startsWith(expected)) return null;
const id = scopeKey.slice(expected.length).trim();
return id.length > 0 ? id : null;
}
/**
* Names both products accept.
*
* Nextcloud refuses `\ / < > : " | ? *` and a trailing dot or space outright, so
* a trip called "Japan: 2026" would fail to get a folder at all. The replacement
* is visible: the label in the returned option is the name that was really
* used, not the one that was asked for.
*/
function sanitizeName(name: string): string {
const cleaned = name
// eslint-disable-next-line no-control-regex
.replace(/[\\/<>:"|?*\u0000-\u001f]/g, '-')
.replace(/\s+/g, ' ')
.trim()
.replace(/[. ]+$/, '');
return cleaned.slice(0, 200);
}
function parentOf(encodedPath: string): string {
const cut = encodedPath.lastIndexOf('/');
return cut > 0 ? encodedPath.slice(0, cut) : '';
}
/** The document's path inside its scope, decoded, with no leading slash. */
function relativeOf(entry: WebdavEntry, rootDecoded: string): string {
return entry.path.startsWith(`${rootDecoded}/`)
? entry.path.slice(rootDecoded.length + 1)
: entry.name;
}
function documentOf(entry: WebdavEntry, rootDecoded: string): RemoteDocument {
const relative = relativeOf(entry, rootDecoded);
return {
remoteId: entry.fileId ?? `${PATH_ID_PREFIX}${relative}`,
name: entry.name,
size: entry.size,
mimeType: entry.mimeType,
// An ETag is what both products change on every write. When one is missing
// the pair below still differs after a write, which is all the core reads
// out of this field.
remoteVersion: entry.etag || `mtime:${entry.lastModifiedIso ?? ''}:${entry.size ?? ''}`,
contentHash: entry.sha256,
remoteModifiedAt: entry.lastModifiedIso,
// WebDAV has no tombstone in a listing: a deleted file is simply absent, and
// the core's own mass-delete guard is what stands between that and a trip
// emptying itself.
isDeleted: false,
};
}
/**
* Anything that goes wrong, as a result rather than an exception.
*
* `probe` and its siblings always resolve because the settings screen renders
* the answer inline. An error that is not a WebdavError is a bug in this file
* rather than a provider problem, and it still has to come back as data: one
* broken adapter must not take down the run for every other link.
*/
function failureOf<T>(err: unknown): DocResult<T> {
if (err instanceof WebdavError) {
return docFail<T>(err.code, err.detail ?? err.message, err.status);
}
return docFail<T>('unknown', err instanceof Error ? err.message : String(err));
}
@Injectable()
export class WebdavDocumentProvider implements DocumentProvider {
readonly id: string = 'nextcloud';
/**
* Scope key → (remote id → still-encoded href), refilled by every listing.
*
* WebDAV addresses resources by path while the core hands back the stable id
* it stored, and neither product can PROPFIND a Nextcloud file id directly.
* Without this, every download, rename and trash would re-walk the folder
* first. It is a cache, not state: a miss costs one PROPFIND, never a wrong
* answer.
*/
private readonly hrefCache = new Map<string, Map<string, string>>();
constructor(private readonly client: WebdavClient) {}
capabilities(conn: DocumentConnectionRef): DocumentProviderCapabilities {
return {
// Conservative on purpose: most Nextcloud connections are an ordinary
// account's app password, and `webhook_listeners` is admin-only. `probe`
// upgrades this to `webhook-self-registered` once it has asked the
// instance. OpenCloud has no HTTP subscription API at all (its change
// events live on an internal NATS bus that no trip admin can point at
// TREK), so it is `none` rather than a `webhook-manual` that would send
// someone looking for a settings page that does not exist.
push: flavorOf(conn) === 'nextcloud' ? 'webhook-manual' : 'none',
stableId: true,
remoteTrash: true,
replaceInPlace: true,
// False for both: Nextcloud stores whatever checksum the uploader claimed
// without verifying it, and OpenCloud computes SHA1/MD5/ADLER32 but never
// SHA256. See the header of webdav.client.ts.
contentHashInListing: false,
maxUploadBytes: null,
acceptedMimeTypes: null,
canCreateScope: true,
};
}
async probe(
conn: DocumentConnectionRef,
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
const creds = credsOf(conn);
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
try {
const capabilities = this.capabilities(conn);
if (creds.flavor === 'opencloud') {
const account = await this.client.whoAmI(creds);
return docOk({ account, capabilities });
}
// The WebDAV root rather than an OCS endpoint: it is the surface every
// other call uses, and a login name that does not match the account
// answers 404 here while the password checks out fine.
try {
await this.client.propfind(creds, filesRoot(creds), 0);
} catch (err: unknown) {
if (err instanceof WebdavError && err.status === 404) {
return docFail(
'not_found',
`No WebDAV home for login name "${creds.username}". Nextcloud shows the exact spelling under Settings → Personal → Security.`,
404,
);
}
throw err;
}
const canRegister = await this.client.canRegisterWebhooks(creds);
return docOk({
account: creds.username,
capabilities: {
...capabilities,
push: canRegister ? 'webhook-self-registered' : 'webhook-manual',
},
});
} catch (err: unknown) {
return failureOf(err);
}
}
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
const creds = credsOf(conn);
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
const needle = query?.trim().toLowerCase() ?? '';
try {
const options =
creds.flavor === 'nextcloud'
? await this.nextcloudScopes(conn, creds)
: await this.opencloudScopes(creds);
return docOk(needle ? options.filter((option) => option.label.toLowerCase().includes(needle)) : options);
} catch (err: unknown) {
return failureOf(err);
}
}
private async nextcloudScopes(
conn: DocumentConnectionRef,
creds: WebdavCreds,
): Promise<DocumentScopeOption[]> {
const basePath = basePathOf(conn);
let listing;
try {
listing = await this.client.propfind(creds, `${filesRoot(creds)}${encodePath(basePath)}`, 1);
} catch (err: unknown) {
// The base folder does not exist yet on a fresh instance. An empty picker
// plus a working "create" is a better answer than an error that reads
// like the credentials are wrong.
if (err instanceof WebdavError && err.status === 404) return [];
throw err;
}
const options: DocumentScopeOption[] = [];
for (const child of listing.children) {
if (!child.isCollection || !child.fileId) continue;
options.push({
scopeKey: `fileid:${child.fileId}`,
label: child.name,
remoteRootId: child.fileId,
remoteRootPath: `${basePath}/${child.name}`,
});
}
return options;
}
private async opencloudScopes(creds: WebdavCreds): Promise<DocumentScopeOption[]> {
const drives = await this.client.listDrives(creds);
return drives
// `virtual` is the synthetic "Shares" drive, which has no storage of its
// own and cannot hold an upload.
.filter((drive) => drive.driveType !== 'virtual')
.map((drive) => ({
scopeKey: `drive:${drive.id}`,
label: drive.name,
remoteRootId: drive.id,
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
}));
}
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
const creds = credsOf(conn);
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
const safeName = sanitizeName(name);
if (!safeName) return docFail('provider_error', 'The name has no characters the provider accepts');
try {
if (creds.flavor === 'opencloud') {
const drive = await this.client.createDrive(creds, safeName);
return docOk({
scopeKey: `drive:${drive.id}`,
label: drive.name,
remoteRootId: drive.id,
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
});
}
const basePath = basePathOf(conn);
const path = `${filesRoot(creds)}${encodePath(`${basePath}/${safeName}`)}`;
let created: 'created' | 'exists';
try {
created = await this.client.mkcol(creds, path);
} catch (err: unknown) {
// 409 is Sabre's "parent node does not exist", i.e. the base folder has
// never been created. Making it is the whole point of this call.
if (!(err instanceof WebdavError) || err.status !== 409) throw err;
await this.client.mkcol(creds, `${filesRoot(creds)}${encodePath(basePath)}`);
created = await this.client.mkcol(creds, path);
}
if (created === 'exists') {
return docFail(
'conflict',
`"${safeName}" already exists under ${basePath}. Pick it in the list instead of creating it again.`,
405,
);
}
// MKCOL answers an OC-FileId header, but only on Nextcloud and only in the
// padded spelling; one PROPFIND gets the id in the form every listing
// will use, which is the form the scope key has to carry.
const folder = await this.client.propfind(creds, path, 0);
if (!folder.self.fileId) {
return docFail('provider_error', 'The new folder came back without an oc:fileid');
}
return docOk({
scopeKey: `fileid:${folder.self.fileId}`,
label: safeName,
remoteRootId: folder.self.fileId,
remoteRootPath: `${basePath}/${safeName}`,
});
} catch (err: unknown) {
return failureOf(err);
}
}
async resolveScope(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<DocumentScopeOption>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
const { creds, rootPath, scopeId } = resolved.data;
try {
if (creds.flavor === 'opencloud') {
const drive = await this.client.getDrive(creds, scopeId);
if (!drive) return docFail('scope_missing', `The space ${scopeId} no longer exists`);
return docOk({
scopeKey: `drive:${drive.id}`,
label: drive.name,
remoteRootId: drive.id,
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
});
}
const folder = await this.client.propfind(creds, rootPath, 0);
// The path still resolves but the id behind it changed: someone deleted
// the folder and made a new one with the same name, and its contents have
// nothing to do with this trip.
if (folder.self.fileId !== scopeId) {
return docFail(
'scope_missing',
`${scope.remoteRootPath ?? rootPath} is a different folder now (oc:fileid ${folder.self.fileId ?? 'missing'} instead of ${scopeId})`,
);
}
return docOk({
scopeKey: `fileid:${scopeId}`,
label: folder.self.name,
remoteRootId: scopeId,
remoteRootPath: scope.remoteRootPath,
});
} catch (err: unknown) {
if (err instanceof WebdavError && err.status === 404) {
return docFail('scope_missing', err.detail ?? err.message, 404);
}
return failureOf(err);
}
}
async list(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): Promise<DocResult<{
documents: RemoteDocument[];
cursor: string | null;
cursorUnchanged: boolean;
truncated: boolean;
}>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
const { creds, rootPath, rootDecoded } = resolved.data;
try {
// A write anywhere below a collection changes that collection's ETag up to
// the root on both products, so one Depth:0 request answers "has anything
// at all happened here" for a folder of any size.
if (scope.cursor) {
const head = await this.client.propfind(creds, rootPath, 0);
if (head.self.etag && head.self.etag === scope.cursor) {
return docOk({ documents: [], cursor: scope.cursor, cursorUnchanged: true, truncated: false });
}
}
const listing = await this.client.propfind(creds, rootPath, 1);
const documents: RemoteDocument[] = [];
const hrefs = new Map<string, string>();
for (const child of listing.children) {
// Sub-folders are not documents and are not descended into: Depth:1 is
// one request per run whatever the folder holds, and a recursive walk
// would trade that for one request per directory.
if (child.isCollection) continue;
const document = documentOf(child, rootDecoded);
documents.push(document);
hrefs.set(document.remoteId, child.href);
}
this.rememberHrefs(conn, scope, hrefs);
return docOk({
documents,
cursor: listing.self.etag || null,
cursorUnchanged: false,
truncated: listing.truncated,
});
} catch (err: unknown) {
if (err instanceof WebdavError && err.status === 404) {
return docFail('scope_missing', err.detail ?? err.message, 404);
}
return failureOf(err);
}
}
async fetch(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<FetchResult>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
try {
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
if (!href) return docFail('not_found', `${remoteId} is no longer in this folder`);
const download = await this.client.get(resolved.data.creds, href);
return docOk({
body: download.body,
size: download.size,
mimeType: download.mimeType,
remoteVersion: download.etag,
});
} catch (err: unknown) {
return failureOf(err);
}
}
async push(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
req: PushRequest,
): Promise<DocResult<PushResult>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
const { creds, rootPath } = resolved.data;
const name = sanitizeName(req.fileName);
if (!name) return docFail('provider_error', 'The file name has no characters the provider accepts');
try {
let target: string;
if (req.remoteId) {
const href = await this.hrefFor(conn, scope, resolved.data, req.remoteId);
// Replacing something that is gone is not the same as creating it: the
// core records the document as missing and asks a human, rather than
// resurrecting a file somebody deleted upstream on purpose.
if (!href) return docFail('not_found', `${req.remoteId} is no longer in this folder`);
target = href;
} else {
target = `${rootPath}/${encodeURIComponent(name)}`;
}
const result = await this.client.put(creds, target, req.body, {
size: req.size,
mimeType: req.mimeType,
// Without this the file TREK just uploaded looks freshly changed
// upstream on the next run, and the two sides push it back and forth.
mtimeSeconds: req.mtimeSeconds,
sha256: req.sha256,
ifMatch: req.expectedRemoteVersion,
// Creating means creating: an unconditional PUT to a name that is
// already taken overwrites somebody else's document without a word.
// Two trips bound to one folder, or a person who put a `receipt.pdf`
// there by hand, are enough for that to happen.
ifNoneMatch: req.remoteId ? undefined : '*',
});
// Best effort, and deliberately not fatal: the bytes are already stored,
// so failing here would make the core upload them again on every run. The
// anchors are a recovery aid, not part of the document's identity.
try {
await this.client.proppatch(creds, target, {
[TREK_PROP_DOC_UID]: req.trekDocUid,
[TREK_PROP_TRIP_UID]: req.trekTripUid,
});
} catch (err: unknown) {
if (!(err instanceof WebdavError)) throw err;
}
let { fileId, etag } = result;
// A reverse proxy that drops the OC-* headers leaves the upload done and
// unidentifiable, so ask for what it swallowed.
if (!fileId || !etag) {
const stored = await this.client.propfind(creds, target, 0);
fileId ??= stored.self.fileId;
etag ||= stored.self.etag;
}
if (!fileId || !etag) {
return docFail('provider_error', 'The upload succeeded but the instance never named the stored file');
}
this.rememberHrefs(conn, scope, new Map([[fileId, target]]));
return docOk({
remoteId: fileId,
remoteVersion: etag,
remoteModifiedAt: new Date(req.mtimeSeconds * 1000).toISOString(),
// Neither product de-duplicates by content; a second upload of the same
// bytes is a second file.
deduplicated: false,
});
} catch (err: unknown) {
return failureOf(err);
}
}
async rename(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
name: string,
): Promise<DocResult<{ remoteVersion: string }>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
const safeName = sanitizeName(name);
if (!safeName) return docFail('provider_error', 'The new name has no characters the provider accepts');
try {
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
if (!href) return docFail('not_found', `${remoteId} is no longer in this folder`);
// Within the same parent: a rename moves a document inside the trip's
// folder, it never moves it out of the scope the trip is bound to.
const destination = `${parentOf(href)}/${encodeURIComponent(safeName)}`;
const moved = await this.client.move(resolved.data.creds, href, destination);
this.rememberHrefs(conn, scope, new Map([[remoteId, destination]]));
if (moved.etag) return docOk({ remoteVersion: moved.etag });
const renamed = await this.client.propfind(resolved.data.creds, destination, 0);
return docOk({ remoteVersion: renamed.self.etag });
} catch (err: unknown) {
return failureOf(err);
}
}
async trash(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
remoteId: string,
): Promise<DocResult<void>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
try {
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
// Already gone is the state the caller asked for.
if (!href) return docOk(undefined);
await this.client.delete(resolved.data.creds, href);
this.forgetHref(conn, scope, remoteId);
return docOk(undefined);
} catch (err: unknown) {
return failureOf(err);
}
}
/**
* Subscribe to the four file events, one subscription each.
*
* A failure here is never a broken connection: the endpoint is admin-only, so
* the ordinary case of a non-admin app password is a plain "no", the link
* stays on polling, and `probe` already reported `webhook-manual` for exactly
* this account. Partial registrations are rolled back so a retry does not
* leave a pile of orphaned subscriptions behind.
*
* Two things a self-hoster has to know, neither of which this call can fix:
* Nextcloud delivers webhooks through its background-job queue, so the
* notification is as prompt as that instance's cron and no prompter, and
* `allow_local_remote_servers` must be on or a callback to a LAN address is
* refused at delivery time while the subscription still looks healthy.
*/
async registerWebhook(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
callbackUrl: string,
secret: string,
): Promise<DocResult<{ subscriptionId: string }>> {
const resolved = this.resolve(conn, scope);
if (docFailed(resolved)) return resolved;
const { creds } = resolved.data;
if (creds.flavor === 'opencloud') {
return docFail('forbidden', 'OpenCloud has no webhook API a trip admin can subscribe to');
}
const created: number[] = [];
try {
if (!(await this.client.canRegisterWebhooks(creds))) {
return docFail(
'forbidden',
'This Nextcloud account may not manage webhooks, so the link keeps polling instead',
403,
);
}
// Deliberately unscoped: the obvious `eventFilter` on the changed path
// needs `$regex`, which the app's query engine does not implement, and the
// exception it throws stops delivery for every other webhook on the same
// event. A subscription that says "look now" about a folder this trip does
// not care about costs one listing; the filter costs the whole channel.
for (const event of WEBHOOK_EVENTS) {
created.push(
await this.client.createWebhook(creds, {
uri: callbackUrl,
event,
secretHeader: WEBHOOK_SECRET_HEADER,
secret,
}),
);
}
return docOk({ subscriptionId: created.join(',') });
} catch (err: unknown) {
for (const id of created) {
await this.client.deleteWebhook(creds, id).catch(() => undefined);
}
return failureOf(err);
}
}
async unregisterWebhook(conn: DocumentConnectionRef, subscriptionId: string): Promise<DocResult<void>> {
const creds = credsOf(conn);
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
try {
for (const part of subscriptionId.split(',')) {
const id = Number(part.trim());
if (Number.isInteger(id) && id > 0) {
await this.client.deleteWebhook(creds, id);
}
}
return docOk(undefined);
} catch (err: unknown) {
return failureOf(err);
}
}
/** Connection plus scope key turned into an addressable root, or the reason it is not one. */
private resolve(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
): DocResult<ResolvedScope> {
const creds = credsOf(conn);
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
const scopeId = scopeIdOf(scope.scopeKey, creds.flavor);
if (!scopeId) {
return docFail('scope_missing', `"${scope.scopeKey}" is not a scope key this provider issued`);
}
if (creds.flavor === 'nextcloud') {
const userPath = scope.remoteRootPath ? normalizePath(scope.remoteRootPath) : null;
if (!userPath || userPath === '/') {
return docFail('scope_missing', 'The binding has no folder path, and a file id cannot be addressed over WebDAV');
}
const rootPath = `${filesRoot(creds)}${encodePath(userPath)}`;
return docOk({ creds, rootPath, rootDecoded: `${filesRoot(creds)}${userPath}`, scopeId });
}
// The stored root is the space's own webDavUrl. Only its path is used, and
// it is re-based on the connection's origin: the URL came out of a provider
// response, and following the host in it would hand an attacker who can
// answer for that instance a way past the SSRF guard.
const path = scope.remoteRootPath ? davPathOf(scope.remoteRootPath) : null;
const rootPath = path ?? `/dav/spaces/${encodeURIComponent(scopeId)}`;
let decoded: string;
try {
decoded = normalizePath(decodeURIComponent(rootPath));
} catch {
// A stored path with a stray percent sign is still usable as-is; failing
// the whole binding over a decoding quirk would be worse than the quirk.
decoded = rootPath;
}
return docOk({ creds, rootPath, rootDecoded: decoded, scopeId });
}
private cacheKey(conn: DocumentConnectionRef, scope: DocumentScopeRef): string {
return `${conn.connectionId}:${scope.scopeKey}`;
}
private rememberHrefs(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
hrefs: Map<string, string>,
): void {
const key = this.cacheKey(conn, scope);
const existing = this.hrefCache.get(key);
if (existing) {
for (const [id, href] of hrefs) existing.set(id, href);
return;
}
if (this.hrefCache.size >= MAX_CACHED_SCOPES) {
// Map iterates in insertion order, so the first key is the oldest scope.
const oldest = this.hrefCache.keys().next();
if (!oldest.done) this.hrefCache.delete(oldest.value);
}
this.hrefCache.set(key, new Map(hrefs));
}
private forgetHref(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): void {
this.hrefCache.get(this.cacheKey(conn, scope))?.delete(remoteId);
}
/** The href for a remote id, re-walking the folder once when the cache cannot say. */
private async hrefFor(
conn: DocumentConnectionRef,
scope: DocumentScopeRef,
resolved: ResolvedScope,
remoteId: string,
): Promise<string | null> {
if (remoteId.startsWith(PATH_ID_PREFIX)) {
const relative = remoteId.slice(PATH_ID_PREFIX.length);
// The path comes out of a listing, which is the provider's word and not
// TREK's. A `..` in it would address a file outside the folder the trip is
// bound to. Every other id here is opaque, this one is a path and has to
// be treated like one.
if (!relative || relative.split('/').some(seg => seg === '..' || seg === '.')) return null;
return `${resolved.rootPath}/${encodePath(relative)}`;
}
const cached = this.hrefCache.get(this.cacheKey(conn, scope))?.get(remoteId);
if (cached) return cached;
const listing = await this.client.propfind(resolved.creds, resolved.rootPath, 1);
const hrefs = new Map<string, string>();
for (const child of listing.children) {
if (child.isCollection) continue;
hrefs.set(child.fileId ?? `${PATH_ID_PREFIX}${relativeOf(child, resolved.rootDecoded)}`, child.href);
}
this.rememberHrefs(conn, scope, hrefs);
return hrefs.get(remoteId) ?? null;
}
}
/**
* The two registrations.
*
* They carry an id and nothing else. The registry maps a connection's
* `provider_id` onto a provider instance and Nest maps a class onto a token, so
* two product ids need two classes, but both run the implementation above, and
* that implementation never asks which one it is.
*
* The constructors are spelled out rather than inherited: Nest reads
* `design:paramtypes` off the concrete class, and a subclass without its own
* constructor is emitted without that metadata and resolved with no arguments.
*/
@Injectable()
export class NextcloudDocumentProvider extends WebdavDocumentProvider {
override readonly id = 'nextcloud';
constructor(client: WebdavClient) {
super(client);
}
}
@Injectable()
export class OpencloudDocumentProvider extends WebdavDocumentProvider {
override readonly id = 'opencloud';
constructor(client: WebdavClient) {
super(client);
}
}
@@ -12,7 +12,6 @@ import {
splitIntoRuns,
spillChains,
dayWindow,
parseClock,
effectiveRangeKm,
parseAvoid,
formatDistance,
@@ -34,6 +33,7 @@ interface StoredDay {
}
interface VisitRow {
check_in: string | null;
/** The drive does not read these two (#2357): they are the booking as get_roadtrip_context reports it. */
check_out: string | null;
checkout_day: number | null;
id: number;
@@ -43,6 +43,7 @@ interface VisitRow {
lat: number | null;
lng: number | null;
time: string | null;
end_time: string | null;
duration_minutes: number | null;
end_day: number;
leg_transport_mode: string | null;
@@ -72,7 +73,8 @@ export class RoadtripPlanService {
);
const visits = this.db.all<VisitRow>(
`SELECT a.id, a.day_id, a.place_id, p.name, p.lat, p.lng,
COALESCE(a.assignment_time, p.place_time) AS time, p.duration_minutes, a.end_day,
COALESCE(a.assignment_time, p.place_time) AS time, COALESCE(a.assignment_end_time, p.end_time) AS end_time,
p.duration_minutes, a.end_day,
a.leg_transport_mode, a.incoming_leg_transport_mode, p.stop_type, p.fill_percent, stay.check_in, stay.check_out, checkout.day_number AS checkout_day
FROM day_assignments a JOIN days d ON d.id = a.day_id JOIN places p ON p.id = a.place_id
LEFT JOIN day_accommodations stay ON stay.id = (SELECT id FROM day_accommodations WHERE place_id = p.id AND start_day_id = d.id ORDER BY id LIMIT 1)
@@ -117,8 +119,10 @@ export class RoadtripPlanService {
lat: v.lat!,
lng: v.lng!,
time: v.time,
// The visit's end time is when the drive leaves it, the same reading the
// planner makes in the browser (useRoadtripRoutes).
leaveAt: v.end_time,
checkInTime: v.check_in,
...(v.checkout_day != null && parseClock(v.check_out) !== null ? { checkoutAt: v.checkout_day * 1440 + parseClock(v.check_out)! } : {}),
dwellMinutes: v.duration_minutes,
endDay: v.end_day === 1,
legMode: v.leg_transport_mode,
@@ -27,7 +27,7 @@ export class RoadtripPlanningMcp {
@Tool({
name: 'get_roadtrip_context',
description:
'Read the saved roadtrip days, visits, pinned times, stays, stop types, fill levels, travel modes, via points, followed tracks and manual day endings. Includes the shared driving preferences for this trip. No routing request and no browser needed. Coordinates missing from a visit prevent it from being routed. Use calculate_roadtrip to get the derived day layout. Edit visits with the existing place and assignment tools; change their saved order with reorder_day_assignments. Settings, visits and manual boundaries all belong to the shared trip.',
'Read the saved roadtrip days, visits, pinned times, end times (when the drive leaves a visit), stays, stop types, fill levels, travel modes, via points, followed tracks and manual day endings. Includes the shared driving preferences for this trip. No routing request and no browser needed. Coordinates missing from a visit prevent it from being routed. Use calculate_roadtrip to get the derived day layout. Edit visits with the existing place and assignment tools; change their saved order with reorder_day_assignments. Settings, visits and manual boundaries all belong to the shared trip.',
inputSchema: { tripId: z.number().int().positive() },
annotations: TOOL_ANNOTATIONS_READONLY,
access: { group: 'trips', mode: 'read' },
@@ -40,7 +40,7 @@ export class RoadtripPlanningMcp {
@Tool({
name: 'calculate_roadtrip',
description:
'Calculate a saved trip without an open browser, using the same daily scheduling and vehicle-range rules as the planner. Returns derived days, arrivals, departures, driving limits, fuel warnings, automatic pauses and routing failures. Fixed visit times retain priority; no accommodation or place is created. Optional settings are a read-only preview, not saved. Distances are metres, durations seconds, dwell times minutes. A failed leg or conflicting window means the output is incomplete: never present its totals as a complete route. Up to 150 legs; requests are paced for public routing servers and can take time. Existing place/assignment tools edit stops, stays and ordering, update_roadtrip_settings saves preferences, set_assignment_end_day and set_day_boundary set exceptions. Recalculate after changes. Plugin profiles follow installed route-provider permissions.',
'Calculate a saved trip without an open browser, using the same daily scheduling and vehicle-range rules as the planner. Returns derived days, arrivals, departures, driving limits, fuel warnings, automatic pauses and routing failures. Fixed visit times retain priority. The end time of a visit is when the drive leaves it, in place of its stay; reaching the visit after it is reported as a missedLeave warning. No accommodation or place is created. Optional settings are a read-only preview, not saved. Distances are metres, durations seconds, dwell times minutes. A failed leg or conflicting window means the output is incomplete: never present its totals as a complete route. Up to 150 legs; requests are paced for public routing servers and can take time. Existing place/assignment tools edit stops, stays and ordering, update_roadtrip_settings saves preferences, set_assignment_end_day and set_day_boundary set exceptions. Recalculate after changes. Plugin profiles follow installed route-provider permissions.',
inputSchema: roadtripPlanRequestSchema.shape,
annotations: TOOL_ANNOTATIONS_READONLY,
access: { group: 'trips', mode: 'read' },
+78 -1
View File
@@ -43,6 +43,11 @@ const { db } = vi.hoisted(() => {
check_out TEXT, confirmation TEXT, notes TEXT, created_at TEXT DEFAULT (datetime('now')));`);
tmp.exec(`CREATE TABLE assignment_participants (assignment_id INTEGER NOT NULL, user_id INTEGER NOT NULL,
UNIQUE(assignment_id, user_id));`);
// A start that reorders a day re-pins that day's vias in the same transaction, so
// the sort reads this table whenever it moves a stop.
tmp.exec(`CREATE TABLE roadtrip_vias (id INTEGER PRIMARY KEY AUTOINCREMENT, day_id INTEGER NOT NULL,
after_order_index INTEGER NOT NULL, sequence INTEGER NOT NULL DEFAULT 0, lat REAL NOT NULL, lng REAL NOT NULL,
created_at TEXT DEFAULT CURRENT_TIMESTAMP);`);
tmp.exec(`CREATE TABLE tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, color TEXT, created_at TEXT);`);
tmp.exec(`CREATE TABLE place_tags (place_id INTEGER NOT NULL, tag_id INTEGER NOT NULL);`);
// StorageRegistryService (behind StorageModule, now in this module chain) reads
@@ -55,7 +60,8 @@ const { canAccessTrip } = vi.hoisted(() => ({ canAccessTrip: vi.fn() }));
vi.mock('../../src/db/database', () => ({
db, canAccessTrip, isOwner: vi.fn(() => true), getPlaceWithTags: vi.fn(), closeDb: () => {}, reinitialize: () => {},
}));
vi.mock('../../src/websocket', () => ({ broadcast: vi.fn() }));
const { broadcast } = vi.hoisted(() => ({ broadcast: vi.fn() }));
vi.mock('../../src/websocket', () => ({ broadcast }));
const { reconcileTripSkeletons } = vi.hoisted(() => ({ reconcileTripSkeletons: vi.fn() }));
import { JourneyDomainService } from '../../src/nest/journey/journey-domain.service';
@@ -213,6 +219,77 @@ describe('Assignments e2e (real auth guard + temp SQLite)', () => {
expect(reconcileTripSkeletons).toHaveBeenCalledWith(5, undefined);
});
describe('PUT /:id/time and the order of the day', () => {
const seedDay = (times: (string | null)[]) => times.map((time, i) => {
const id = seedAssignment(3, 2, i);
if (time) db.prepare('UPDATE day_assignments SET assignment_time = ? WHERE id = ?').run(time, id);
return id;
});
const dayOrder = () =>
(db.prepare('SELECT id FROM day_assignments WHERE day_id = 3 ORDER BY order_index, id').all() as { id: number }[]).map(r => r.id);
const eventsSent = () => broadcast.mock.calls.map(call => call[1]);
beforeEach(() => broadcast.mockClear());
it('keeps the untimed stops in front of the stop that gets a start', async () => {
const [a, b, c] = seedDay([null, null, null]);
const res = await request(server)
.put(`/api/trips/5/assignments/${c}/time`)
.set('Cookie', sessionCookie(1))
.set('X-Socket-Id', 'sock-1')
.send({ place_time: '14:00', end_time: null });
expect(res.status).toBe(200);
expect(res.body.assignment).toMatchObject({ id: c, assignment_time: '14:00', order_index: 2 });
expect(dayOrder()).toEqual([a, b, c]);
expect(eventsSent()).toEqual(['assignment:updated']);
});
it('sorts the timed stops, keeps the untimed head first and sends the whole day', async () => {
const [a, b, c] = seedDay([null, '15:00', null]);
const res = await request(server)
.put(`/api/trips/5/assignments/${c}/time`)
.set('Cookie', sessionCookie(1))
.set('X-Socket-Id', 'sock-1')
.send({ place_time: '10:00', end_time: null });
expect(res.status).toBe(200);
expect(dayOrder()).toEqual([a, c, b]);
// No socket left out, so the writer gets the order too.
expect(broadcast).toHaveBeenCalledWith('5', 'assignment:reordered', { dayId: 3, orderedIds: [a, c, b] }, undefined);
// No located stops and no vias on this day, so there is nothing to re-pin.
expect(eventsSent()).not.toContain('roadtripVia:changed');
});
it('stores the order it sends: a day with a gap in its keys is numbered from 0', async () => {
// The gap a deleted stop leaves. Clients number the ids they are sent by position.
const [a, b, c] = [0, 4, 7].map(key => seedAssignment(3, 2, key));
db.prepare('UPDATE day_assignments SET assignment_time = ? WHERE id = ?').run('15:00', b);
const res = await request(server)
.put(`/api/trips/5/assignments/${c}/time`)
.set('Cookie', sessionCookie(1))
.send({ place_time: '10:00', end_time: null });
expect(res.status).toBe(200);
const sent = broadcast.mock.calls.find(call => call[1] === 'assignment:reordered')?.[2] as { orderedIds: number[] };
expect(sent.orderedIds).toEqual([a, c, b]);
const keyOf = (id: number) => (db.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect(sent.orderedIds.map(keyOf)).toEqual([0, 1, 2]);
expect(res.body.assignment).toMatchObject({ id: c, order_index: 1 });
});
it('leaves a day dragged out of time order alone when only the End changes', async () => {
const [a, b] = seedDay(['14:00', '10:00']);
db.prepare("UPDATE day_assignments SET assignment_end_time = '11:00' WHERE id = ?").run(b);
const res = await request(server)
.put(`/api/trips/5/assignments/${b}/time`)
.set('Cookie', sessionCookie(1))
.set('X-Socket-Id', 'sock-1')
.send({ place_time: '10:00', end_time: null });
expect(res.status).toBe(200);
expect(res.body.assignment).toMatchObject({ id: b, assignment_time: '10:00', assignment_end_time: null });
expect(dayOrder()).toEqual([a, b]);
expect(eventsSent()).toEqual(['assignment:updated']);
});
});
it('200 participants (access-only)', async () => {
const id = seedAssignment();
db.prepare('INSERT INTO assignment_participants (assignment_id, user_id) VALUES (?, 2)').run(id);
+441
View File
@@ -0,0 +1,441 @@
/**
* Document sync e2e: drives /api/trips/:tripId/docsync through the REAL
* AddonGuard, JwtAuthGuard and TripAccessGuard, over the real DI services
* against a temp SQLite carrying the full schema.
*
* Two things are replaced and nothing else:
*
* - `AddonsService.isAddonEnabled`, because the gate is what is under test;
* - every provider adapter, because a test must never open a socket to
* somebody's document archive. Each one is a spy that fails loudly if a
* route this file exercises turns out to reach a provider after all.
*
* What is pinned here is the access model, which is the part of this feature
* that would hurt most if it drifted: the addon gate answering 404 before 401,
* a stranger being unable to see that a trip exists, and a plain member being
* able to READ where their documents go while only the owner may change it.
* That last split is unusual for TREK (most trip routes treat members alike),
* so it is worth a test that fails if someone "tidies" it away.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import cookieParser from 'cookie-parser';
import type { Server } from 'http';
import { Test } from '@nestjs/testing';
import { sessionCookie } from './harness';
const { db } = vi.hoisted(() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const Database = require('better-sqlite3');
const tmp = new Database(':memory:');
tmp.exec('PRAGMA journal_mode = WAL');
tmp.exec('PRAGMA foreign_keys = ON');
return { db: tmp };
});
vi.mock('../../src/db/database', () => ({
db,
closeDb: () => {},
reinitialize: () => {},
getPlaceWithTags: () => null,
canAccessTrip: (tripId: number | string, userId: number) =>
db
.prepare(
'SELECT t.id, t.user_id FROM trips t LEFT JOIN trip_members m ON m.trip_id = t.id AND m.user_id = ? WHERE t.id = ? AND (t.user_id = ? OR m.user_id IS NOT NULL)',
)
.get(userId, tripId, userId),
isOwner: () => false,
}));
const { isAddonEnabled } = vi.hoisted(() => ({ isAddonEnabled: vi.fn(() => true) }));
vi.mock('../../src/websocket', () => ({ broadcastToUser: vi.fn(), broadcast: vi.fn() }));
/**
* The SSRF guard resolves DNS, and `paperless.example.com` does not exist. In
* CI that is a lookup failure, which the guard correctly reports as "not
* allowed". The guard is not what this file is testing, and a suite that needs
* a working resolver is a suite that fails on someone's train.
*/
vi.mock('../../src/utils/ssrfGuard', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../src/utils/ssrfGuard')>();
return {
...actual,
checkSsrf: vi.fn(async (url: string) => ({
allowed: !url.includes('blocked.invalid'),
isPrivate: false,
error: url.includes('blocked.invalid') ? 'blocked in test' : undefined,
})),
};
});
import { createTables } from '../../src/db/schema';
import { runMigrations } from '../../src/db/migrations';
import { createTrip, createUser } from '../helpers/factories';
import { DocSyncModule } from '../../src/nest/doc-sync/doc-sync.module';
import { DocSyncMcp } from '../../src/nest/doc-sync/doc-sync.mcp';
import type { McpContext } from '../../src/nest-mcp';
import { DatabaseModule } from '../../src/nest/database/database.module';
import { AddonsService } from '../../src/nest/addons/addons.service';
import { DOCUMENT_PROVIDERS } from '../../src/nest/doc-sync/document-provider';
import { PaperlessDocumentProvider } from '../../src/nest/doc-sync/providers/paperless.provider';
import { PapraDocumentProvider } from '../../src/nest/doc-sync/providers/papra.provider';
import { SynologyDriveDocumentProvider } from '../../src/nest/doc-sync/providers/synology-drive.provider';
import { NextcloudDocumentProvider, OpencloudDocumentProvider } from '../../src/nest/doc-sync/providers/webdav.provider';
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
/** A provider that answers plausibly but never opens a socket. */
function fakeProvider(id: string) {
const caps = {
push: 'none' as const,
stableId: true,
remoteTrash: true,
replaceInPlace: true,
contentHashInListing: true,
maxUploadBytes: null,
acceptedMimeTypes: null,
canCreateScope: true,
};
return {
id,
capabilities: () => caps,
probe: vi.fn(async () => ({ success: true as const, data: { account: `${id}-account`, capabilities: caps } })),
listScopes: vi.fn(async () => ({
success: true as const,
data: [{ scopeKey: `tag:1`, label: 'Japan 2026', remoteRootId: '1', remoteRootPath: '/TREK/japan' }],
})),
createScope: vi.fn(async (_c: unknown, name: string) => ({
success: true as const,
data: { scopeKey: `tag:2`, label: name, remoteRootId: '2', remoteRootPath: `/TREK/${name}` },
})),
resolveScope: vi.fn(async () => ({
success: true as const,
data: { scopeKey: 'tag:1', label: 'Japan 2026', remoteRootId: '1', remoteRootPath: '/TREK/japan' },
})),
list: vi.fn(async () => ({ success: true as const, data: { documents: [], cursor: 'c1', cursorUnchanged: false, truncated: false } })),
fetch: vi.fn(async () => ({ success: false as const, error: { code: 'not_found' as const } })),
push: vi.fn(async () => ({ success: false as const, error: { code: 'not_found' as const } })),
rename: vi.fn(async () => ({ success: true as const, data: { remoteVersion: 'v2' } })),
trash: vi.fn(async () => ({ success: true as const, data: undefined })),
};
}
describe('Document sync e2e (real guards + real services + temp SQLite)', () => {
let server: Server;
let app: Awaited<ReturnType<typeof build>>;
let ownerId: number;
let memberId: number;
let strangerId: number;
let tripId: number;
async function build() {
const providers = [
[PaperlessDocumentProvider, 'paperless'],
[PapraDocumentProvider, 'papra'],
[NextcloudDocumentProvider, 'nextcloud'],
[OpencloudDocumentProvider, 'opencloud'],
[SynologyDriveDocumentProvider, 'synologydrive'],
] as const;
let builder = Test.createTestingModule({ imports: [DatabaseModule, DocSyncModule] })
.overrideProvider(AddonsService)
.useValue({ isAddonEnabled });
const fakes = providers.map(([, id]) => fakeProvider(id));
providers.forEach(([token], i) => {
builder = builder.overrideProvider(token).useValue(fakes[i]);
});
builder = builder.overrideProvider(DOCUMENT_PROVIDERS).useValue(fakes);
const moduleRef = await builder.compile();
const nest = moduleRef.createNestApplication();
nest.use(cookieParser());
nest.useGlobalFilters(new TrekExceptionFilter());
nest.useGlobalPipes(new ZodValidationPipe());
await nest.init();
return nest;
}
beforeAll(async () => {
createTables(db as never);
runMigrations(db as never);
ownerId = createUser(db as never, { username: 'owner', email: 'owner@test.local' }).user.id;
memberId = createUser(db as never, { username: 'member', email: 'member@test.local' }).user.id;
strangerId = createUser(db as never, { username: 'stranger', email: 'stranger@test.local' }).user.id;
tripId = createTrip(db as never, ownerId, { title: 'Japan', start_date: '2026-10-01', end_date: '2026-10-07' }).id;
db.prepare('INSERT INTO trip_members (trip_id, user_id) VALUES (?, ?)').run(tripId, memberId);
// Providers are off by default, exactly as a fresh install ships them.
db.prepare("UPDATE document_providers SET enabled = 1 WHERE id IN ('paperless', 'nextcloud')").run();
app = await build();
server = app.getHttpServer();
});
afterAll(async () => {
await app?.close();
db.close();
});
beforeEach(() => {
isAddonEnabled.mockReturnValue(true);
});
it('answers 404 rather than 401 when the addon is off, to everyone alike', async () => {
isAddonEnabled.mockReturnValue(false);
await request(server).get(`/api/trips/${tripId}/docsync/status`).expect(404);
await request(server)
.get(`/api/trips/${tripId}/docsync/status`)
.set('Cookie', sessionCookie(ownerId))
.expect(404);
});
it('refuses an unauthenticated caller', async () => {
await request(server).get(`/api/trips/${tripId}/docsync/status`).expect(401);
});
it('hides the trip from a stranger behind a 404', async () => {
await request(server)
.get(`/api/trips/${tripId}/docsync/status`)
.set('Cookie', sessionCookie(strangerId))
.expect(404);
});
it('lists only the providers an admin switched on', async () => {
const res = await request(server)
.get(`/api/trips/${tripId}/docsync/providers`)
.set('Cookie', sessionCookie(ownerId))
.expect(200);
expect(res.body.map((p: { id: string }) => p.id).sort()).toEqual(['nextcloud', 'paperless']);
// Every listed provider carries its form fields, so the client renders the
// connection form from data rather than from a hard-coded list.
expect(res.body[0].fields.length).toBeGreaterThan(0);
});
it('lets a plain member read the status but not change the binding', async () => {
await request(server)
.get(`/api/trips/${tripId}/docsync/status`)
.set('Cookie', sessionCookie(memberId))
.expect(200);
await request(server)
.put(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(memberId))
.send({ providerId: 'paperless', baseUrl: 'https://paperless.example.com', credentials: { api_token: 'x' } })
.expect(403);
});
it('refuses a provider the admin has not switched on', async () => {
await request(server)
.put(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.send({ providerId: 'papra', baseUrl: 'https://papra.example.com', credentials: { api_key: 'x', organization_id: 'org_1' } })
.expect(400);
});
it('rejects an unknown provider id at the contract, not in the handler', async () => {
await request(server)
.put(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.send({ providerId: 'dropbox', baseUrl: 'https://example.com', credentials: {} })
.expect(400);
});
it('stores a connection and never gives the secret back', async () => {
const res = await request(server)
.put(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.send({
providerId: 'paperless',
baseUrl: 'https://paperless.example.com/',
credentials: { api_token: 'super-secret-token' },
})
.expect(200);
expect(res.body.baseUrl).toBe('https://paperless.example.com');
expect(res.body.secrets.api_token).toBe('••••••••');
expect(JSON.stringify(res.body)).not.toContain('super-secret-token');
// And it is encrypted at rest, not merely hidden on the way out.
const row = db.prepare('SELECT secrets FROM document_connections WHERE trip_id = ?').get(tripId) as { secrets: string };
expect(row.secrets).toMatch(/^enc:v1:/);
expect(row.secrets).not.toContain('super-secret-token');
});
it('keeps the stored secret when the form comes back with a blank field', async () => {
await request(server)
.put(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.send({ providerId: 'paperless', baseUrl: 'https://paperless.example.com', credentials: { api_token: '' } })
.expect(200);
const row = db.prepare('SELECT secrets FROM document_connections WHERE trip_id = ?').get(tripId) as { secrets: string };
expect(row.secrets).toMatch(/^enc:v1:/);
});
it('creates a folder or tag through the connection the path names', async () => {
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.expect(200);
const res = await request(server)
.post(`/api/trips/${tripId}/docsync/connections/${conns.body[0].id}/scopes`)
.set('Cookie', sessionCookie(ownerId))
.send({ name: 'Norway' })
.expect(201);
expect(res.body).toMatchObject({ scopeKey: 'tag:2', label: 'Norway' });
});
it('still takes a scope request from a client that sends the connection in the body too', async () => {
// The body field was dropped in favour of the path. A client from before
// that change has it stripped by the contract instead of being refused.
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId));
await request(server)
.post(`/api/trips/${tripId}/docsync/connections/${conns.body[0].id}/scopes`)
.set('Cookie', sessionCookie(ownerId))
.send({ connectionId: conns.body[0].id, name: 'Norway' })
.expect(201);
});
it('creates no scope through a connection that belongs to another trip', async () => {
const otherTripId = createTrip(db as never, ownerId, { title: 'Elsewhere' }).id;
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId));
await request(server)
.post(`/api/trips/${otherTripId}/docsync/connections/${conns.body[0].id}/scopes`)
.set('Cookie', sessionCookie(ownerId))
.send({ name: 'Norway' })
.expect(404);
});
it('does not let a plain member create a folder or tag', async () => {
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId));
await request(server)
.post(`/api/trips/${tripId}/docsync/connections/${conns.body[0].id}/scopes`)
.set('Cookie', sessionCookie(memberId))
.send({ name: 'Norway' })
.expect(403);
});
it('binds the trip to a scope and reports it to every member', async () => {
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId))
.expect(200);
const connectionId = conns.body[0].id;
await request(server)
.post(`/api/trips/${tripId}/docsync/links`)
.set('Cookie', sessionCookie(ownerId))
.send({ connectionId, scopeKey: 'tag:1', remoteLabel: 'Japan 2026', direction: 'both', deletePolicy: 'unlink', conflictPolicy: 'manual', syncEnabled: true })
.expect(200);
const asMember = await request(server)
.get(`/api/trips/${tripId}/docsync/links`)
.set('Cookie', sessionCookie(memberId))
.expect(200);
expect(asMember.body).toHaveLength(1);
expect(asMember.body[0].remoteLabel).toBe('Japan 2026');
// The shared secret behind the webhook URL is never handed out, not even to
// the owner: it exists so a provider can prove itself, not to be read.
expect(asMember.body[0].webhookSecret).toBe('••••••••');
});
it('changes only the setting a patch names, through the real contract', async () => {
// The unit tests call the handler directly and never see the pipe. Through
// it, a schema that fills in defaults turns one switch into a full reset.
const links = await request(server)
.get(`/api/trips/${tripId}/docsync/links`)
.set('Cookie', sessionCookie(ownerId));
const linkId = links.body[0].id;
await request(server)
.patch(`/api/trips/${tripId}/docsync/links/${linkId}`)
.set('Cookie', sessionCookie(ownerId))
.send({ direction: 'pull', deletePolicy: 'trash' })
.expect(200);
const paused = await request(server)
.patch(`/api/trips/${tripId}/docsync/links/${linkId}`)
.set('Cookie', sessionCookie(ownerId))
.send({ syncEnabled: false })
.expect(200);
expect(paused.body).toMatchObject({
syncEnabled: false,
direction: 'pull',
deletePolicy: 'trash',
conflictPolicy: 'manual',
remoteLabel: 'Japan 2026',
});
});
it('still names the store of a binding after the admin switches its provider off', async () => {
// The providers list drops a provider that is off, so the binding itself is
// the only thing left that can tell a member where the documents went.
db.prepare("UPDATE document_providers SET enabled = 0 WHERE id = 'paperless'").run();
try {
const providers = await request(server)
.get(`/api/trips/${tripId}/docsync/providers`)
.set('Cookie', sessionCookie(memberId))
.expect(200);
expect(providers.body.map((p: { id: string }) => p.id)).not.toContain('paperless');
const links = await request(server)
.get(`/api/trips/${tripId}/docsync/links`)
.set('Cookie', sessionCookie(memberId))
.expect(200);
expect(links.body[0]).toMatchObject({ providerId: 'paperless', providerName: 'Paperless-ngx' });
// The assistant reads the same bindings through its own tool.
const ctx = { userId: memberId, scopes: null, isStaticToken: false } as McpContext;
const result = app.get(DocSyncMcp).getTripDocumentSync({ tripId }, ctx);
const status = JSON.parse(result.content[0].text) as { links: Array<Record<string, unknown>> };
expect(status.links[0]).toMatchObject({ providerId: 'paperless', providerName: 'Paperless-ngx' });
} finally {
db.prepare("UPDATE document_providers SET enabled = 1 WHERE id = 'paperless'").run();
}
});
it('refuses a binding whose connection belongs to another trip', async () => {
const otherTripId = createTrip(db as never, ownerId, { title: 'Other' }).id;
const conns = await request(server)
.get(`/api/trips/${tripId}/docsync/connections`)
.set('Cookie', sessionCookie(ownerId));
await request(server)
.post(`/api/trips/${otherTripId}/docsync/links`)
.set('Cookie', sessionCookie(ownerId))
.send({ connectionId: conns.body[0].id, scopeKey: 'tag:1' })
.expect(400);
});
it('answers 200 with a verdict in the body when a probe fails, not an error status', async () => {
// The settings form needs a field to render, not an exception: the same
// contract the photo providers pin as CRITICAL in their own e2e suite.
const res = await request(server)
.post(`/api/trips/${tripId}/docsync/connections/test`)
.set('Cookie', sessionCookie(ownerId))
.send({ providerId: 'paperless', baseUrl: 'https://blocked.invalid', credentials: {} })
.expect(200);
expect(res.body.connected).toBe(false);
expect(typeof res.body.error).toBe('string');
});
it('unbinds without deleting anything', async () => {
const links = await request(server)
.get(`/api/trips/${tripId}/docsync/links`)
.set('Cookie', sessionCookie(ownerId));
const linkId = links.body[0].id;
const res = await request(server)
.delete(`/api/trips/${tripId}/docsync/links/${linkId}`)
.set('Cookie', sessionCookie(ownerId))
.expect(200);
expect(res.body.documentsKept).toBe(true);
expect(db.prepare('SELECT COUNT(*) AS n FROM trip_document_links WHERE id = ?').get(linkId)).toEqual({ n: 0 });
});
it('accepts an unknown webhook token without revealing that it is unknown', async () => {
// A 404 here would let anyone enumerate which bindings exist.
await request(server).post('/api/docsync/webhook/definitely-not-a-token').expect(200);
});
});
@@ -403,6 +403,75 @@ describe('Tool: update_assignment_time', () => {
});
});
// Same service method as PUT /assignments/:id/time, so the same order and the same
// three events. A: untimed, B: 15:00, C: untimed, D gets 10:00.
it('keeps untimed stops in place, sorts the timed ones and sends the day and its vias like REST', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const day = createDay(testDb, trip.id);
const place = createPlace(testDb, trip.id);
const [a, b, c, d] = [0, 1, 2, 3].map(i => createDayAssignment(testDb, day.id, place.id, { order_index: i }).id);
testDb.prepare('UPDATE day_assignments SET assignment_time = ? WHERE id = ?').run('15:00', b);
const via = Number(testDb.prepare(
'INSERT INTO roadtrip_vias (day_id, after_order_index, sequence, lat, lng) VALUES (?, 1, 0, 48.1, 11.5)'
).run(day.id).lastInsertRowid);
await withHarness(user.id, async (h) => {
const result = await h.client.callTool({ name: 'update_assignment_time', arguments: { tripId: trip.id, assignmentId: d, place_time: '10:00' } });
expect((parseToolResult(result) as { assignment: { assignment_time: string } }).assignment.assignment_time).toBe('10:00');
});
const order = (testDb.prepare('SELECT id FROM day_assignments WHERE day_id = ? ORDER BY order_index').all(day.id) as { id: number }[]).map(r => r.id);
expect(order).toEqual([a, d, b, c]);
expect(testDb.prepare('SELECT after_order_index FROM roadtrip_vias WHERE id = ?').get(via)).toEqual({ after_order_index: 2 });
expect(broadcastMock).toHaveBeenCalledWith(trip.id, 'assignment:reordered', expect.objectContaining({ dayId: day.id, orderedIds: [a, d, b, c] }));
expect(broadcastMock).toHaveBeenCalledWith(trip.id, 'roadtripVia:changed', expect.objectContaining({
dayId: day.id,
vias: [expect.objectContaining({ id: via, after_order_index: 2 })],
}));
});
it('sends only the row when the start leaves the day as it was', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const day = createDay(testDb, trip.id);
const place = createPlace(testDb, trip.id);
const [a, b, c] = [0, 1, 2].map(i => createDayAssignment(testDb, day.id, place.id, { order_index: i }).id);
await withHarness(user.id, async (h) => {
await h.client.callTool({ name: 'update_assignment_time', arguments: { tripId: trip.id, assignmentId: c, place_time: '14:00' } });
});
const order = (testDb.prepare('SELECT id FROM day_assignments WHERE day_id = ? ORDER BY order_index').all(day.id) as { id: number }[]).map(r => r.id);
expect(order).toEqual([a, b, c]);
expect(broadcastMock.mock.calls.map(call => call[1])).toEqual(['assignment:updated']);
});
it('leaves a day out of time order as it is when the call names only the end', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const day = createDay(testDb, trip.id);
const place = createPlace(testDb, trip.id);
// B starts before A and was put behind it on purpose.
const [a, b] = [0, 1].map(i => createDayAssignment(testDb, day.id, place.id, { order_index: i }).id);
testDb.prepare("UPDATE day_assignments SET assignment_time = '14:00' WHERE id = ?").run(a);
testDb.prepare("UPDATE day_assignments SET assignment_time = '10:00' WHERE id = ?").run(b);
const via = Number(testDb.prepare(
'INSERT INTO roadtrip_vias (day_id, after_order_index, sequence, lat, lng) VALUES (?, 0, 0, 48.1, 11.5)'
).run(day.id).lastInsertRowid);
await withHarness(user.id, async (h) => {
const result = await h.client.callTool({ name: 'update_assignment_time', arguments: { tripId: trip.id, assignmentId: b, end_time: '11:30' } });
expect((parseToolResult(result) as { assignment: { assignment_time: string; assignment_end_time: string } }).assignment)
.toMatchObject({ assignment_time: '10:00', assignment_end_time: '11:30' });
});
const order = (testDb.prepare('SELECT id FROM day_assignments WHERE day_id = ? ORDER BY order_index').all(day.id) as { id: number }[]).map(r => r.id);
expect(order).toEqual([a, b]);
expect(testDb.prepare('SELECT after_order_index FROM roadtrip_vias WHERE id = ?').get(via)).toEqual({ after_order_index: 0 });
expect(broadcastMock.mock.calls.map(call => call[1])).toEqual(['assignment:updated']);
});
it('returns error when assignment not found', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
+13 -2
View File
@@ -10,6 +10,7 @@
* (it replaced ADMIN-BR-001 when the old admin bridge died with the cron move).
*/
import { describe, it, expect, vi, beforeAll, beforeEach, afterEach, afterAll } from 'vitest';
import { ADDON_IDS, MCP_GATED_ADDON_IDS } from '../../../src/addons';
// ── DB setup ──────────────────────────────────────────────────────────────────
@@ -502,9 +503,19 @@ describe('updateAddon', () => {
// real flip of an MCP-relevant addon → invalidate
expect((updateAddon('packing', { enabled: false }) as any).mcpAffected).toBe(true);
expect((updateAddon('packing', { enabled: true }) as any).mcpAffected).toBe(true);
// real flip of an addon with no MCP surface → sessions survive
// real flip of an addon with no MCP surface → sessions survive. Taken from
// the list rather than named, because an addon that grows MCP tools joins it
// and would otherwise turn this assertion false without changing anything
// it is actually about (documents did, when document sync landed).
const noMcp = Object.values(ADDON_IDS).find(id => !MCP_GATED_ADDON_IDS.includes(id));
if (noMcp) {
const flip = updateAddon(noMcp, { enabled: false }) as any;
if (!flip.error) expect(flip.mcpAffected).toBe(false);
}
// and the one this change put on the list carries the opposite verdict
const docsFlip = updateAddon('documents', { enabled: false }) as any;
if (!docsFlip.error) expect(docsFlip.mcpAffected).toBe(false);
if (!docsFlip.error) expect(docsFlip.mcpAffected).toBe(true);
});
it('ADMIN-SVC-087 — refuses to enable a photo provider while journey is off', () => {
@@ -99,13 +99,32 @@ describe('AssignmentOpsController (parity with the per-assignment op routes)', (
it('PUT /:id/time 404 missing, else updates', () => {
expect(thrown(() => new AssignmentOpsController(svc({ getAssignmentForTrip: vi.fn().mockReturnValue(undefined) } as Partial<AssignmentsService>)).time(user, '5', '9', {}))).toEqual({ status: 404, body: { error: 'Assignment not found' } });
const updateTime = vi.fn().mockReturnValue({ id: 9 }); const broadcast = vi.fn(); const reconcile = vi.fn();
const updateTime = vi.fn().mockReturnValue({ assignment: { id: 9 }, reordered: null, vias: null }); const broadcast = vi.fn(); const reconcile = vi.fn();
const s = svc({ getAssignmentForTrip: vi.fn().mockReturnValue({ id: 9 }), updateTime, broadcast, reconcile } as Partial<AssignmentsService>);
expect(new AssignmentOpsController(s).time(user, '5', '9', { place_time: '10:00' }, 'sock')).toEqual({ assignment: { id: 9 } });
expect(updateTime).toHaveBeenCalledWith('9', '10:00', undefined);
// Nothing moved, so the row is all collaborators hear about.
expect(broadcast.mock.calls).toEqual([['5', 'assignment:updated', { assignment: { id: 9 } }, 'sock']]);
expect(reconcile).toHaveBeenCalledWith('5', 'sock');
});
it('PUT /:id/time sends the whole day and the re-pinned vias to every socket when a start moved stops', () => {
const reordered = { dayId: 3, orderedIds: [7, 9, 8] };
const vias = { dayId: 3, vias: [{ id: 40, day_id: 3, after_order_index: 1, sequence: 0, lat: 1, lng: 2 }] };
const updateTime = vi.fn().mockReturnValue({ assignment: { id: 9 }, reordered, vias }); const broadcast = vi.fn();
const s = svc({ getAssignmentForTrip: vi.fn().mockReturnValue({ id: 9 }), updateTime, broadcast } as Partial<AssignmentsService>);
expect(new AssignmentOpsController(s).time(user, '5', '9', { place_time: '10:00' }, 'sock')).toEqual({ assignment: { id: 9 } });
expect(broadcast.mock.calls).toEqual([
['5', 'assignment:updated', { assignment: { id: 9 } }, 'sock'],
// The writer's socket too, so the order and the vias pinned to it arrive
// together there as well, a replayed offline save included.
['5', 'assignment:reordered', reordered, undefined],
['5', 'roadtripVia:changed', vias, undefined],
]);
});
it('PUT /:id/notes 404 missing, else updates + broadcasts (#2163)', () => {
expect(thrown(() => new AssignmentOpsController(svc({ getAssignmentForTrip: vi.fn().mockReturnValue(undefined) } as Partial<AssignmentsService>)).notes(user, '5', '9', { notes: 'n' }))).toEqual({ status: 404, body: { error: 'Assignment not found' } });
const updateNotes = vi.fn().mockReturnValue({ id: 9, notes: 'Book the 10:00 entry' }); const broadcast = vi.fn(); const reconcile = vi.fn();
@@ -348,25 +348,56 @@ describe('getParticipants / setParticipants', () => {
// ── updateTime / setLegTransportMode ──────────────────────────────────────────
describe('updateTime', () => {
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
/** The day the way every reader lists it: by order_index, then insertion. */
const dayOrder = (dayId: number) =>
(testDb.prepare('SELECT id FROM day_assignments WHERE day_id = ? ORDER BY order_index, created_at, id').all(dayId) as { id: number }[]).map(r => r.id);
/**
* A day of stops at the given order_index keys, each with its own start (null for
* none). Times go in by SQL so the setup itself never sorts anything.
*/
function dayOf(stops: [string | null, number][]) {
const { trip, day, place } = fixture();
const ids = stops.map(([time, orderIndex]) => {
const a = createDayAssignment(testDb, day.id, place.id, { order_index: orderIndex });
if (time) testDb.prepare('UPDATE day_assignments SET assignment_time = ? WHERE id = ?').run(time, a.id);
return a.id;
});
return { trip, day, ids };
}
function addVia(dayId: number, afterOrderIndex: number, sequence = 0): number {
return Number(testDb.prepare(
'INSERT INTO roadtrip_vias (day_id, after_order_index, sequence, lat, lng) VALUES (?, ?, ?, 48.1, 11.5)'
).run(dayId, afterOrderIndex, sequence).lastInsertRowid);
}
const viaAnchors = (dayId: number) =>
testDb.prepare('SELECT id, after_order_index, sequence FROM roadtrip_vias WHERE day_id = ? ORDER BY id').all(dayId);
it('ASG-SVC-021: persists both times and re-selects the nested shape', () => {
const { day, place } = fixture();
const a = createDayAssignment(testDb, day.id, place.id);
const updated = svc.updateTime(a.id, '09:00', '11:30');
expect(updated).toMatchObject({ id: a.id, assignment_time: '09:00', assignment_end_time: '11:30' });
const { assignment } = svc.updateTime(a.id, '09:00', '11:30');
expect(assignment).toMatchObject({ id: a.id, assignment_time: '09:00', assignment_end_time: '11:30' });
});
it('ASG-SVC-022: a truthy place_time auto-sorts the day chronologically (untimed keep relative position)', () => {
const { day, place } = fixture();
const a = createDayAssignment(testDb, day.id, place.id, { order_index: 0 });
const b = createDayAssignment(testDb, day.id, place.id, { order_index: 1 });
const c = createDayAssignment(testDb, day.id, place.id, { order_index: 2 });
it('ASG-SVC-022: a start sorts the timed stops and leaves every untimed stop behind the one it followed', () => {
// The tester's day: two stops without a time, then a start on the last one. It
// used to jump to the top, because every untimed stop was appended after it.
const untimedHead = dayOf([[null, 0], [null, 1], [null, 2]]);
svc.updateTime(untimedHead.ids[2], '14:00', null);
expect(dayOrder(untimedHead.day.id)).toEqual(untimedHead.ids);
svc.updateTime(c.id, '08:00', null);
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect([order(c.id), order(a.id), order(b.id)]).toEqual([0, 1, 2]);
const between = dayOf([['09:00', 0], [null, 1], [null, 2]]);
svc.updateTime(between.ids[2], '14:00', null);
expect(dayOrder(between.day.id)).toEqual(between.ids);
svc.updateTime(a.id, '07:00', null);
expect([order(a.id), order(c.id), order(b.id)]).toEqual([0, 1, 2]);
// Timed stops still keep time order among themselves. A has nothing timed in
// front of it and stays first.
const [a, b, c] = dayOf([[null, 0], ['15:00', 1], [null, 2]]).ids;
svc.updateTime(c, '10:00', null);
expect([order(a), order(c), order(b)]).toEqual([0, 1, 2]);
});
it('ASG-SVC-022b: a booked night sorts by its check-in, which is where its hour lives', () => {
@@ -384,34 +415,33 @@ describe('updateTime', () => {
svc.updateTime(afternoon.id, '16:00', null);
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect(order(night.id)).toBeLessThan(order(afternoon.id));
});
it('ASG-SVC-023: clearing with null persists but skips the auto-sort (falsy gate)', () => {
const { day, place } = fixture();
const a = createDayAssignment(testDb, day.id, place.id, { order_index: 0 });
const b = createDayAssignment(testDb, day.id, place.id, { order_index: 1 });
svc.updateTime(b.id, '06:00', null); // sorts b first
const updated = svc.updateTime(b.id, null, null); // clear — no re-sort
expect(updated!.assignment_time).toBeNull();
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect(order(b.id)).toBe(0);
expect(order(a.id)).toBe(1);
const { day, ids: [a, b, c] } = dayOf([['09:00', 0], [null, 1], ['12:00', 2]]);
svc.updateTime(b, '06:00', null); // sorts b first
expect(dayOrder(day.id)).toEqual([b, a, c]);
// Dragged back out of time order on purpose: a sort now would put b first again,
// with c behind it.
svc.reorderAssignments(day.id, [a, b, c]);
const { assignment } = svc.updateTime(c, null, null); // clear, no re-sort
expect(assignment!.assignment_time).toBeNull();
expect(dayOrder(day.id)).toEqual([a, b, c]);
});
it('ASG-SVC-027: an empty-string time clears the override like null (and skips the auto-sort)', () => {
const { day, place } = fixture();
const a = createDayAssignment(testDb, day.id, place.id, { order_index: 0 });
const b = createDayAssignment(testDb, day.id, place.id, { order_index: 1 });
svc.updateTime(b.id, '06:00', '07:00'); // sorts b first
const updated = svc.updateTime(b.id, '', '');
expect(updated!.assignment_time).toBeNull();
expect(updated!.assignment_end_time).toBeNull();
expect(testDb.prepare('SELECT assignment_time FROM day_assignments WHERE id = ?').get(b.id)).toEqual({ assignment_time: null });
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect(order(b.id)).toBe(0);
expect(order(a.id)).toBe(1);
const { day, ids: [a, b, c] } = dayOf([['09:00', 0], [null, 1], ['12:00', 2]]);
svc.updateTime(b, '06:00', '07:00'); // sorts b first
expect(dayOrder(day.id)).toEqual([b, a, c]);
svc.reorderAssignments(day.id, [a, b, c]);
const { assignment } = svc.updateTime(c, '', '');
expect(assignment!.assignment_time).toBeNull();
expect(assignment!.assignment_end_time).toBeNull();
expect(testDb.prepare('SELECT assignment_time FROM day_assignments WHERE id = ?').get(c)).toEqual({ assignment_time: null });
expect(dayOrder(day.id)).toEqual([a, b, c]);
});
it('ASG-SVC-024: a time without ":" sorts last among timed via the 99:99 sentinel', () => {
@@ -420,10 +450,164 @@ describe('updateTime', () => {
const b = createDayAssignment(testDb, day.id, place.id, { order_index: 1 });
svc.updateTime(a.id, 'morning', null); // no colon → '99:99' in the sort
svc.updateTime(b.id, '13:00', null);
const order = (id: number) => (testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(id) as { order_index: number }).order_index;
expect(order(b.id)).toBe(0);
expect(order(a.id)).toBe(1);
});
it('ASG-SVC-033: writes nothing but the time when the day is already in order', () => {
// Keys with gaps, as a deleted stop leaves them. A day the start leaves in order
// keeps them: nothing about it changed.
const { day, ids: [a, b, c] } = dayOf([['09:00', 0], [null, 5], [null, 9]]);
const via = addVia(day.id, 0);
const changes = () => (testDb.prepare('SELECT total_changes() AS n').get() as { n: number }).n;
const before = changes();
const update = svc.updateTime(c, '14:00', null);
expect(changes() - before).toBe(1);
expect([order(a), order(b), order(c)]).toEqual([0, 5, 9]);
expect(viaAnchors(day.id)).toEqual([{ id: via, after_order_index: 0, sequence: 0 }]);
expect(update.reordered).toBeNull();
expect(update.vias).toBeNull();
});
it('ASG-SVC-034: numbers a reordered day from 0, the key each client gives a stop of the order it is sent', () => {
// A gap, as a deleted stop leaves it. Kept, the writer would read 0, 4, 7 back and
// everyone else would apply 0, 1, 2, and a note at 3 would sit behind a different
// stop for each of them.
const { day, ids: [a, b, c] } = dayOf([[null, 0], ['15:00', 4], [null, 7]]);
const changes = () => (testDb.prepare('SELECT total_changes() AS n').get() as { n: number }).n;
const before = changes();
const update = svc.updateTime(c, '10:00', null);
expect(update.reordered).toEqual({ dayId: day.id, orderedIds: [a, c, b] });
expect(update.reordered!.orderedIds.map(order)).toEqual([0, 1, 2]);
// The time, then c and b. A already holds 0 and is not written.
expect(changes() - before).toBe(3);
});
it('ASG-SVC-035: numbers a day with colliding keys from 0 again, since a shared key cannot hold an order', () => {
// A move drops a stop in at order_index 0 next to the one already there.
const { day, ids: [a, b, c] } = dayOf([[null, 0], ['15:00', 0], [null, 0]]);
svc.updateTime(c, '10:00', null);
expect(dayOrder(day.id)).toEqual([a, c, b]);
expect([order(a), order(c), order(b)]).toEqual([0, 1, 2]);
});
it('ASG-SVC-036: keeps each drawn road behind the stop it was drawn after, counting located stops only', () => {
// X has no coordinates, so the router never sees it and the vias do not count it.
const { trip, day, ids: [a, x, b, c, d] } = dayOf([[null, 0], [null, 1], ['15:00', 2], [null, 3], [null, 4]]);
const unmapped = createPlace(testDb, trip.id, { name: 'Somewhere unmapped' });
testDb.prepare('UPDATE places SET lat = NULL, lng = NULL WHERE id = ?').run(unmapped.id);
testDb.prepare('UPDATE day_assignments SET place_id = ? WHERE id = ?').run(unmapped.id, x);
const afterA = addVia(day.id, 0);
const afterB = addVia(day.id, 1);
const afterBToo = addVia(day.id, 1, 1);
addVia(day.id, 2); // after C
// D at 10:00 goes in front of B at 15:00, and C stays behind B: A, X, D, B, C.
const update = svc.updateTime(d, '10:00', null);
expect(dayOrder(day.id)).toEqual([a, x, d, b, c]);
// A keeps its leg, B's two points follow B to 2 in their order, and C, now last,
// has no leg left to bend.
expect(viaAnchors(day.id)).toEqual([
{ id: afterA, after_order_index: 0, sequence: 0 },
{ id: afterB, after_order_index: 2, sequence: 0 },
{ id: afterBToo, after_order_index: 2, sequence: 1 },
]);
expect(update.reordered).toEqual({ dayId: day.id, orderedIds: [a, x, d, b, c] });
expect(update.vias?.dayId).toBe(day.id);
expect(update.vias?.vias.map(v => [v.id, v.after_order_index])).toEqual([[afterA, 0], [afterB, 2], [afterBToo, 2]]);
});
it('ASG-SVC-037: reports no vias when the reorder leaves every one on its stop', () => {
const { day, ids: [a, b, c] } = dayOf([[null, 0], ['15:00', 1], [null, 2]]);
const via = addVia(day.id, 0);
const update = svc.updateTime(c, '10:00', null);
expect(update.reordered).toEqual({ dayId: day.id, orderedIds: [a, c, b] });
expect(viaAnchors(day.id)).toEqual([{ id: via, after_order_index: 0, sequence: 0 }]);
expect(update.vias).toBeNull();
});
it('ASG-SVC-038: leaves every via alone when the sort moves only stops without coordinates', () => {
// X has no coordinates. It goes between A and B, and the located stops stay A, B.
const { trip, day, ids: [a, b, x] } = dayOf([['09:00', 0], ['12:00', 1], [null, 2]]);
const unmapped = createPlace(testDb, trip.id, { name: 'Somewhere unmapped' });
testDb.prepare('UPDATE places SET lat = NULL, lng = NULL WHERE id = ?').run(unmapped.id);
testDb.prepare('UPDATE day_assignments SET place_id = ? WHERE id = ?').run(unmapped.id, x);
const afterA = addVia(day.id, 0);
// Behind the day's last stop: the drive into the next day on a trip with connected
// days. It was deleted here, since a last stop has no leg of its own.
const intoTomorrow = addVia(day.id, 1);
const update = svc.updateTime(x, '10:00', null);
expect(dayOrder(day.id)).toEqual([a, x, b]);
expect(update.reordered).toEqual({ dayId: day.id, orderedIds: [a, x, b] });
expect(viaAnchors(day.id)).toEqual([
{ id: afterA, after_order_index: 0, sequence: 0 },
{ id: intoTomorrow, after_order_index: 1, sequence: 0 },
]);
expect(update.vias).toBeNull();
});
it('ASG-SVC-039: keeps the via behind the last stop when that stop is still last', () => {
const { day, ids: [a, b, c, d] } = dayOf([[null, 0], ['15:00', 1], [null, 2], ['18:00', 3]]);
const afterB = addVia(day.id, 1);
const intoTomorrow = addVia(day.id, 3);
// C at 10:00 goes in front of B. D stays last, and so does the drive out of it.
const update = svc.updateTime(c, '10:00', null);
expect(dayOrder(day.id)).toEqual([a, c, b, d]);
expect(viaAnchors(day.id)).toEqual([
{ id: afterB, after_order_index: 2, sequence: 0 },
{ id: intoTomorrow, after_order_index: 3, sequence: 0 },
]);
expect(update.vias?.vias.map(v => [v.id, v.after_order_index])).toEqual([[afterB, 2], [intoTomorrow, 3]]);
});
it('ASG-SVC-040: an End saved with the start as it stood leaves a day dragged out of time order alone', () => {
// B belongs first by time, and the traveller put it second on purpose.
const { day, ids: [a, b, c] } = dayOf([['14:00', 0], ['10:00', 1], ['16:00', 2]]);
testDb.prepare("UPDATE day_assignments SET assignment_end_time = '11:00' WHERE id = ?").run(b);
const afterA = addVia(day.id, 0);
// The stay dialog taking the End off, then the place form setting a new one.
for (const end of [null, '12:00']) {
const update = svc.updateTime(b, '10:00', end);
expect(update.assignment).toMatchObject({ assignment_time: '10:00', assignment_end_time: end });
expect(update.reordered).toBeNull();
expect(update.vias).toBeNull();
expect(dayOrder(day.id)).toEqual([a, b, c]);
expect(viaAnchors(day.id)).toEqual([{ id: afterA, after_order_index: 0, sequence: 0 }]);
}
// A start that moves still sorts, and the via follows A.
const update = svc.updateTime(b, '09:00', '12:00');
expect(dayOrder(day.id)).toEqual([b, a, c]);
expect(update.reordered).toEqual({ dayId: day.id, orderedIds: [b, a, c] });
expect(viaAnchors(day.id)).toEqual([{ id: afterA, after_order_index: 1, sequence: 0 }]);
});
it('ASG-SVC-041: a visit given the start its place already had does not sort either', () => {
// The place form fills Start from the place when the visit has none of its own.
const { trip, day, ids: [a, b] } = dayOf([['14:00', 0], [null, 1]]);
const timedPlace = createPlace(testDb, trip.id, { name: 'Opens at ten' });
testDb.prepare("UPDATE places SET place_time = '10:00' WHERE id = ?").run(timedPlace.id);
testDb.prepare('UPDATE day_assignments SET place_id = ? WHERE id = ?').run(timedPlace.id, b);
const update = svc.updateTime(b, '10:00', '11:00');
expect(update.reordered).toBeNull();
expect(dayOrder(day.id)).toEqual([a, b]);
});
});
describe('updateNotes (#2163)', () => {
@@ -451,7 +635,9 @@ describe('updateNotes (#2163)', () => {
svc.updateTime(a.id, '09:00', '10:00');
const updated = svc.updateNotes(a.id, 'note');
expect(updated).toMatchObject({ assignment_time: '09:00', assignment_end_time: '10:00' });
expect((testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(a.id) as { order_index: number }).order_index).toBe(0);
// 3, not 0: a start on a day already in order no longer renumbers it, so the key
// this note edit must not touch is still the one the stop was created with.
expect((testDb.prepare('SELECT order_index FROM day_assignments WHERE id = ?').get(a.id) as { order_index: number }).order_index).toBe(3);
});
});
@@ -0,0 +1,647 @@
/**
* The configuration half of document sync: the connection a trip admin creates
* and the binding that points it at a folder or a tag.
*
* The database here is real in-memory SQLite with the full schema, and the
* at-rest crypto is real too. Both are deliberate. Almost everything this
* service owns is either a credential or a decision about one, and a spy on
* `db.run` would only prove that some strings were handed along. Reading the
* row back is the assertion. A stubbed `maybe_encrypt_api_key` would let a
* regression that stores a Paperless token in plaintext pass every case below.
*
* Only the SSRF guard is faked, because it resolves DNS: the guard is not what
* is under test, and a suite that needs a working resolver is a suite that
* fails on someone's train.
*/
import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest';
const { testDb, dbMock } = vi.hoisted(() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const Database = require('better-sqlite3');
const db = new Database(':memory:');
db.exec('PRAGMA journal_mode = WAL');
db.exec('PRAGMA foreign_keys = ON');
db.exec('PRAGMA busy_timeout = 5000');
return {
testDb: db,
dbMock: {
db,
closeDb: () => {},
reinitialize: () => {},
getPlaceWithTags: () => null,
canAccessTrip: () => undefined,
isOwner: () => false,
},
};
});
vi.mock('../../../../src/db/database', () => dbMock);
// The whole guard surface, not only `checkSsrf`: modules on the import graph
// pull other names off it, and a factory mock that omits one throws on access
// rather than falling back to the real export.
const { checkSsrf } = vi.hoisted(() => ({ checkSsrf: vi.fn() }));
vi.mock('../../../../src/utils/ssrfGuard', () => ({
checkSsrf,
safeFetch: vi.fn(),
safeFetchFollow: vi.fn(),
safeFetchAdminConfigured: vi.fn(),
safeFetchLlm: vi.fn(),
createPinnedDispatcher: vi.fn(() => ({})),
SsrfBlockedError: class extends Error {},
}));
import { DOCSYNC_SECRET_MASK, type DocsyncConnectionInput, type DocsyncLinkInput } from '@trek/shared';
import { createTables } from '../../../../src/db/schema';
import { runMigrations } from '../../../../src/db/migrations';
import { resetTestDb } from '../../../helpers/test-db';
import { createTrip, createUser } from '../../../helpers/factories';
import { DatabaseService } from '../../../../src/nest/database/database.service';
import {
DocSyncConfigService,
type ConnectionRow,
type LinkRow,
} from '../../../../src/nest/doc-sync/doc-sync-config.service';
import type { DocumentProviderRegistry } from '../../../../src/nest/doc-sync/document-provider.registry';
// `if (!result.success)` does not narrow in this workspace: strictNullChecks is
// off, so the boolean discriminant stops discriminating. The domain's own
// predicate is what every call site uses instead.
import { docFailed } from '../../../../src/nest/doc-sync/document-provider';
const svc = new DocSyncConfigService(new DatabaseService(testDb), {} as DocumentProviderRegistry);
const TOKEN = 'paperless-token-1234';
let OWNER = 0;
let MEMBER = 0;
let TRIP = 0;
function connectionInput(over: Partial<DocsyncConnectionInput> = {}): DocsyncConnectionInput {
return {
providerId: 'paperless',
baseUrl: 'https://paperless.example.com',
credentials: { api_token: TOKEN },
allowInsecureTls: false,
...over,
};
}
async function connect(over: Partial<DocsyncConnectionInput> = {}, userId = OWNER): Promise<ConnectionRow> {
const result = await svc.upsertConnection(TRIP, userId, connectionInput(over));
if (docFailed(result)) throw new Error(`fixture connection refused: ${JSON.stringify(result.error)}`);
return result.data;
}
function linkInput(connectionId: number, over: Partial<DocsyncLinkInput> = {}): DocsyncLinkInput {
return {
connectionId,
scopeKey: 'tag:1',
remoteLabel: 'Japan 2026',
direction: 'both',
deletePolicy: 'unlink',
conflictPolicy: 'manual',
syncEnabled: true,
...over,
};
}
function link(connectionId: number, over: Partial<DocsyncLinkInput> = {}): LinkRow {
const result = svc.createLink(TRIP, OWNER, linkInput(connectionId, over));
if (docFailed(result)) throw new Error(`fixture link refused: ${JSON.stringify(result.error)}`);
return result.data;
}
function storedSecret(connectionId: number): string | undefined {
const row = svc.getConnection(connectionId) as ConnectionRow;
return svc.toRef(row).secrets.api_token;
}
beforeAll(() => {
createTables(testDb);
runMigrations(testDb);
});
beforeEach(() => {
// The document_* tables are not in resetTestDb's list, and they reference
// trips and users, so they go first, before the rows they hang off vanish.
testDb.exec('DELETE FROM document_sync_items; DELETE FROM trip_document_links; DELETE FROM document_connections;');
resetTestDb(testDb);
vi.clearAllMocks();
checkSsrf.mockResolvedValue({ allowed: true, isPrivate: false, resolvedIp: '203.0.113.10' });
OWNER = createUser(testDb, { username: 'owner', email: 'owner@test.local' }).user.id;
MEMBER = createUser(testDb, { username: 'member', email: 'member@test.local' }).user.id;
TRIP = createTrip(testDb, OWNER, { title: 'Japan' }).id;
testDb.prepare('INSERT INTO trip_members (trip_id, user_id) VALUES (?, ?)').run(TRIP, MEMBER);
});
afterAll(() => {
testDb.close();
});
describe('upsertConnection secrets', () => {
it('keeps the stored credential when the form comes back with the field blank', async () => {
const created = await connect();
await connect({ credentials: { api_token: '' } });
expect(storedSecret(created.id)).toBe(TOKEN);
});
it('keeps it when the form posts back the mask it was shown', async () => {
// This is what makes a GET-then-PUT round trip safe: the client renders the
// form from the masked view and never has to hold the real credential.
const created = await connect();
await connect({ credentials: { api_token: DOCSYNC_SECRET_MASK } });
expect(storedSecret(created.id)).toBe(TOKEN);
});
it('keeps it when the field is missing from the payload altogether', async () => {
const created = await connect();
await connect({ credentials: {} });
expect(storedSecret(created.id)).toBe(TOKEN);
});
it('replaces it when a new credential actually arrives', async () => {
const created = await connect();
await connect({ credentials: { api_token: 'rotated-token' } });
expect(storedSecret(created.id)).toBe('rotated-token');
});
it('updates the trip\'s connection in place rather than adding a second one', async () => {
const first = await connect();
const second = await connect({ baseUrl: 'https://paperless.example.com/', credentials: {} });
expect(second.id).toBe(first.id);
const count = testDb
.prepare('SELECT COUNT(*) AS n FROM document_connections WHERE trip_id = ?')
.get(TRIP) as { n: number };
expect(count.n).toBe(1);
});
it('carries a non-secret setting forward when a later save omits it', async () => {
const created = await svc.upsertConnection(TRIP, OWNER, {
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'anna', app_password: 'app-pw', base_path: '/TREK' },
allowInsecureTls: false,
});
expect(created.success).toBe(true);
await svc.upsertConnection(TRIP, OWNER, {
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'anna' },
allowInsecureTls: false,
});
const row = svc.listConnections(TRIP).find((c) => c.provider_id === 'nextcloud') as ConnectionRow;
expect(svc.toRef(row).settings).toMatchObject({ login_name: 'anna', base_path: '/TREK' });
expect(svc.toRef(row).secrets.app_password).toBe('app-pw');
});
});
describe('publicConnection', () => {
it('reports that a secret is set without ever carrying its value', async () => {
const created = await connect();
const view = svc.publicConnection(svc.getConnection(created.id) as ConnectionRow);
expect(view.secrets).toEqual({ api_token: DOCSYNC_SECRET_MASK });
expect(JSON.stringify(view)).not.toContain(TOKEN);
});
it('leaves a secret it has no value for out of the map entirely', async () => {
// Masking an unset field would tell the client "already configured" and the
// connection form would render an optional credential as one that is done.
const created = await svc.upsertConnection(TRIP, OWNER, {
providerId: 'synologydrive',
baseUrl: 'https://nas.example.com:5001',
credentials: { username: 'anna', password: 'nas-pw' },
allowInsecureTls: false,
});
expect(created.success).toBe(true);
const row = svc.listConnections(TRIP).find((c) => c.provider_id === 'synologydrive') as ConnectionRow;
expect(svc.publicConnection(row).secrets).toEqual({ password: DOCSYNC_SECRET_MASK });
});
it('reads a capabilities blob left behind by another build as unknown, not as a crash', async () => {
const created = await connect();
testDb.prepare('UPDATE document_connections SET capabilities = ? WHERE id = ?').run('{not json', created.id);
const view = svc.publicConnection(svc.getConnection(created.id) as ConnectionRow);
expect(view.capabilities).toBeNull();
});
});
describe('a secret the provider earned itself', () => {
// The shape the Synology adapter stores: account digest, colon, DSM's token.
const EARNED = 'a1b2c3:DEVICE-7';
function nas(credentials: Record<string, string> = { username: 'anna', password: 'nas-pw', otp_code: '123456' }) {
return connect({ providerId: 'synologydrive', baseUrl: 'https://nas.example.com:5001', credentials });
}
function secretsOf(connectionId: number): Readonly<Record<string, string>> {
return svc.toRef(svc.getConnection(connectionId) as ConnectionRow).secrets;
}
it('is stored through the ref the adapter was handed, encrypted with the form\'s secrets', async () => {
const conn = await nas();
svc.toRef(conn).saveSecret!('device_token', EARNED);
expect(secretsOf(conn.id)).toEqual({ password: 'nas-pw', otp_code: '123456', device_token: EARNED });
expect((svc.getConnection(conn.id) as ConnectionRow).secrets).not.toContain('DEVICE-7');
});
it('never reaches a client, not even as a mask', async () => {
const conn = await nas();
svc.saveEarnedSecret(conn.id, 'device_token', EARNED);
const view = svc.publicConnection(svc.getConnection(conn.id) as ConnectionRow);
expect(view.secrets).toEqual({ password: DOCSYNC_SECRET_MASK, otp_code: DOCSYNC_SECRET_MASK });
expect(JSON.stringify(view)).not.toContain('DEVICE-7');
expect(JSON.stringify(view)).not.toContain('device_token');
});
it('survives a form edit, whatever the form sends for the secrets it does show', async () => {
const conn = await nas();
svc.saveEarnedSecret(conn.id, 'device_token', EARNED);
await nas({ username: 'anna', password: 'rotated', otp_code: '' });
await nas({ username: 'anna', password: DOCSYNC_SECRET_MASK });
expect(secretsOf(conn.id)).toMatchObject({ password: 'rotated', device_token: EARNED });
});
it('cannot be planted or overwritten through the form', async () => {
const conn = await nas();
await nas({ username: 'anna', device_token: 'a1b2c3:PLANTED' });
expect(secretsOf(conn.id).device_token).toBeUndefined();
svc.saveEarnedSecret(conn.id, 'device_token', EARNED);
await nas({ username: 'anna', device_token: 'a1b2c3:PLANTED' });
expect(secretsOf(conn.id).device_token).toBe(EARNED);
expect(svc.toRef(svc.getConnection(conn.id) as ConnectionRow).settings).not.toHaveProperty('device_token');
});
it('is written against the row as it is now, so a run holding an old ref cannot undo a password edit', async () => {
const conn = await nas();
const ref = svc.toRef(conn);
await nas({ username: 'anna', password: 'rotated' });
ref.saveSecret!('device_token', EARNED);
expect(secretsOf(conn.id)).toMatchObject({ password: 'rotated', device_token: EARNED });
});
it('is dropped with null, and the form\'s secrets stay', async () => {
const conn = await nas();
svc.saveEarnedSecret(conn.id, 'device_token', EARNED);
svc.saveEarnedSecret(conn.id, 'device_token', null);
expect(secretsOf(conn.id)).toEqual({ password: 'nas-pw', otp_code: '123456' });
});
it('brings back no connection that was deleted while a run still held its ref', async () => {
const conn = await nas();
const ref = svc.toRef(conn);
svc.deleteConnection(conn.id);
ref.saveSecret!('device_token', EARNED);
expect(svc.getConnection(conn.id)).toBeUndefined();
});
});
describe('validateBaseUrl', () => {
it('stores a private address, because a self-hosted instance is the point', async () => {
checkSsrf.mockResolvedValue({ allowed: false, isPrivate: true, error: 'private network' });
const verdict = await svc.validateBaseUrl('https://paperless.fritz.box');
expect(verdict).toEqual({ success: true, data: { url: 'https://paperless.fritz.box', isPrivate: true } });
const created = await connect({ baseUrl: 'https://paperless.fritz.box' });
expect(created.base_url).toBe('https://paperless.fritz.box');
});
it('refuses an address the guard rejects for any other reason, and writes nothing', async () => {
checkSsrf.mockResolvedValue({ allowed: false, isPrivate: false, error: 'Could not resolve hostname (ENOTFOUND)' });
const result = await svc.upsertConnection(TRIP, OWNER, connectionInput({ baseUrl: 'https://nope.invalid' }));
expect(result.success).toBe(false);
expect(result.success === false && result.error.code).toBe('ssrf_blocked');
expect(svc.listConnections(TRIP)).toHaveLength(0);
});
it('strips trailing slashes so the adapter can append its own path', async () => {
const created = await connect({ baseUrl: 'https://paperless.example.com///' });
expect(created.base_url).toBe('https://paperless.example.com');
});
it('refuses an empty address before it asks the guard anything', async () => {
const result = await svc.validateBaseUrl(' ');
expect(result.success).toBe(false);
expect(checkSsrf).not.toHaveBeenCalled();
});
});
describe('upsertConnection required fields', () => {
it('names the missing credential instead of storing a connection that cannot authenticate', async () => {
const result = await svc.upsertConnection(TRIP, OWNER, connectionInput({ credentials: {} }));
expect(result.success).toBe(false);
expect(result.success === false && result.error.detail).toContain('api_token');
expect(svc.listConnections(TRIP)).toHaveLength(0);
});
it('refuses a missing required field that is not a secret either', async () => {
const result = await svc.upsertConnection(TRIP, OWNER, {
providerId: 'papra',
baseUrl: 'https://papra.example.com',
credentials: { api_key: 'ppapi_x' },
allowInsecureTls: false,
});
expect(result.success).toBe(false);
expect(result.success === false && result.error.detail).toContain('organization_id');
});
it('accepts a provider whose optional credential is left blank', async () => {
// Synology's OTP is a one-time code, not a stored credential; demanding it
// would make every save after the first one impossible.
const result = await svc.upsertConnection(TRIP, OWNER, {
providerId: 'synologydrive',
baseUrl: 'https://nas.example.com:5001',
credentials: { username: 'anna', password: 'nas-pw', otp_code: '' },
allowInsecureTls: false,
});
expect(result.success).toBe(true);
});
});
describe('enabledProviderIds', () => {
it('names only the providers an admin switched on, in the order the shelf shows them', () => {
// Every document provider ships OFF, so an empty answer on a fresh install
// is the correct one and a non-empty one would be a policy change.
expect(svc.enabledProviderIds()).toEqual([]);
testDb.prepare("UPDATE document_providers SET enabled = 1 WHERE id IN ('nextcloud', 'paperless')").run();
expect(svc.enabledProviderIds()).toEqual(['paperless', 'nextcloud']);
});
});
describe('recordProbe', () => {
it('keeps what the last good probe learned when a later one fails', async () => {
// A token that expired overnight must not make TREK forget that this
// instance can accept uploads; the capabilities describe the provider, not
// the state of the credential.
const conn = await connect();
svc.recordProbe(conn.id, 'ok', null, { push: 'multipart' });
svc.recordProbe(conn.id, 'failed', '401 Unauthorized', null);
const view = svc.publicConnection(svc.getConnection(conn.id) as ConnectionRow);
expect(view).toMatchObject({
lastProbeState: 'failed',
lastProbeError: '401 Unauthorized',
capabilities: { push: 'multipart' },
});
});
});
describe('deleteConnection', () => {
it('drops the bindings that hung off it and leaves the other connection alone', async () => {
const doomed = await connect();
const kept = await connect({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'anna', app_password: 'app-pw' },
});
const doomedLink = link(doomed.id, { scopeKey: 'tag:1' });
const keptLink = link(kept.id, { scopeKey: 'folder:1' });
testDb
.prepare('INSERT INTO document_sync_items (link_id, trip_id, trek_doc_uid) VALUES (?, ?, ?)')
.run(doomedLink.id, TRIP, 'uid-a');
svc.deleteConnection(doomed.id);
expect(svc.getConnection(doomed.id)).toBeUndefined();
expect(svc.listLinks(TRIP).map((l) => l.id)).toEqual([keptLink.id]);
expect(testDb.prepare('SELECT COUNT(*) AS n FROM document_sync_items').get()).toEqual({ n: 0 });
});
});
describe('toRef', () => {
it('names the row and not only its id, which a backup restored in place hands out again', async () => {
const first = await connect();
testDb.prepare("UPDATE document_connections SET created_at = '2026-09-01 08:00:00' WHERE id = ?").run(first.id);
const before = svc.toRef(svc.getConnection(first.id) as ConnectionRow);
// What the restore of an older backup does to this table: the row is gone
// and the id sequence is back where it stood before the row was made.
testDb.prepare('DELETE FROM document_connections WHERE id = ?').run(first.id);
testDb.prepare("UPDATE sqlite_sequence SET seq = ? WHERE name = 'document_connections'").run(first.id - 1);
const after = svc.toRef(await connect());
expect(after.connectionId).toBe(before.connectionId);
expect(before.createdAt).toBe('2026-09-01 08:00:00');
expect(after.createdAt).not.toBe(before.createdAt);
});
});
describe('updateLink', () => {
it('changes only the fields the patch names', async () => {
const conn = await connect();
const created = link(conn.id, { direction: 'both', conflictPolicy: 'manual' });
const patched = svc.updateLink(created.id, { direction: 'pull', syncEnabled: false });
expect(patched).toMatchObject({
direction: 'pull',
sync_enabled: 0,
conflict_policy: 'manual',
remote_label: created.remote_label,
});
});
it('hands back the binding untouched when the patch is empty', async () => {
const conn = await connect();
const created = link(conn.id);
expect(svc.updateLink(created.id, {})).toEqual(created);
});
});
describe('toScopeRef', () => {
it('hands the adapter the anchor and the cursor of that binding, each in its own field', async () => {
// The adapters take a ref object rather than positional arguments because
// an id and a path swapped compile cleanly and then read the wrong folder.
const conn = await connect();
const created = svc.createLink(
TRIP,
OWNER,
linkInput(conn.id, { scopeKey: 'fileid:437', remoteRootId: '437', remoteRootPath: '/TREK/japan' }),
);
expect(created.success).toBe(true);
const row = created.success ? created.data : ({} as LinkRow);
testDb.prepare('UPDATE trip_document_links SET remote_cursor = ? WHERE id = ?').run('etag-9', row.id);
expect(svc.toScopeRef(svc.getLink(row.id) as LinkRow)).toEqual({
linkId: row.id,
tripId: TRIP,
scopeKey: 'fileid:437',
remoteRootId: '437',
remoteRootPath: '/TREK/japan',
cursor: 'etag-9',
});
});
});
describe('createLink', () => {
it('returns the binding that already exists rather than making a second one', async () => {
const conn = await connect();
const first = link(conn.id);
const again = svc.createLink(TRIP, MEMBER, linkInput(conn.id, { remoteLabel: 'renamed' }));
expect(again.success && again.data.id).toBe(first.id);
expect(svc.listLinks(TRIP)).toHaveLength(1);
});
it('binds a second scope of the same connection separately', async () => {
const conn = await connect();
const first = link(conn.id, { scopeKey: 'tag:1' });
const second = link(conn.id, { scopeKey: 'tag:2' });
expect(second.id).not.toBe(first.id);
expect(svc.listLinks(TRIP)).toHaveLength(2);
});
it('refuses a connection that belongs to somebody else\'s trip', async () => {
const otherTrip = createTrip(testDb, MEMBER, { title: 'Not yours' }).id;
const conn = await connect();
const result = svc.createLink(otherTrip, MEMBER, linkInput(conn.id));
expect(result.success).toBe(false);
expect(result.success === false && result.error.code).toBe('not_found');
expect(svc.listLinks(otherTrip)).toHaveLength(0);
});
it('refuses a connection id that does not exist at all', async () => {
const result = svc.createLink(TRIP, OWNER, linkInput(9999));
expect(result.success).toBe(false);
});
it('stores the anchor the picker sent back with the scope key', async () => {
const conn = await connect();
const result = svc.createLink(
TRIP,
OWNER,
linkInput(conn.id, { remoteRootId: '17', remoteRootPath: '/TREK/japan', remoteLabel: 'Japan 2026' }),
);
expect(result.success && result.data).toMatchObject({
remote_scope_key: 'tag:1',
remote_root_id: '17',
remote_root_path: '/TREK/japan',
remote_label: 'Japan 2026',
});
});
it('stores NULL for an anchor field the client left out', async () => {
// Synology binds by path alone, so its options carry no root id at all.
const conn = await connect();
const result = svc.createLink(TRIP, OWNER, linkInput(conn.id));
expect(result.success && result.data).toMatchObject({ remote_root_id: null, remote_root_path: null });
});
it('gives every binding its own webhook token and keeps the secret out of the public view', async () => {
// A leaked webhook URL has to identify exactly one binding, so revoking one
// cannot be used to poke another.
const conn = await connect();
const first = link(conn.id, { scopeKey: 'tag:1' });
const second = link(conn.id, { scopeKey: 'tag:2' });
expect(first.webhook_token).toBeTruthy();
expect(second.webhook_token).not.toBe(first.webhook_token);
expect(svc.getLinkByToken(first.webhook_token as string)?.id).toBe(first.id);
const secret = svc.webhookSecret(first);
expect(secret).toBeTruthy();
const view = svc.publicLink(first, 'https://trek.example.com');
expect(view.webhookSecret).toBe(DOCSYNC_SECRET_MASK);
expect(JSON.stringify(view)).not.toContain(secret);
});
});
describe('publicLink', () => {
it('names the store of a binding whose provider the admin has since switched off', async () => {
// The client only learns names for the providers that are on, so this is
// the one place a binding left behind can still get its name from.
const conn = await connect();
const bound = link(conn.id);
testDb.prepare("UPDATE document_providers SET enabled = 0 WHERE id = 'paperless'").run();
expect(svc.publicLink(bound, null)).toMatchObject({ providerId: 'paperless', providerName: 'Paperless-ngx' });
});
it('falls back to the provider id for a provider this build does not know', async () => {
const conn = await connect();
const bound = link(conn.id);
expect(svc.publicLink({ ...bound, provider_id: 'dropbox' }, null).providerName).toBe('dropbox');
});
});
describe('deleteLink', () => {
it('takes the pairing rows with it and leaves the other binding\'s alone', async () => {
const conn = await connect();
const doomed = link(conn.id, { scopeKey: 'tag:1' });
const kept = link(conn.id, { scopeKey: 'tag:2' });
const insert = testDb.prepare(
'INSERT INTO document_sync_items (link_id, trip_id, trek_doc_uid) VALUES (?, ?, ?)',
);
insert.run(doomed.id, TRIP, 'uid-a');
insert.run(doomed.id, TRIP, 'uid-b');
insert.run(kept.id, TRIP, 'uid-c');
svc.deleteLink(doomed.id);
expect(svc.getLink(doomed.id)).toBeUndefined();
expect(svc.getLink(kept.id)).toBeDefined();
const rows = testDb.prepare('SELECT link_id FROM document_sync_items').all() as { link_id: number }[];
expect(rows.map((r) => r.link_id)).toEqual([kept.id]);
});
it('leaves the connection behind, so unbinding one scope does not log the trip out', async () => {
const conn = await connect();
svc.deleteLink(link(conn.id).id);
expect(svc.getConnection(conn.id)).toBeDefined();
});
});
describe('markOrphanedLinks', () => {
async function bind(ownerUserId: number, providerId: 'paperless' | 'nextcloud'): Promise<LinkRow> {
const credentials =
providerId === 'paperless' ? { api_token: TOKEN } : { login_name: 'anna', app_password: 'app-pw' };
const conn = await connect({ providerId, credentials, baseUrl: `https://${providerId}.example.com` }, ownerUserId);
return link(conn.id, { scopeKey: `scope:${providerId}` });
}
it('disables the binding whose credential owner left the trip', async () => {
const orphan = await bind(MEMBER, 'paperless');
testDb.prepare('DELETE FROM trip_members WHERE trip_id = ? AND user_id = ?').run(TRIP, MEMBER);
expect(svc.markOrphanedLinks()).toBe(1);
expect(svc.getLink(orphan.id)).toMatchObject({ last_sync_state: 'orphaned', sync_enabled: 0 });
});
it('counts the trip owner as being on the trip, although no membership row says so', async () => {
// TREK's owner has no trip_members row; reading membership without the
// owner would orphan every binding the trip admin created, which is most
// of them.
const mine = await bind(OWNER, 'paperless');
expect(svc.markOrphanedLinks()).toBe(0);
expect(svc.getLink(mine.id)).toMatchObject({ last_sync_state: 'never', sync_enabled: 1 });
});
it('leaves a still-valid binding alone while orphaning the one beside it', async () => {
const orphan = await bind(MEMBER, 'paperless');
const kept = await bind(OWNER, 'nextcloud');
testDb.prepare('DELETE FROM trip_members WHERE trip_id = ? AND user_id = ?').run(TRIP, MEMBER);
expect(svc.markOrphanedLinks()).toBe(1);
expect(svc.getLink(orphan.id)?.last_sync_state).toBe('orphaned');
expect(svc.getLink(kept.id)).toMatchObject({ last_sync_state: 'never', sync_enabled: 1 });
});
it('counts nothing on a second sweep, so the job stays quiet after the first one', async () => {
await bind(MEMBER, 'paperless');
testDb.prepare('DELETE FROM trip_members WHERE trip_id = ? AND user_id = ?').run(TRIP, MEMBER);
expect(svc.markOrphanedLinks()).toBe(1);
expect(svc.markOrphanedLinks()).toBe(0);
});
it('does not touch a binding the user had already switched off', async () => {
const paused = await bind(MEMBER, 'paperless');
svc.updateLink(paused.id, { syncEnabled: false });
testDb.prepare('DELETE FROM trip_members WHERE trip_id = ? AND user_id = ?').run(TRIP, MEMBER);
expect(svc.markOrphanedLinks()).toBe(0);
expect(svc.getLink(paused.id)?.last_sync_state).toBe('never');
});
});
@@ -0,0 +1,412 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import crypto from 'crypto';
import type { Request } from 'express';
// Nothing here touches SQLite; the stub keeps the import chain from opening a
// database and running 238 migrations for a handler that never queries one.
vi.mock('../../../../src/db/database', () => ({
db: {},
closeDb: () => {},
reinitialize: () => {},
getPlaceWithTags: () => null,
canAccessTrip: () => undefined,
isOwner: () => false,
}));
import { DocSyncWebhookController } from '../../../../src/nest/doc-sync/doc-sync-webhook.controller';
import type { DocSyncConfigService, LinkRow } from '../../../../src/nest/doc-sync/doc-sync-config.service';
import type { DocSyncService } from '../../../../src/nest/doc-sync/doc-sync.service';
import type { DatabaseService } from '../../../../src/nest/database/database.service';
/**
* The webhook endpoint, with both services stubbed.
*
* It is the only route in TREK a stranger on the internet can reach with a
* guessed URL, so what matters is what it does NOT do: it never says whether a
* token exists, never says whether a secret matched, and never acts on the
* payload. Every case below therefore asserts two things: the answer, which is
* always the same, and whether a run was scheduled, which is the only place the
* decision is visible at all.
*
* The signature is computed here from a fixed id and timestamp, exactly as the
* provider would. Timers are faked, because the endpoint now collects a burst
* before it runs: providers fire once per document, and twenty files dropped
* into a watched folder used to be twenty runs, nineteen of which the service
* answered `busy` and threw away.
*/
const SECRET = 'M7dQ2vLp5rTn8kYw1xZc4bJh';
const link = (over: Partial<LinkRow> = {}): LinkRow => ({
id: 4,
trip_id: 1,
connection_id: 2,
provider_id: 'papra',
remote_scope_key: 'tag:1',
remote_root_id: '1',
remote_root_path: '/TREK/japan',
remote_label: 'Japan 2026',
direction: 'both',
delete_policy: 'unlink',
conflict_policy: 'manual',
sync_enabled: 1,
webhook_token: 'tok-live',
webhook_secret: 'enc:v1:whatever',
webhook_subscription_id: null,
remote_cursor: null,
last_sync_at: null,
last_sync_state: 'never',
last_sync_error: null,
failure_count: 0,
next_attempt_at: null,
...over,
});
const config = {
getLinkByToken: vi.fn((token: string) => (token === 'tok-live' ? link() : undefined)),
// Looked up again when the timer fires, so a binding switched off during the
// window does not get one last run out of a stale row.
getLink: vi.fn((id: number) => (id === 4 ? link() : undefined)),
webhookSecret: vi.fn(() => SECRET),
};
const settings = new Map<string, string>();
const db = {
get: vi.fn((_sql: string, key?: unknown) => {
const value = settings.get(String(key));
return value === undefined ? undefined : { value };
}),
};
const sync = {
syncLink: vi.fn(async (_link: LinkRow) => ({ state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 })),
// The Documents addon and the binding's provider, as one answer. What goes
// into it is the service's business and tested there.
isSwitchedOff: vi.fn((_link: LinkRow) => false),
};
const controller = new DocSyncWebhookController(
config as unknown as DocSyncConfigService,
sync as unknown as DocSyncService,
db as unknown as DatabaseService,
);
/** Only what the handler reads: headers, the parsed body, and the raw bytes. */
function makeReq(headers: Record<string, string> = {}, opts: { body?: unknown; rawBody?: Buffer | undefined } = {}): Request {
return {
headers,
body: opts.body,
rawBody: opts.rawBody,
get: (name: string) => headers[name.toLowerCase()],
} as unknown as Request;
}
const WEBHOOK_ID = 'msg_2f8a';
const WEBHOOK_TS = '1758200000';
function sign(payload: string, secret = SECRET, id = WEBHOOK_ID, ts = WEBHOOK_TS): string {
return crypto.createHmac('sha256', Buffer.from(secret)).update(`${id}.${ts}.${payload}`).digest('base64');
}
function papraReq(payload: string, signature: string, sent = payload): Request {
return makeReq(
{ 'webhook-id': WEBHOOK_ID, 'webhook-timestamp': WEBHOOK_TS, 'webhook-signature': `v1,${signature}` },
{ rawBody: Buffer.from(sent, 'utf8') },
);
}
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers();
settings.clear();
sync.isSwitchedOff.mockReturnValue(false);
config.getLinkByToken.mockImplementation((token: string) => (token === 'tok-live' ? link() : undefined));
config.getLink.mockImplementation((id: number) => (id === 4 ? link() : undefined));
config.webhookSecret.mockReturnValue(SECRET);
});
afterEach(() => {
vi.runOnlyPendingTimers();
vi.useRealTimers();
});
/** Let the debounce window pass, so a scheduled run actually happens. */
function settle() {
vi.advanceTimersByTime(6000);
}
describe('an unknown token', () => {
it('answers as if it were known, so nobody can enumerate which tokens exist', () => {
expect(controller.nudge('tok-guessed', makeReq())).toEqual({ received: true });
});
it('schedules no run', () => {
controller.nudge('tok-guessed', makeReq());
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
describe('a binding whose sync is switched off', () => {
it('is left alone, and says nothing about it', () => {
config.getLinkByToken.mockReturnValue(link({ sync_enabled: 0 }));
expect(controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }))).toEqual({ received: true });
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
describe('a shared-secret header', () => {
it('triggers the run when it matches', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
settle();
expect(sync.syncLink.mock.calls[0][0]).toMatchObject({ id: 4 });
});
it('triggers nothing when it is wrong, and the answer looks identical', () => {
const wrong = `${SECRET.slice(0, -1)}X`;
expect(controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': wrong }))).toEqual({ received: true });
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('triggers nothing when the header is missing altogether', () => {
controller.nudge('tok-live', makeReq());
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('is not demanded from a binding that carries no secret: the token alone authenticates there', () => {
config.webhookSecret.mockReturnValue('');
controller.nudge('tok-live', makeReq());
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
});
describe('a Papra standard-webhooks signature', () => {
const payload = JSON.stringify({ event: 'document.created', documentId: 'doc_1' });
it('triggers the run when it covers the bytes that arrived', () => {
controller.nudge('tok-live', papraReq(payload, sign(payload)));
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('triggers nothing when the payload was altered after it was signed', () => {
const tampered = JSON.stringify({ event: 'document.created', documentId: 'doc_999' });
expect(controller.nudge('tok-live', papraReq(payload, sign(payload), tampered))).toEqual({ received: true });
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('triggers nothing when the signature was made with another secret', () => {
controller.nudge('tok-live', papraReq(payload, sign(payload, 'someone-elses-secret-value')));
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('triggers nothing when the id or timestamp does not match what was signed', () => {
const req = makeReq(
{ 'webhook-id': 'msg_other', 'webhook-timestamp': WEBHOOK_TS, 'webhook-signature': `v1,${sign(payload)}` },
{ rawBody: Buffer.from(payload, 'utf8') },
);
controller.nudge('tok-live', req);
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('accepts a header carrying several signatures, as a key rotation sends', () => {
const req = makeReq(
{
'webhook-id': WEBHOOK_ID,
'webhook-timestamp': WEBHOOK_TS,
'webhook-signature': `v1,${sign(payload, 'the-previous-secret')} v1,${sign(payload)}`,
},
{ rawBody: Buffer.from(payload, 'utf8') },
);
controller.nudge('tok-live', req);
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('verifies against the parsed body when a mount left no raw bytes behind', () => {
const body = { event: 'document.created', documentId: 'doc_1' };
const req = makeReq(
{ 'webhook-id': WEBHOOK_ID, 'webhook-timestamp': WEBHOOK_TS, 'webhook-signature': `v1,${sign(JSON.stringify(body))}` },
{ body },
);
controller.nudge('tok-live', req);
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('is ignored when the timestamp header is missing, rather than verified without it', () => {
const req = makeReq(
{ 'webhook-id': WEBHOOK_ID, 'webhook-signature': `v1,${sign(payload)}` },
{ rawBody: Buffer.from(payload, 'utf8') },
);
controller.nudge('tok-live', req);
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
describe('a credential of the wrong length', () => {
// crypto.timingSafeEqual throws on buffers of unequal length, so a one-byte
// header would turn into a 500 that tells a prober the secret is longer than
// what they sent.
it('is rejected rather than thrown over, whether it arrives as a header or as a signature', () => {
expect(() => controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': 'a' }))).not.toThrow();
expect(() =>
controller.nudge('tok-live', papraReq('{}', 'short')),
).not.toThrow();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('is rejected the same way when it is longer than the stored secret', () => {
expect(() => controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': `${SECRET}extra` }))).not.toThrow();
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
/**
* A burst of calls is one run.
*
* Every provider here fires per document: a folder of twenty files is twenty
* calls within a second or two. Each used to start its own run. The service's
* in-flight guard then answered `busy` to nineteen of them, so nineteen
* announcements were thrown away and the one run that did start had begun
* before most of the changes landed.
*/
describe('a burst of nudges', () => {
it('runs once for twenty calls rather than twenty times', () => {
for (let i = 0; i < 20; i += 1) {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
}
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('answers every one of them immediately, before any run happens', () => {
const answers = [];
for (let i = 0; i < 5; i += 1) {
answers.push(controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET })));
}
// Paperless gives the call five seconds before it counts it as failed.
expect(answers).toEqual(Array.from({ length: 5 }, () => ({ received: true })));
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('runs again for a burst that arrives after the window closed', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('keeps one window per binding, so a busy trip cannot starve a quiet one', () => {
const other = link({ id: 9, webhook_token: 'tok-other' });
config.getLinkByToken.mockImplementation((t: string) =>
t === 'tok-live' ? link() : t === 'tok-other' ? other : undefined);
config.getLink.mockImplementation((id: number) => (id === 4 ? link() : id === 9 ? other : undefined));
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
controller.nudge('tok-other', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
expect(sync.syncLink.mock.calls.map((c) => c[0].id).sort()).toEqual([4, 9]);
});
it('drops the run when the binding is switched off inside the window', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
config.getLink.mockReturnValue(link({ sync_enabled: 0 }));
settle();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('drops the run when the binding is deleted inside the window', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
config.getLink.mockReturnValue(undefined);
settle();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('forgets its pending timers when the module goes down', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
controller.onModuleDestroy();
settle();
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
/**
* The switches an admin expects to mean "off".
*
* The scheduler obeys the Documents addon, the binding's provider and the
* app_settings kill switch; the webhook obeyed none of them, so switching
* document sync off stopped the poll while every provider holding a webhook
* carried on driving full runs.
*/
describe('the admin switches', () => {
it('does nothing while the addon or the provider of the binding is switched off', () => {
sync.isSwitchedOff.mockReturnValue(true);
expect(controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }))).toEqual({ received: true });
settle();
expect(sync.isSwitchedOff.mock.calls[0][0]).toMatchObject({ id: 4, provider_id: 'papra' });
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('does nothing while the kill switch is set', () => {
settings.set('docsync_sync_enabled', 'false');
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('drops a scheduled run when the provider goes off inside the window', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
sync.isSwitchedOff.mockReturnValue(true);
settle();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('runs normally while both are on', () => {
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
});
/**
* A nudge that lands on a run already in flight.
*
* `syncLink` answers `busy` and returns, so the nudge was thrown away, and the
* changes it was about may well have landed after the running pass read the
* folder, which means waiting out a whole poll interval for them. Asked again
* once, and only once, so two clients cannot keep each other going.
*/
describe('a nudge that arrives mid-run', () => {
it('asks again once when the run was busy', async () => {
sync.syncLink.mockResolvedValueOnce({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
await Promise.resolve();
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('gives up after that one retry rather than chasing itself', async () => {
sync.syncLink.mockResolvedValue({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
for (let i = 0; i < 5; i += 1) { settle(); await Promise.resolve(); }
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('does not retry a run that worked', async () => {
// Set explicitly: clearAllMocks drops the calls but keeps the
// implementation the previous case installed.
sync.syncLink.mockResolvedValue({ state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
await Promise.resolve();
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
});
@@ -0,0 +1,836 @@
import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest';
import { HttpException } from '@nestjs/common';
import type { Request } from 'express';
/**
* DocSyncController built with `new`, against a real in-memory SQLite.
*
* The guards are e2e's job (tests/e2e/doc-sync.e2e.test.ts drives the real
* AddonGuard, JwtAuthGuard and TripAccessGuard). What is left here is what the
* handler itself decides once a request is through the door, and all of it is
* the kind that would be expensive to notice late: who counts as the owner, a
* form field that must not lose a secret it never saw, and a webhook whose
* callback URL is derived from the request rather than from configuration.
*
* The config service and the provider registry are the real ones: the merge
* this file pins runs through actual encryption and actual provider-field rows,
* and a stub of either would only be a restatement of the controller. The
* reconciler is a spy: a run is a side effect here, not the subject.
*/
const { testDb, dbMock } = vi.hoisted(() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const Database = require('better-sqlite3');
const db = new Database(':memory:');
db.exec('PRAGMA foreign_keys = ON');
db.exec('PRAGMA busy_timeout = 5000');
return {
testDb: db,
dbMock: {
db,
closeDb: () => {},
reinitialize: () => {},
getPlaceWithTags: () => null,
canAccessTrip: () => undefined,
isOwner: () => false,
},
};
});
vi.mock('../../../../src/db/database', () => dbMock);
vi.mock('../../../../src/websocket', () => ({ broadcast: vi.fn(), broadcastToUser: vi.fn() }));
/**
* The SSRF guard resolves DNS, and none of these hostnames exist. It is not
* what this file tests, and a suite that needs a resolver is a suite that fails
* on a train.
*/
const { checkSsrf } = vi.hoisted(() => ({
checkSsrf: vi.fn(async (url: string) => ({
allowed: !url.includes('blocked.invalid'),
isPrivate: false,
error: url.includes('blocked.invalid') ? 'blocked in test' : undefined,
})),
}));
vi.mock('../../../../src/utils/ssrfGuard', async (importOriginal) => ({
...(await importOriginal<typeof import('../../../../src/utils/ssrfGuard')>()),
checkSsrf,
}));
import { createTables } from '../../../../src/db/schema';
import { runMigrations } from '../../../../src/db/migrations';
import { createTrip, createUser } from '../../../helpers/factories';
import { DatabaseService } from '../../../../src/nest/database/database.service';
import { DocSyncController } from '../../../../src/nest/doc-sync/doc-sync.controller';
import { DocSyncConfigService } from '../../../../src/nest/doc-sync/doc-sync-config.service';
import { DocumentProviderRegistry } from '../../../../src/nest/doc-sync/document-provider.registry';
import type {
DocumentConnectionRef,
DocumentProvider,
DocumentScopeRef,
} from '../../../../src/nest/doc-sync/document-provider';
import type { LinkRow } from '../../../../src/nest/doc-sync/doc-sync-config.service';
import type { DocSyncService } from '../../../../src/nest/doc-sync/doc-sync.service';
import type { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
import type {
DocsyncConnectionDto,
DocsyncConnectionTestDto,
DocsyncLinkDto,
} from '../../../../src/nest/doc-sync/doc-sync.dto';
import type { User } from '../../../../src/types';
const CAPS = {
push: 'webhook-self-registered' as const,
stableId: true,
remoteTrash: true,
replaceInPlace: true,
contentHashInListing: true,
maxUploadBytes: null,
acceptedMimeTypes: null,
canCreateScope: true,
};
function fakeProvider(id: string) {
return {
id,
capabilities: () => CAPS,
probe: vi.fn(async (_conn: DocumentConnectionRef) => ({ success: true as const, data: { account: `${id}-account`, capabilities: CAPS } })),
listScopes: vi.fn(async (_conn: DocumentConnectionRef, _query?: string) => ({ success: true as const, data: [] })),
createScope: vi.fn(async () => ({ success: false as const, error: { code: 'provider_error' as const } })),
resolveScope: vi.fn(async () => ({ success: false as const, error: { code: 'not_found' as const } })),
list: vi.fn(async () => ({ success: true as const, data: { documents: [], cursor: null, cursorUnchanged: false, truncated: false } })),
fetch: vi.fn(async () => ({ success: false as const, error: { code: 'not_found' as const } })),
push: vi.fn(async () => ({ success: false as const, error: { code: 'not_found' as const } })),
rename: vi.fn(async () => ({ success: true as const, data: { remoteVersion: 'v2' } })),
trash: vi.fn(async () => ({ success: true as const, data: undefined })),
registerWebhook: vi.fn(async (_conn: DocumentConnectionRef, _scope: DocumentScopeRef, _callbackUrl: string, _secret: string) => ({
success: true as const,
data: { subscriptionId: 'sub-7' },
})),
unregisterWebhook: vi.fn(async (_conn: DocumentConnectionRef, _subscriptionId: string) => ({ success: true as const, data: undefined })),
};
}
const paperless = fakeProvider('paperless');
const nextcloud = fakeProvider('nextcloud');
const sync = {
syncLink: vi.fn(async (_link: LinkRow, _opts?: { full?: boolean }) => ({ state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 })),
retryShelvedItems: vi.fn((_linkId: number) => {}),
isSwitchedOff: vi.fn((_link: LinkRow) => false),
status: vi.fn(() => ({ links: [], items: {} })),
resolveConflict: vi.fn(async () => true),
};
const registry = new DocumentProviderRegistry([paperless, nextcloud] as unknown as DocumentProvider[]);
const dbs = new DatabaseService(testDb);
const config = new DocSyncConfigService(dbs, registry);
const realtime = { broadcast: vi.fn() };
const controller = new DocSyncController(config, sync as unknown as DocSyncService, registry, dbs, realtime as unknown as RealtimeService);
/** Only what publicOrigin reads: header lookup plus the connection's own scheme. */
function makeReq(headers: Record<string, string> = {}, protocol = 'http'): Request {
return {
protocol,
headers,
get: (name: string) => headers[name.toLowerCase()],
} as unknown as Request;
}
function connBody(over: Partial<DocsyncConnectionDto> = {}): DocsyncConnectionDto {
return {
providerId: 'paperless',
baseUrl: 'https://paperless.example.com',
credentials: {},
allowInsecureTls: false,
...over,
} as DocsyncConnectionDto;
}
function linkBody(connectionId: number, over: Partial<DocsyncLinkDto> = {}): DocsyncLinkDto {
return {
connectionId,
scopeKey: 'tag:1',
remoteRootId: '1',
remoteRootPath: '/TREK/japan',
remoteLabel: 'Japan 2026',
direction: 'both',
deletePolicy: 'unlink',
conflictPolicy: 'manual',
syncEnabled: true,
...over,
} as DocsyncLinkDto;
}
async function thrown(fn: () => unknown): Promise<HttpException> {
try {
await fn();
} catch (err) {
expect(err).toBeInstanceOf(HttpException);
return err as HttpException;
}
throw new Error('expected the handler to throw');
}
let owner: User;
let member: User;
let admin: User;
let tripId: number;
let otherTripId: number;
beforeAll(() => {
createTables(testDb);
runMigrations(testDb);
const o = createUser(testDb, { username: 'owner', email: 'owner@test.local' }).user;
const m = createUser(testDb, { username: 'member', email: 'member@test.local' }).user;
const a = createUser(testDb, { username: 'admin', email: 'admin@test.local', role: 'admin' }).user;
owner = { id: o.id, role: 'user' } as User;
member = { id: m.id, role: 'user' } as User;
admin = { id: a.id, role: 'admin' } as User;
tripId = createTrip(testDb, o.id, { title: 'Japan' }).id;
otherTripId = createTrip(testDb, m.id, { title: 'Norway' }).id;
testDb.prepare('INSERT INTO trip_members (trip_id, user_id) VALUES (?, ?)').run(tripId, m.id);
});
afterAll(() => {
testDb.close();
});
beforeEach(() => {
testDb.exec('DELETE FROM document_sync_items; DELETE FROM trip_document_links; DELETE FROM document_connections; DELETE FROM trip_files');
// Providers ship switched off; these three are what the cases below need.
testDb.prepare("UPDATE document_providers SET enabled = 0").run();
testDb.prepare("UPDATE document_providers SET enabled = 1 WHERE id IN ('paperless', 'nextcloud', 'papra')").run();
vi.clearAllMocks();
});
async function storedPaperless(credentials: Record<string, string> = { api_token: 'stored-token' }) {
const res = await config.upsertConnection(tripId, Number(owner.id), connBody({ credentials }));
if ('error' in res) throw new Error(`fixture failed: ${res.error.code} ${res.error.detail ?? ''}`);
return res.data;
}
describe('who may change a trip binding', () => {
it('lets the trip owner through', async () => {
await expect(controller.resolve(String(tripId), '1', owner, { keep: 'trek' })).resolves.toEqual({ success: true });
});
it('lets an instance admin through on a trip they neither own nor joined', async () => {
await expect(controller.resolve(String(otherTripId), '1', admin, { keep: 'trek' })).resolves.toEqual({ success: true });
});
it('refuses a plain member with the message the client renders', async () => {
const err = await thrown(() => controller.resolve(String(tripId), '1', member, { keep: 'trek' }));
expect(err.getStatus()).toBe(403);
expect(err.message).toBe('Only the trip owner can change document sync');
});
it('answers 404 for a trip that does not exist, rather than the 403 that would confirm it', async () => {
const err = await thrown(() => controller.resolve('999999', '1', member, { keep: 'trek' }));
expect(err.getStatus()).toBe(404);
expect(err.message).toBe('Trip not found');
});
});
describe('providers', () => {
it('reports an enabled provider with no registered adapter as unavailable instead of hiding it', () => {
const rows = controller.providers() as Array<{ id: string; available: boolean }>;
const byId = Object.fromEntries(rows.map((r) => [r.id, r.available]));
// papra is switched on in the DB, but no adapter is registered in this suite.
expect(byId.papra).toBe(false);
expect(byId.paperless).toBe(true);
expect(byId.nextcloud).toBe(true);
});
it('hands the form flags to the client as booleans rather than the 0/1 the column stores', () => {
const rows = controller.providers() as Array<{ id: string; fields: Array<{ field_key: string; secret: boolean; required: boolean }> }>;
const fields = rows.find((r) => r.id === 'paperless')!.fields;
const token = fields.find((f) => f.field_key === 'api_token')!;
expect(token.secret).toBe(true);
expect(token.required).toBe(true);
expect(fields.find((f) => f.field_key === 'allow_insecure_tls')!.secret).toBe(false);
});
});
describe('storing a connection', () => {
it('hands the stored connection back with the secret masked instead of echoed', async () => {
const res = (await controller.upsertConnection(
String(tripId),
owner,
connBody({ credentials: { api_token: 'super-secret' } }),
)) as { id: number; secrets: Record<string, string> };
expect(res.secrets.api_token).not.toContain('super-secret');
const listed = controller.listConnections(String(tripId)) as Array<{ id: number; secrets: Record<string, string> }>;
expect(listed).toHaveLength(1);
expect(listed[0].id).toBe(res.id);
// The list is what the form renders from, so the secret must not survive
// the round trip either.
expect(listed[0].secrets.api_token).not.toContain('super-secret');
});
it('refuses a base URL the SSRF guard turns down', async () => {
const err = await thrown(() => controller.upsertConnection(String(tripId), owner, connBody({ baseUrl: 'https://blocked.invalid' })));
expect(err.getStatus()).toBe(400);
expect(config.listConnections(tripId)).toHaveLength(0);
});
it('refuses a provider the instance admin has not switched on, and names it', async () => {
testDb.prepare("UPDATE document_providers SET enabled = 0 WHERE id = 'papra'").run();
const err = await thrown(() =>
controller.upsertConnection(String(tripId), owner, connBody({ providerId: 'papra', credentials: { api_key: 'k', organization_id: 'org_1' } })),
);
expect(err.getStatus()).toBe(400);
expect(err.message).toBe('Provider: "papra" is not enabled, contact server administrator');
});
});
describe('probing form values', () => {
it('keeps the stored secret when the form leaves the field blank', async () => {
await storedPaperless();
await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: '' } }) as DocsyncConnectionTestDto);
expect(paperless.probe.mock.calls[0][0]).toMatchObject({ secrets: { api_token: 'stored-token' } });
});
it('takes a freshly typed secret over the stored one', async () => {
await storedPaperless();
await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: 'typed-now' } }) as DocsyncConnectionTestDto);
expect(paperless.probe.mock.calls[0][0]).toMatchObject({ secrets: { api_token: 'typed-now' } });
});
it('keeps a stored non-secret setting the form did not resend', async () => {
const res = await config.upsertConnection(tripId, Number(owner.id), connBody({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'alice', app_password: 'app-pw', base_path: '/TREK' },
}));
expect('error' in res).toBe(false);
await controller.testConnection(String(tripId), owner, connBody({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { app_password: '' },
}) as DocsyncConnectionTestDto);
expect(nextcloud.probe.mock.calls[0][0]).toMatchObject({
settings: { login_name: 'alice', base_path: '/TREK' },
secrets: { app_password: 'app-pw' },
});
});
it('takes a freshly typed setting over the stored one', async () => {
const res = await config.upsertConnection(tripId, Number(owner.id), connBody({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'alice', app_password: 'app-pw' },
}));
expect('error' in res).toBe(false);
await controller.testConnection(String(tripId), owner, connBody({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'bob' },
}) as DocsyncConnectionTestDto);
expect(nextcloud.probe.mock.calls[0][0]).toMatchObject({ settings: { login_name: 'bob' } });
});
it('answers with a body instead of throwing when no adapter is registered', async () => {
await expect(
controller.testConnection(String(tripId), owner, connBody({ providerId: 'papra', credentials: { api_key: 'k' } }) as DocsyncConnectionTestDto),
).resolves.toEqual({ connected: false, error: 'unknown_provider' });
});
it('answers with a body instead of throwing when the URL is refused', async () => {
await expect(
controller.testConnection(String(tripId), owner, connBody({ baseUrl: 'https://blocked.invalid' }) as DocsyncConnectionTestDto),
).resolves.toEqual({ connected: false, error: 'ssrf_blocked' });
expect(paperless.probe).not.toHaveBeenCalled();
});
it('answers with a body instead of throwing when the provider rejects the credentials', async () => {
paperless.probe.mockResolvedValueOnce({ success: false, error: { code: 'unauthorized', detail: 'token expired' } } as never);
await expect(
controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: 'nope' } }) as DocsyncConnectionTestDto),
).resolves.toEqual({ connected: false, error: 'unauthorized', detail: 'token expired' });
});
it('writes a successful probe onto the stored connection, so the status panel has something to show', async () => {
const conn = await storedPaperless();
await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: '' } }) as DocsyncConnectionTestDto);
const row = config.getConnection(conn.id)!;
expect(row.last_probe_state).toBe('ok');
expect(JSON.parse(row.capabilities!)).toMatchObject({ push: 'webhook-self-registered' });
});
it('leaves no probe record behind when the form values belong to no stored connection yet', async () => {
await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: 'first-try' } }) as DocsyncConnectionTestDto);
expect(config.listConnections(tripId)).toHaveLength(0);
expect(paperless.probe.mock.calls[0][0]).toMatchObject({ connectionId: 0, secrets: { api_token: 'first-try' } });
});
});
describe('a secret the adapter earned itself', () => {
const EARNED = 'a1b2c3:DEVICE-SECRET';
it('is in neither connection payload the form is rendered from', async () => {
testDb.prepare("UPDATE document_providers SET enabled = 1 WHERE id = 'synologydrive'").run();
const nas = connBody({
providerId: 'synologydrive',
baseUrl: 'https://nas.example.com:5001',
credentials: { username: 'anna', password: 'nas-pw' },
});
const created = (await controller.upsertConnection(String(tripId), owner, nas)) as { id: number };
config.saveEarnedSecret(created.id, 'device_token', EARNED);
const listed = JSON.stringify(controller.listConnections(String(tripId)));
const edited = JSON.stringify(
await controller.upsertConnection(String(tripId), owner, { ...nas, credentials: { username: 'anna', password: 'rotated' } }),
);
for (const payload of [listed, edited]) {
expect(payload).not.toContain('DEVICE-SECRET');
expect(payload).not.toContain('device_token');
}
expect(config.toRef(config.getConnection(created.id)!).secrets.device_token).toBe(EARNED);
});
it('reaches a probe of the saved form, which gets no way to write', async () => {
const conn = await storedPaperless();
config.saveEarnedSecret(conn.id, 'device_token', EARNED);
const res = await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: '' } }) as DocsyncConnectionTestDto);
const ref = paperless.probe.mock.calls[0][0];
// The row as well as its id: together they are where the saved connection keeps its device token.
expect(ref).toMatchObject({ connectionId: conn.id, createdAt: conn.created_at, secrets: { device_token: EARNED } });
expect(ref.createdAt).not.toBe('');
expect(ref.saveSecret).toBeUndefined();
expect(JSON.stringify(res)).not.toContain('DEVICE-SECRET');
});
it('gives a probe of an unsaved form no way to write either', async () => {
await controller.testConnection(String(tripId), owner, connBody({ credentials: { api_token: 'first-try' } }) as DocsyncConnectionTestDto);
const ref = paperless.probe.mock.calls[0][0];
expect(ref.connectionId).toBe(0);
expect(ref.createdAt).toBe('');
expect(ref.saveSecret).toBeUndefined();
});
});
describe('reading the trip state', () => {
it('asks the reconciler with a number, because the path hands the handler a string', () => {
controller.status(String(tripId));
expect(sync.status).toHaveBeenCalledWith(tripId);
});
});
describe('removing a connection', () => {
it('unbinds without touching the documents that came through it', async () => {
const conn = await storedPaperless();
const fileId = Number(
testDb.prepare('INSERT INTO trip_files (trip_id, filename, original_name) VALUES (?, ?, ?)')
.run(tripId, 'stored.pdf', 'boarding.pdf').lastInsertRowid,
);
expect(controller.deleteConnection(String(tripId), String(conn.id), owner)).toEqual({ success: true });
expect(config.getConnection(conn.id)).toBeUndefined();
expect(testDb.prepare('SELECT id FROM trip_files WHERE id = ?').get(fileId)).toBeTruthy();
});
it('refuses a connection that belongs to a different trip', async () => {
const conn = await storedPaperless();
const err = await thrown(() => controller.deleteConnection(String(otherTripId), String(conn.id), admin));
expect(err.getStatus()).toBe(404);
expect(config.getConnection(conn.id)).toBeTruthy();
expect(realtime.broadcast).not.toHaveBeenCalled();
});
it('pings the trip once for every binding that went with it, and for no other', async () => {
const conn = await storedPaperless();
const cloud = await config.upsertConnection(tripId, Number(owner.id), connBody({
providerId: 'nextcloud',
baseUrl: 'https://cloud.example.com',
credentials: { login_name: 'alice', app_password: 'app-pw', base_path: '/TREK' },
}));
if ('error' in cloud) throw new Error(`fixture failed: ${cloud.error.code}`);
const bind = (connectionId: number, scopeKey: string) => {
const res = config.createLink(tripId, Number(owner.id), linkBody(connectionId, { scopeKey }));
if ('error' in res) throw new Error(`fixture failed: ${res.error.code}`);
return res.data.id;
};
const first = bind(conn.id, 'tag:1');
const second = bind(conn.id, 'tag:2');
bind(cloud.data.id, 'folder:1');
controller.deleteConnection(String(tripId), String(conn.id), owner);
expect(realtime.broadcast.mock.calls).toEqual([
[tripId, 'docsync:changed', { linkId: first, pulled: 0, pushed: 0 }],
[tripId, 'docsync:changed', { linkId: second, pulled: 0, pushed: 0 }],
]);
});
});
describe('the scope picker', () => {
it('passes the search text to the adapter and hands back what it offers', async () => {
const conn = await storedPaperless();
paperless.listScopes.mockResolvedValueOnce({
success: true,
data: [{ scopeKey: 'tag:1', label: 'Japan 2026', remoteRootId: '1', remoteRootPath: '/TREK/japan' }],
} as never);
const res = await controller.listScopes(String(tripId), String(conn.id), owner, 'japan');
expect(paperless.listScopes.mock.calls[0][1]).toBe('japan');
expect(res).toMatchObject({ scopes: [{ scopeKey: 'tag:1' }] });
});
it('reports a provider failure in the body, so the picker can say why it is empty', async () => {
const conn = await storedPaperless();
paperless.listScopes.mockResolvedValueOnce({ success: false, error: { code: 'unauthorized' } } as never);
await expect(controller.listScopes(String(tripId), String(conn.id), owner)).resolves.toEqual({ scopes: [], error: 'unauthorized' });
});
it('refuses a connection whose provider has no adapter on this instance', async () => {
const id = Number(
testDb.prepare('INSERT INTO document_connections (trip_id, provider_id, owner_user_id, base_url) VALUES (?, ?, ?, ?)')
.run(tripId, 'papra', Number(owner.id), 'https://papra.example.com').lastInsertRowid,
);
const err = await thrown(() => controller.listScopes(String(tripId), String(id), owner));
expect(err.getStatus()).toBe(400);
expect(err.message).toBe('Provider not available');
});
it('refuses a connection that belongs to a different trip', async () => {
const conn = await storedPaperless();
const err = await thrown(() => controller.listScopes(String(otherTripId), String(conn.id), admin));
expect(err.getStatus()).toBe(404);
expect(paperless.listScopes).not.toHaveBeenCalled();
});
it('hands back the container the adapter created', async () => {
const conn = await storedPaperless();
paperless.createScope.mockResolvedValueOnce({
success: true,
data: { scopeKey: 'tag:9', label: 'Norway', remoteRootId: '9', remoteRootPath: '/TREK/norway' },
} as never);
await expect(controller.createScope(String(tripId), String(conn.id), owner, { name: 'Norway' }))
.resolves.toMatchObject({ scopeKey: 'tag:9', label: 'Norway' });
expect(paperless.createScope).toHaveBeenCalledWith(expect.objectContaining({ connectionId: conn.id }), 'Norway');
});
it('creates nothing through a connection that belongs to a different trip', async () => {
// The connection comes from the path and nowhere else, so the path is what
// has to match the trip the caller was admitted to.
const conn = await storedPaperless();
const err = await thrown(() => controller.createScope(String(otherTripId), String(conn.id), admin, { name: 'Norway' }));
expect(err.getStatus()).toBe(404);
expect(paperless.createScope).not.toHaveBeenCalled();
});
it('turns a refused creation into a 400 carrying the provider code', async () => {
const conn = await storedPaperless();
paperless.createScope.mockResolvedValueOnce({ success: false, error: { code: 'forbidden' } } as never);
const err = await thrown(() => controller.createScope(String(tripId), String(conn.id), owner, { name: 'Norway' }));
expect(err.getStatus()).toBe(400);
expect(err.message).toBe('forbidden');
});
});
describe('creating a binding', () => {
it('subscribes at the provider and remembers the subscription id', async () => {
const conn = await storedPaperless();
const link = (await controller.createLink(
String(tripId),
owner,
linkBody(conn.id),
makeReq({ 'x-forwarded-proto': 'https', 'x-forwarded-host': 'trek.example' }),
)) as { id: number };
const row = config.getLink(link.id)!;
expect(row.webhook_subscription_id).toBe('sub-7');
expect(paperless.registerWebhook.mock.calls[0][2]).toBe(`https://trek.example/api/docsync/webhook/${row.webhook_token}`);
// The secret handed to the provider is the plaintext one, not the stored blob.
expect(paperless.registerWebhook.mock.calls[0][3]).toBe(config.webhookSecret(row));
});
it('keeps the binding when the provider refuses the subscription, because polling still carries it', async () => {
const conn = await storedPaperless();
paperless.registerWebhook.mockResolvedValueOnce({ success: false, error: { code: 'forbidden' } } as never);
const link = (await controller.createLink(
String(tripId),
owner,
linkBody(conn.id),
makeReq({ 'x-forwarded-host': 'trek.example' }),
)) as { id: number; syncEnabled: boolean };
const row = config.getLink(link.id)!;
expect(row.webhook_subscription_id).toBeNull();
expect(row.sync_enabled).toBe(1);
});
it('does not offer a webhook when no host reaches TREK', async () => {
const conn = await storedPaperless();
const link = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { webhookUrl: string | null };
expect(paperless.registerWebhook).not.toHaveBeenCalled();
expect(link.webhookUrl).toBeNull();
});
it('starts a first run at once instead of letting the user wait out a poll interval', async () => {
const conn = await storedPaperless();
await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq({ host: 'trek.example' }));
expect(sync.syncLink).toHaveBeenCalledTimes(1);
expect(sync.syncLink.mock.calls[0][1]).toEqual({ full: true });
});
it('pings the whole trip, so a member already on the Files tab gets the sync button', async () => {
const conn = await storedPaperless();
const link = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
// Three arguments: no socket is skipped, the owner's own button reads it too.
expect(realtime.broadcast).toHaveBeenCalledWith(tripId, 'docsync:changed', { linkId: link.id, pulled: 0, pushed: 0 });
});
it('refuses a binding for a connection that belongs to a different trip', async () => {
const conn = await storedPaperless();
const err = await thrown(() =>
controller.createLink(String(otherTripId), admin, linkBody(conn.id), makeReq({ host: 'trek.example' })),
);
expect(err.getStatus()).toBe(400);
expect(sync.syncLink).not.toHaveBeenCalled();
expect(realtime.broadcast).not.toHaveBeenCalled();
});
});
describe('changing a binding', () => {
it('writes only the fields the request carried', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
const res = controller.updateLink(String(tripId), String(created.id), owner, { syncEnabled: false }, makeReq()) as {
syncEnabled: boolean;
direction: string;
remoteLabel: string;
};
expect(res.syncEnabled).toBe(false);
expect(res.direction).toBe('both');
expect(res.remoteLabel).toBe('Japan 2026');
});
it('refuses a link that belongs to a different trip', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
const err = await thrown(() => controller.updateLink(String(otherTripId), String(created.id), admin, { syncEnabled: false }, makeReq()));
expect(err.getStatus()).toBe(404);
expect(config.getLink(created.id)!.sync_enabled).toBe(1);
});
});
describe('a manual run', () => {
it('passes the full flag through to the reconciler', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
sync.syncLink.mockClear();
await controller.syncNow(String(tripId), String(created.id), { full: true });
expect(sync.syncLink.mock.calls[0][0]).toMatchObject({ id: created.id });
expect(sync.syncLink.mock.calls[0][1]).toEqual({ full: true });
});
it('refuses a link that belongs to a different trip', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
sync.syncLink.mockClear();
const err = await thrown(() => controller.syncNow(String(otherTripId), String(created.id), { full: false }));
expect(err.getStatus()).toBe(404);
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('refuses a binding an admin switched off with a code, and touches nothing', async () => {
// A code rather than a sentence so the client can say it in the reader's
// language; the MCP tool answers with the same one. Refused before the
// shelved rows are cleared, so the binding resumes exactly as it was.
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
sync.syncLink.mockClear();
sync.isSwitchedOff.mockReturnValueOnce(true);
const err = await thrown(() => controller.syncNow(String(tripId), String(created.id), { full: false }));
expect(err.getStatus()).toBe(409);
expect(err.getResponse()).toEqual({ error: 'provider_disabled' });
expect(sync.isSwitchedOff.mock.calls[0][0]).toMatchObject({ id: created.id });
expect(sync.retryShelvedItems).not.toHaveBeenCalled();
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
describe('the document list', () => {
async function seedItems() {
const conn = await storedPaperless();
const fileId = Number(
testDb.prepare('INSERT INTO trip_files (trip_id, filename, original_name) VALUES (?, ?, ?)')
.run(tripId, 'stored.pdf', 'boarding.pdf').lastInsertRowid,
);
const linkId = Number(
testDb.prepare(
`INSERT INTO trip_document_links (trip_id, connection_id, provider_id, remote_scope_key)
VALUES (?, ?, ?, ?)`,
).run(tripId, conn.id, 'paperless', 'tag:1').lastInsertRowid,
);
const insert = testDb.prepare('INSERT INTO document_sync_items (link_id, trip_id, file_id, trek_doc_uid, state) VALUES (?, ?, ?, ?, ?)');
insert.run(linkId, tripId, fileId, 'uid-1', 'synced');
insert.run(linkId, tripId, null, 'uid-2', 'conflict');
// Another trip's row, which must never appear in this trip's list.
insert.run(linkId, otherTripId, null, 'uid-3', 'conflict');
}
it('carries the local file name, so the list reads as documents rather than as ids', async () => {
await seedItems();
const rows = controller.items(String(tripId)) as Array<{ trek_doc_uid: string; file_name: string | null }>;
expect(rows).toHaveLength(2);
expect(rows.find((r) => r.trek_doc_uid === 'uid-1')!.file_name).toBe('boarding.pdf');
});
it('narrows to one state when asked, and never leaves the trip', async () => {
await seedItems();
const rows = controller.items(String(tripId), 'conflict') as Array<{ trek_doc_uid: string }>;
expect(rows.map((r) => r.trek_doc_uid)).toEqual(['uid-2']);
});
});
describe('resolving a conflict', () => {
it('refuses an item that is not in conflict rather than pretending it did something', async () => {
sync.resolveConflict.mockResolvedValueOnce(false);
const err = await thrown(() => controller.resolve(String(tripId), '5', owner, { keep: 'provider' }));
expect(err.getStatus()).toBe(400);
expect(err.message).toBe('Item is not in conflict');
});
});
describe('removing a binding', () => {
it('unsubscribes at the provider and keeps both copies of the documents', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(
String(tripId),
owner,
linkBody(conn.id),
makeReq({ host: 'trek.example' }),
)) as { id: number };
const fileId = Number(
testDb.prepare('INSERT INTO trip_files (trip_id, filename, original_name) VALUES (?, ?, ?)')
.run(tripId, 'stored.pdf', 'boarding.pdf').lastInsertRowid,
);
testDb
.prepare('INSERT INTO document_sync_items (link_id, trip_id, file_id, trek_doc_uid, state) VALUES (?, ?, ?, ?, ?)')
.run(created.id, tripId, fileId, 'uid-1', 'synced');
const res = await controller.deleteLink(String(tripId), String(created.id), owner);
expect(res).toEqual({ success: true, documentsKept: true });
expect(paperless.unregisterWebhook.mock.calls[0][1]).toBe('sub-7');
expect(config.getLink(created.id)).toBeUndefined();
expect(testDb.prepare('SELECT id FROM trip_files WHERE id = ?').get(fileId)).toBeTruthy();
});
it('pings the trip that the binding is gone', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
realtime.broadcast.mockClear();
await controller.deleteLink(String(tripId), String(created.id), owner);
expect(realtime.broadcast.mock.calls).toEqual([
[tripId, 'docsync:changed', { linkId: created.id, pulled: 0, pushed: 0 }],
]);
});
it('refuses a link that belongs to a different trip', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
realtime.broadcast.mockClear();
const err = await thrown(() => controller.deleteLink(String(otherTripId), String(created.id), admin));
expect(err.getStatus()).toBe(404);
expect(config.getLink(created.id)).toBeTruthy();
expect(realtime.broadcast).not.toHaveBeenCalled();
});
});
describe('the callback origin a provider is given', () => {
async function linkedTrip() {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
return config.getLink(created.id)!;
}
it('builds the callback URL from the forwarded proto and host', async () => {
const row = await linkedTrip();
const [link] = controller.listLinks(String(tripId), makeReq({ 'x-forwarded-proto': 'https', 'x-forwarded-host': 'docs.example' })) as Array<{ webhookUrl: string }>;
expect(link.webhookUrl).toBe(`https://docs.example/api/docsync/webhook/${row.webhook_token}`);
});
it('takes the first hop when a proxy chain appends its own protocol', async () => {
await linkedTrip();
const [link] = controller.listLinks(String(tripId), makeReq({ 'x-forwarded-proto': 'https, http', 'x-forwarded-host': 'docs.example' })) as Array<{ webhookUrl: string }>;
expect(link.webhookUrl).toMatch(/^https:\/\/docs\.example\//);
});
it('falls back to the Host header and the connection scheme when nothing is forwarded', async () => {
await linkedTrip();
const [link] = controller.listLinks(String(tripId), makeReq({ host: 'trek.lan:3001' }, 'http')) as Array<{ webhookUrl: string }>;
expect(link.webhookUrl).toMatch(/^http:\/\/trek\.lan:3001\//);
});
it('offers no callback URL when a proxy strips the host', async () => {
await linkedTrip();
const [link] = controller.listLinks(String(tripId), makeReq()) as Array<{ webhookUrl: string | null }>;
expect(link.webhookUrl).toBeNull();
});
});
/**
* A manual run is also a request to try the given-up rows once more.
*
* The scheduler must never do this: the attempt limit exists precisely so a
* document a provider refuses is not re-uploaded on every tick. A person
* pressing the button is the signal that something changed.
*/
describe('a manual run and the shelved rows', () => {
it('clears the attempt counters of the link it runs', async () => {
const conn = await storedPaperless();
const created = (await controller.createLink(String(tripId), owner, linkBody(conn.id), makeReq())) as { id: number };
sync.retryShelvedItems.mockClear();
await controller.syncNow(String(tripId), String(created.id), { full: false });
expect(sync.retryShelvedItems).toHaveBeenCalledWith(created.id);
});
it('does not clear them for a link the caller does not own', async () => {
sync.retryShelvedItems.mockClear();
await expect(controller.syncNow(String(tripId), '999999', { full: false })).rejects.toThrow();
expect(sync.retryShelvedItems).not.toHaveBeenCalled();
});
});
/**
* The trip the route was authorised against is now passed on to the service.
*
* `assertCanManage` proves the caller may manage the trip in the URL and says
* nothing about whether the item id in the path belongs to that trip: the id is
* a plain integer, and nothing tied the two together. The check itself lives in
* the service (where the row is), so what belongs here is that the trip reaches
* it at all.
*/
describe('resolving a conflict carries its trip', () => {
it('passes the trip from the URL to the service', async () => {
sync.resolveConflict.mockClear();
await controller.resolve(String(tripId), '42', owner, { keep: 'trek' } as never);
expect(sync.resolveConflict).toHaveBeenCalledWith(42, 'trek', tripId);
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,492 @@
/**
* The scheduling shell around document sync, built with `new` and plain mocks:
* no container, no timer, no provider.
*
* What the job itself decides is small, and all of it is the kind that breaks
* silently. The addon gate and the kill switch are re-read per tick, which is
* the promise the docblock makes an admin ("toggle it and it takes effect")
* and the only thing that makes the difference between this job and the
* Dawarich one visible. The interval is typed by a human into app_settings, so
* '5' and '999999' both have to land somewhere the cron parser accepts. And one
* unreachable NAS must not take the other trips' bindings down with it.
*
* The last block is the exception: an admin's provider switch lives in the
* service's query, so it runs the tick over the real service and a database.
*/
import { describe, it, expect, vi, beforeAll, beforeEach, afterAll, afterEach } from 'vitest';
import Database from 'better-sqlite3';
const log = vi.hoisted(() => ({
LOG_LEVEL: 'error',
logInfo: vi.fn(),
logError: vi.fn(),
logWarn: vi.fn(),
logDebug: vi.fn(),
}));
vi.mock('../../../../src/nest/audit/audit-log.logger', () => log);
import { ADDON_IDS } from '../../../../src/addons';
import { DocSyncJob } from '../../../../src/nest/doc-sync/doc-sync.job';
import { SETTING_POLL_INTERVAL, SETTING_SYNC_ENABLED } from '../../../../src/nest/doc-sync/doc-sync.constants';
import { DocSyncConfigService, type LinkRow } from '../../../../src/nest/doc-sync/doc-sync-config.service';
import { DocSyncService } from '../../../../src/nest/doc-sync/doc-sync.service';
import { DocumentProviderRegistry } from '../../../../src/nest/doc-sync/document-provider.registry';
import type { DocumentProvider } from '../../../../src/nest/doc-sync/document-provider';
import type { AddonsService } from '../../../../src/nest/addons/addons.service';
import { DatabaseService } from '../../../../src/nest/database/database.service';
import type { CronRegistrarService } from '../../../../src/nest/scheduling/cron-registrar.service';
import { AllowedFileTypesService } from '../../../../src/nest/files/allowed-file-types.service';
import type { FilesService } from '../../../../src/nest/files/files.service';
import type { StorageService } from '../../../../src/nest/storage/storage.service';
import type { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
import { createTables } from '../../../../src/db/schema';
import { runMigrations } from '../../../../src/db/migrations';
import { createTrip, createUser } from '../../../helpers/factories';
const link = (id: number): LinkRow => ({ id, provider_id: 'paperless' } as LinkRow);
const RUN = { state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 };
interface Setup {
/** Absent means no app_settings row at all, which is the shipped state. */
interval?: string;
killSwitch?: string;
addonOn: boolean;
registrarEnabled: boolean;
links: LinkRow[];
orphaned: number;
}
function makeJob(over: Partial<Setup> = {}) {
const setup: Setup = { addonOn: true, registrarEnabled: true, links: [], orphaned: 0, ...over };
const settings = new Map<string, string>();
if (setup.interval !== undefined) settings.set(SETTING_POLL_INTERVAL, setup.interval);
if (setup.killSwitch !== undefined) settings.set(SETTING_SYNC_ENABLED, setup.killSwitch);
// Shaped like the real `get<T>(sql, ...params)` so the stub cannot drift from
// the signature the job calls, and so a case can tell the two keys apart.
const db = {
get: vi.fn((_sql: string, key?: unknown) => {
const value = settings.get(String(key));
return value === undefined ? undefined : { value };
}),
};
let onTick: (() => void | Promise<void>) | undefined;
const registrar = {
isEnabled: vi.fn(() => setup.registrarEnabled),
register: vi.fn((_name: string, _expression: string, cb: () => void | Promise<void>) => {
onTick = cb;
return setup.registrarEnabled;
}),
unregister: vi.fn(),
};
// Both stubs carry the real signatures, so a case can read back which link a
// run was asked for instead of only that some run happened.
const sync = {
dueLinks: vi.fn((_limit?: number) => setup.links),
syncLink: vi.fn(async (_link: LinkRow, _opts?: { full?: boolean }) => RUN),
};
const config = { markOrphanedLinks: vi.fn(() => setup.orphaned) };
const addons = { isAddonEnabled: vi.fn(() => setup.addonOn) };
const job = new DocSyncJob(
db as unknown as DatabaseService,
sync as unknown as DocSyncService,
config as unknown as DocSyncConfigService,
addons as unknown as AddonsService,
registrar as unknown as CronRegistrarService,
);
return { job, db, registrar, sync, config, addons, takeTick: () => onTick };
}
beforeEach(() => vi.clearAllMocks());
describe('DocSyncJob bootstrap', () => {
it('schedules nothing, reads nothing and logs nothing while the registrar is off', () => {
// The test gate. A job that registered past it would have every suite boot
// start polling whatever document store sits in the fixture database.
const { job, registrar, db } = makeJob({ registrarEnabled: false });
job.onApplicationBootstrap();
expect(registrar.register).not.toHaveBeenCalled();
expect(db.get).not.toHaveBeenCalled();
expect(log.logInfo).not.toHaveBeenCalled();
});
it('registers one cron under a name of its own, on the minute', () => {
// Every minute, with the tick deciding whether it is due. Baking the
// interval into the expression at bootstrap meant a changed setting did
// nothing until a restart, which is the opposite of what this job's own
// comment promises, and nothing re-registers it (auto-backup has a start()
// its settings save calls; this has no such path).
const { job, registrar } = makeJob();
job.onApplicationBootstrap();
expect(registrar.register).toHaveBeenCalledWith('docsync', '* * * * *', expect.any(Function));
expect(log.logInfo).toHaveBeenCalledWith('Document sync: polling every 300s');
});
it('reads the interval from its own app_settings key', () => {
const { job, db } = makeJob({ interval: '600' });
job.onApplicationBootstrap();
expect(db.get).toHaveBeenCalledWith('SELECT value FROM app_settings WHERE key = ?', SETTING_POLL_INTERVAL);
});
it('hands the registrar the tick itself, so a fired cron reaches the sync', async () => {
const { job, sync, takeTick } = makeJob({ links: [link(1)] });
job.onApplicationBootstrap();
const tick = takeTick();
expect(tick).toBeTypeOf('function');
await tick?.();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('does not decide at bootstrap whether the addon is on', () => {
// Asking here would freeze the answer for the life of the process, and the
// bug would read as "the documents toggle needs a restart".
const { job, addons } = makeJob();
job.onApplicationBootstrap();
expect(addons.isAddonEnabled).not.toHaveBeenCalled();
});
});
describe('DocSyncJob tick', () => {
it('does nothing at all while the documents addon is off', async () => {
const { job, sync, config, db } = makeJob({ addonOn: false, links: [link(1)] });
await job.tick();
expect(sync.dueLinks).not.toHaveBeenCalled();
expect(sync.syncLink).not.toHaveBeenCalled();
expect(config.markOrphanedLinks).not.toHaveBeenCalled();
expect(db.get).not.toHaveBeenCalled();
});
it('asks the addon gate again on every tick, so switching it on needs no restart', async () => {
const { job, addons, sync } = makeJob({ addonOn: false, links: [link(1)] });
await job.tick();
expect(sync.syncLink).not.toHaveBeenCalled();
addons.isAddonEnabled.mockReturnValue(true);
await job.tick();
expect(addons.isAddonEnabled).toHaveBeenCalledTimes(2);
expect(addons.isAddonEnabled).toHaveBeenLastCalledWith(ADDON_IDS.DOCUMENTS);
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('stops the run on the kill switch, without touching the bindings', async () => {
const { job, sync, config, db } = makeJob({ killSwitch: 'false', links: [link(1)] });
await job.tick();
expect(db.get).toHaveBeenCalledWith('SELECT value FROM app_settings WHERE key = ?', SETTING_SYNC_ENABLED);
expect(config.markOrphanedLinks).not.toHaveBeenCalled();
expect(sync.dueLinks).not.toHaveBeenCalled();
});
it('keeps syncing on anything that is not exactly the string false', async () => {
// The setting is absent by default, so an unrecognised value has to mean ON.
// Treating '0' or 'off' as a stop would silently disable sync for anyone who
// typed the switch by hand into app_settings.
for (const value of [undefined, '', '0', 'off', 'no', 'FALSE', 'true']) {
vi.clearAllMocks();
const { job, sync } = makeJob({ killSwitch: value, links: [link(1)] });
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
}
});
it('re-reads the kill switch per tick instead of remembering the first answer', async () => {
const { job, db } = makeJob({ links: [link(1)] });
await job.tick();
await job.tick();
const killSwitchReads = db.get.mock.calls.filter((c) => c[1] === SETTING_SYNC_ENABLED).length;
expect(killSwitchReads).toBe(2);
});
it('sweeps bindings whose credential owner left the trip before it syncs anything', async () => {
const { job, config, sync } = makeJob({ orphaned: 2, links: [link(1)] });
await job.tick();
expect(config.markOrphanedLinks).toHaveBeenCalledTimes(1);
expect(config.markOrphanedLinks.mock.invocationCallOrder[0]).toBeLessThan(
sync.dueLinks.mock.invocationCallOrder[0],
);
expect(log.logInfo).toHaveBeenCalledWith('Document sync: 2 link(s) orphaned, owner no longer on the trip');
});
it('stays quiet when the sweep found nothing', async () => {
const { job, config } = makeJob({ orphaned: 0 });
await job.tick();
expect(config.markOrphanedLinks).toHaveBeenCalledTimes(1);
expect(log.logInfo).not.toHaveBeenCalled();
});
it('runs the remaining bindings after one of them throws', async () => {
// An unreachable NAS on one trip is not a reason to skip a Paperless
// binding on another, and the two are ordinary neighbours in one list.
const { job, sync } = makeJob({ links: [link(1), link(2), link(3)] });
sync.syncLink.mockRejectedValueOnce(new Error('EHOSTUNREACH'));
await expect(job.tick()).resolves.toBeUndefined();
expect(sync.syncLink).toHaveBeenCalledTimes(3);
expect(log.logError).toHaveBeenCalledWith('Document sync: link 1 failed: EHOSTUNREACH');
});
it('logs a non-Error rejection by value rather than as an empty message', async () => {
const { job, sync } = makeJob({ links: [link(7)] });
sync.syncLink.mockRejectedValueOnce('ECONNREFUSED');
await job.tick();
expect(log.logError).toHaveBeenCalledWith('Document sync: link 7 failed: ECONNREFUSED');
});
it('never rejects, so a failure inside it cannot escape into the scheduler', async () => {
const { job, sync } = makeJob();
sync.dueLinks.mockImplementation(() => {
throw new Error('database is locked');
});
await expect(job.tick()).resolves.toBeUndefined();
expect(log.logError).toHaveBeenCalledWith('Document sync tick failed: database is locked');
});
it('logs a failure thrown as a bare value by what it printed, not as an empty message', async () => {
const { job, addons } = makeJob();
addons.isAddonEnabled.mockImplementation(() => {
throw 'SQLITE_BUSY';
});
await expect(job.tick()).resolves.toBeUndefined();
expect(log.logError).toHaveBeenCalledWith('Document sync tick failed: SQLITE_BUSY');
});
it('recovers on the next due tick after a failed one', async () => {
// A failed pass still spends its interval, on purpose: a tick that throws
// every time would otherwise run on every minute the cron fires. It does not
// block anything permanently: the next due tick works normally.
vi.useFakeTimers();
try {
vi.setSystemTime(new Date(10_000_000));
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
sync.dueLinks.mockImplementationOnce(() => {
throw new Error('down');
});
await job.tick();
expect(sync.syncLink).not.toHaveBeenCalled();
expect(log.logError).toHaveBeenCalledTimes(1);
vi.setSystemTime(new Date(10_300_000));
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});
it('leaves the choice of what is due to the sync service', async () => {
// The job passes the rows through untouched: the backoff, the circuit
// breaker and the ordering all live in dueLinks, and a filter copied up
// here would be a second place to keep them in step with.
const links = [link(4), link(5)];
const { job, sync } = makeJob({ links });
await job.tick();
expect(sync.syncLink.mock.calls.map((c) => c[0])).toEqual(links);
});
});
/**
* The interval, now that the cron fires every minute.
*
* It used to be baked into the cron expression at bootstrap, so the setting did
* nothing until a restart while this file's own header said otherwise. The tick
* now decides, which also means the clamp has to hold here instead of in the
* expression: an interval of 0 must not turn the minute cron into a run every
* minute, and a huge one must not park the job for a day.
*/
describe('DocSyncJob due-ness', () => {
// Fake system time rather than a Date.now spy: the spy has to be re-applied
// after the suite-wide clearAllMocks, and a cleared spy answers undefined,
// which reads as "no time has passed" and fails the case for the wrong reason.
const at = (ms: number) => vi.setSystemTime(new Date(ms));
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it('runs the first tick it is given', async () => {
const { job, sync } = makeJob({ links: [link(1)] });
at(1_000_000);
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('does nothing on a tick inside the interval', async () => {
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
at(1_000_000);
await job.tick();
at(1_000_000 + 299_000);
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('runs again once the interval has passed', async () => {
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
at(1_000_000);
await job.tick();
at(1_000_000 + 300_000);
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('keeps the floor: a zero or negative setting still waits a minute', async () => {
for (const setting of ['0', '-300', '10']) {
const { job, sync } = makeJob({ links: [link(1)], interval: setting });
at(2_000_000);
await job.tick();
at(2_000_000 + 59_000);
await job.tick();
expect(sync.syncLink, `interval=${setting} ran again inside the floor`).toHaveBeenCalledTimes(1);
at(2_000_000 + 60_000);
await job.tick();
expect(sync.syncLink, `interval=${setting} did not run at the floor`).toHaveBeenCalledTimes(2);
}
});
it('keeps the ceiling: an absurd setting waits an hour, not a day', async () => {
const { job, sync } = makeJob({ links: [link(1)], interval: '86400' });
at(3_000_000);
await job.tick();
at(3_000_000 + 3_600_000);
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('takes a changed setting without a restart, which is the whole point', async () => {
const settings = new Map<string, string>([[SETTING_POLL_INTERVAL, '3600']]);
const db = {
get: vi.fn((_sql: string, key?: unknown) => {
const value = settings.get(String(key));
return value === undefined ? undefined : { value };
}),
};
const sync = {
dueLinks: vi.fn(() => [link(1)]),
syncLink: vi.fn(async () => RUN),
};
const job = new DocSyncJob(
db as unknown as DatabaseService,
sync as unknown as DocSyncService,
{ markOrphanedLinks: vi.fn(() => 0) } as unknown as DocSyncConfigService,
{ isAddonEnabled: vi.fn(() => true) } as unknown as AddonsService,
{ isEnabled: vi.fn(() => true), register: vi.fn(), unregister: vi.fn() } as unknown as CronRegistrarService,
);
at(4_000_000);
await job.tick();
at(4_000_000 + 120_000);
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(1); // still inside the hour
settings.set(SETTING_POLL_INTERVAL, '60');
await job.tick();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('does not count a tick the kill switch stopped, so switching back on runs at once', async () => {
const { job, sync } = makeJob({ links: [link(1)], interval: '3600', killSwitch: 'false' });
at(5_000_000);
await job.tick();
expect(sync.syncLink).not.toHaveBeenCalled();
});
});
/**
* A provider an admin switched off, over the real service and a real database.
*
* The job leaves what is due to `dueLinks` (see "leaves the choice of what is
* due to the sync service" above), so the switch is a property of that query
* and only shows with the query running. A stubbed `dueLinks` would restate
* whatever this file told it.
*/
describe('DocSyncJob and a provider switched off in the admin panel', () => {
const testDb = new Database(':memory:');
const dbs = new DatabaseService(testDb);
let paperlessLink: number;
let nextcloudLink: number;
const fakeProvider = (id: string) => ({
id,
capabilities: () => ({
push: 'none', stableId: true, remoteTrash: true, replaceInPlace: true, contentHashInListing: true,
maxUploadBytes: null, acceptedMimeTypes: null, canCreateScope: true,
}),
resolveScope: vi.fn(async () => ({ success: true, data: { scopeKey: 'tag:1', label: 'Japan', remoteRootId: '1', remoteRootPath: null } })),
list: vi.fn(async () => ({ success: true, data: { documents: [], cursor: null, cursorUnchanged: false, truncated: false } })),
});
const paperless = fakeProvider('paperless');
const nextcloud = fakeProvider('nextcloud');
const addons = { isAddonEnabled: vi.fn(() => true) } as unknown as AddonsService;
const registry = new DocumentProviderRegistry([paperless, nextcloud] as unknown as DocumentProvider[]);
const config = new DocSyncConfigService(dbs, registry);
const service = new DocSyncService(
dbs, config, registry,
{} as StorageService, {} as FilesService, new AllowedFileTypesService(dbs),
{ broadcast: vi.fn() } as unknown as RealtimeService,
addons,
);
const registrar = { isEnabled: () => false } as unknown as CronRegistrarService;
/** A job of its own per pass, so the interval never decides whether a tick runs. */
const tick = () => new DocSyncJob(dbs, service, config, addons, registrar).tick();
const switchProvider = (id: string, on: boolean) =>
testDb.prepare('UPDATE document_providers SET enabled = ? WHERE id = ?').run(on ? 1 : 0, id);
const linkRow = (id: number) => testDb.prepare('SELECT * FROM trip_document_links WHERE id = ?').get(id);
beforeAll(() => {
createTables(testDb);
runMigrations(testDb);
const ownerId = createUser(testDb, { username: 'owner', email: 'owner@docsync-job.test' }).user.id;
const tripId = createTrip(testDb, ownerId, { title: 'Japan' }).id;
const bind = (providerId: string) => {
const conn = testDb
.prepare(
`INSERT INTO document_connections (trip_id, provider_id, owner_user_id, base_url, secrets, settings)
VALUES (?, ?, ?, 'https://docs.example.com', NULL, '{}')`,
)
.run(tripId, providerId, ownerId);
return Number(testDb
.prepare(
`INSERT INTO trip_document_links
(trip_id, connection_id, provider_id, remote_scope_key, remote_label, direction, delete_policy,
conflict_policy, sync_enabled, created_by)
VALUES (?, ?, ?, 'tag:1', 'Japan', 'both', 'unlink', 'manual', 1, ?)`,
)
.run(tripId, conn.lastInsertRowid, providerId, ownerId).lastInsertRowid);
};
paperlessLink = bind('paperless');
nextcloudLink = bind('nextcloud');
});
afterAll(() => testDb.close());
beforeEach(() => {
switchProvider('paperless', false);
switchProvider('nextcloud', true);
});
it('runs the bindings that may run and leaves the switched-off one exactly as it was', async () => {
const before = linkRow(paperlessLink);
await tick();
expect(nextcloud.list).toHaveBeenCalledTimes(1);
expect(paperless.resolveScope).not.toHaveBeenCalled();
expect(paperless.list).not.toHaveBeenCalled();
expect(linkRow(paperlessLink)).toEqual(before);
expect(linkRow(nextcloudLink)).toMatchObject({ last_sync_state: 'ok' });
});
it('picks the binding up again on the next tick once the provider is back on', async () => {
await tick();
expect(paperless.list).not.toHaveBeenCalled();
switchProvider('paperless', true);
await tick();
expect(paperless.list).toHaveBeenCalledTimes(1);
expect(linkRow(paperlessLink)).toMatchObject({ last_sync_state: 'ok', failure_count: 0 });
});
});
@@ -0,0 +1,267 @@
/**
* The MCP half of document sync, built with `new` and plain mocks: no registry,
* no MCP server, no container.
*
* The tool bodies are three lines each, and what is worth pinning about them is
* not the plumbing but the decisions around it. The access check is the whole
* security story: there is no guard in front of an MCP tool, so a dropped
* `verifyTripAccess` is a stranger reading where somebody's documents live. The
* absence of a connect tool is a decision too: setting a connection up means
* handing over a credential that usually reaches a person's entire document
* archive, and a decision nobody asserts is one somebody re-adds by accident.
* And the refusal for an unconnected trip is the only thing standing between an
* assistant and answering "nothing to sync" to a question about someone's
* boarding pass.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { z, type ZodRawShape } from 'zod';
import { ADDON_IDS } from '../../../../src/addons';
import { DocSyncMcp } from '../../../../src/nest/doc-sync/doc-sync.mcp';
import type { DocSyncConfigService, LinkRow } from '../../../../src/nest/doc-sync/doc-sync-config.service';
import type { DocSyncService } from '../../../../src/nest/doc-sync/doc-sync.service';
import type { AddonsService } from '../../../../src/nest/addons/addons.service';
import type { FilesService } from '../../../../src/nest/files/files.service';
import { getEntry, type ClassRef } from '../../../../src/nest-mcp/metadata';
import type { McpContext, McpTextResult, ToolOptions } from '../../../../src/nest-mcp';
const ctx = { userId: 7, scopes: null, isStaticToken: false } as McpContext;
const link = (id: number, providerId = 'paperless'): LinkRow =>
({ id, provider_id: providerId, trip_id: 3 } as LinkRow);
const RUN = { state: 'ok', pulled: 2, pushed: 1, conflicts: 0, missing: 0 };
interface Setup {
/** `undefined` is what canAccessTrip answers for a trip the user cannot see. */
access: boolean;
links: LinkRow[];
status: Record<string, unknown>;
issues: Array<Record<string, unknown>>;
addonOn: boolean;
/** Bindings whose provider an admin has switched off. */
off: number[];
}
function makeMcp(over: Partial<Setup> = {}) {
const setup: Setup = { access: true, links: [], status: {}, issues: [], addonOn: true, off: [], ...over };
const files = { verifyTripAccess: vi.fn(() => (setup.access ? { id: 3, user_id: 7 } : undefined)) };
const config = { listLinks: vi.fn(() => setup.links) };
const sync = {
status: vi.fn(() => setup.status),
issues: vi.fn(() => setup.issues),
syncLink: vi.fn(async () => RUN),
retryShelvedItems: vi.fn(),
isSwitchedOff: vi.fn((l: LinkRow) => setup.off.includes(l.id)),
};
const addons = { isAddonEnabled: vi.fn(() => setup.addonOn) };
const mcp = new DocSyncMcp(
config as unknown as DocSyncConfigService,
sync as unknown as DocSyncService,
files as unknown as FilesService,
addons as unknown as AddonsService,
);
return { mcp, files, config, sync, addons };
}
/** A success is one JSON text block and carries no error flag. */
function payload(result: McpTextResult): Record<string, unknown> {
expect(result.isError).toBeUndefined();
return JSON.parse(result.content[0].text) as Record<string, unknown>;
}
function refusal(result: McpTextResult): string {
expect(result.isError).toBe(true);
return result.content[0].text;
}
function toolOptions(method: string): ToolOptions {
const entry = getEntry(DocSyncMcp as unknown as ClassRef, method);
if (!entry || entry.kind !== 'tool') throw new Error(`${method} is not a registered MCP tool`);
return entry.options;
}
/**
* Read off the prototype rather than listed here, so a fourth tool fails the
* set assertion below instead of quietly slipping past a table this file keeps.
*/
function registeredMethods(): string[] {
return Object.getOwnPropertyNames(DocSyncMcp.prototype)
.filter((name) => getEntry(DocSyncMcp as unknown as ClassRef, name) !== undefined)
.sort();
}
beforeEach(() => vi.clearAllMocks());
describe('DocSyncMcp surface', () => {
it('offers three tools, none of which can create or change a connection', () => {
expect(registeredMethods()).toEqual(['getTripDocumentSync', 'listIssues', 'syncNow']);
expect(registeredMethods().map((m) => toolOptions(m).name).sort()).toEqual([
'get_trip_document_sync',
'list_trip_document_sync_issues',
'sync_trip_documents',
]);
});
it('asks for nothing but a trip id and a full-run flag', () => {
// The other half of the same decision: a credential surface could arrive
// bolted onto an existing schema rather than as a tool of its own.
const fields = registeredMethods().flatMap((m) => Object.keys(toolOptions(m).inputSchema as ZodRawShape));
expect([...new Set(fields)].sort()).toEqual(['full', 'tripId']);
});
it.each(registeredMethods())('%s disappears while the documents addon is off', (method) => {
const { mcp, addons } = makeMcp({ addonOn: false });
const { when } = toolOptions(method);
expect(typeof when).toBe('function');
expect(when?.(ctx, mcp)).toBe(false);
expect(addons.isAddonEnabled).toHaveBeenCalledWith(ADDON_IDS.DOCUMENTS);
addons.isAddonEnabled.mockReturnValue(true);
expect(when?.(ctx, mcp)).toBe(true);
});
it.each([
['getTripDocumentSync', 'read'],
['listIssues', 'read'],
['syncNow', 'write'],
])('%s claims files:%s, so a read-scoped token cannot move files', (method, mode) => {
expect(toolOptions(method).access).toEqual({ group: 'files', mode });
});
it('advertises the run as a write even though it creates nothing itself', () => {
// It moves bytes in both directions, and a host that trusts readOnlyHint
// would otherwise call it unattended.
expect(toolOptions('syncNow').annotations).toMatchObject({ readOnlyHint: false, destructiveHint: false });
expect(toolOptions('getTripDocumentSync').annotations).toMatchObject({ readOnlyHint: true });
expect(toolOptions('listIssues').annotations).toMatchObject({ readOnlyHint: true });
});
it('refuses a trip id that is not a positive integer before a handler sees it', () => {
const schema = z.object(toolOptions('getTripDocumentSync').inputSchema as ZodRawShape);
expect(schema.safeParse({ tripId: 3 }).success).toBe(true);
expect(schema.safeParse({ tripId: 0 }).success).toBe(false);
expect(schema.safeParse({ tripId: -1 }).success).toBe(false);
expect(schema.safeParse({ tripId: 1.5 }).success).toBe(false);
});
});
describe('DocSyncMcp access', () => {
it.each([
['get_trip_document_sync', (m: DocSyncMcp) => m.getTripDocumentSync({ tripId: 3 }, ctx)],
['list_trip_document_sync_issues', (m: DocSyncMcp) => m.listIssues({ tripId: 3 }, ctx)],
['sync_trip_documents', (m: DocSyncMcp) => m.syncNow({ tripId: 3 }, ctx)],
])('%s tells a user who cannot see the trip nothing beyond that', async (_name, call) => {
const { mcp, files } = makeMcp({ access: false, links: [link(1)] });
const result = await call(mcp);
expect(refusal(result as McpTextResult)).toBe('Trip not found or access denied.');
expect(files.verifyTripAccess).toHaveBeenCalledWith(3, ctx.userId);
});
it('does not reach the sync service at all for a trip the user cannot see', async () => {
const { mcp, sync, config } = makeMcp({ access: false, links: [link(1)] });
await mcp.syncNow({ tripId: 3 }, ctx);
mcp.listIssues({ tripId: 3 }, ctx);
mcp.getTripDocumentSync({ tripId: 3 }, ctx);
expect(sync.syncLink).not.toHaveBeenCalled();
expect(sync.status).not.toHaveBeenCalled();
expect(sync.issues).not.toHaveBeenCalled();
expect(config.listLinks).not.toHaveBeenCalled();
});
});
describe('get_trip_document_sync', () => {
it('hands back the trip status as the service reports it', () => {
const status = { links: [{ id: 1, remoteLabel: 'Japan 2026' }], items: { synced: 4, conflict: 1 } };
const { mcp, sync } = makeMcp({ status });
expect(payload(mcp.getTripDocumentSync({ tripId: 3 }, ctx))).toEqual(status);
expect(sync.status).toHaveBeenCalledWith(3);
});
});
describe('list_trip_document_sync_issues', () => {
it('says the trip is not configured rather than answering with an empty list', () => {
// An empty list and no connection at all read identically to an assistant,
// and "everything is in step" is the wrong answer to give about documents
// that were never being synced.
const { mcp, sync } = makeMcp({ links: [] });
expect(payload(mcp.listIssues({ tripId: 3 }, ctx))).toEqual({ configured: false, issues: [] });
expect(sync.issues).not.toHaveBeenCalled();
});
it('reports an empty list for a configured trip with nothing to decide', () => {
const { mcp } = makeMcp({ links: [link(1)], issues: [] });
expect(payload(mcp.listIssues({ tripId: 3 }, ctx))).toEqual({ configured: true, issues: [] });
});
it('passes the open issues through for a configured trip', () => {
const issues = [{ id: 9, state: 'conflict', remote_name: 'boarding.pdf' }];
const { mcp, sync } = makeMcp({ links: [link(1)], issues });
expect(payload(mcp.listIssues({ tripId: 3 }, ctx))).toEqual({ configured: true, issues });
expect(sync.issues).toHaveBeenCalledWith(3);
});
});
describe('sync_trip_documents', () => {
it('refuses an unconnected trip with the step that would fix it', async () => {
const { mcp, sync } = makeMcp({ links: [] });
const text = refusal(await mcp.syncNow({ tripId: 3 }, ctx));
expect(text).toContain('not connected to a document store');
expect(text).toContain('file manager');
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('runs every binding of the trip and reports each one separately', async () => {
// A trip can be bound to more than one store, and collapsing the runs into
// one total would hide which of them failed.
const links = [link(1, 'paperless'), link(2, 'nextcloud')];
const { mcp, sync } = makeMcp({ links });
const body = payload(await mcp.syncNow({ tripId: 3 }, ctx));
expect(sync.syncLink).toHaveBeenCalledTimes(2);
expect(body.runs).toEqual([
{ linkId: 1, provider: 'paperless', ...RUN },
{ linkId: 2, provider: 'nextcloud', ...RUN },
]);
});
it('un-shelves the documents that gave up before running, as the REST route does', async () => {
// "Run it now" means "try again", including the items parked after too many
// failures. A tool that skipped this reported "in sync" while leaving them
// parked, and the only way out was the button in the web UI.
const links = [link(1, 'paperless'), link(2, 'nextcloud')];
const { mcp, sync } = makeMcp({ links });
await mcp.syncNow({ tripId: 3 }, ctx);
expect(sync.retryShelvedItems.mock.calls).toEqual([[1], [2]]);
});
it('refuses with the same code as the REST route when every binding is switched off', async () => {
const { mcp, sync } = makeMcp({ links: [link(1)], off: [1] });
const text = refusal(await mcp.syncNow({ tripId: 3 }, ctx));
expect(text.startsWith('provider_disabled:')).toBe(true);
expect(text).toContain('switched back on');
// Nothing un-shelved, nothing run: the binding resumes as it was left.
expect(sync.retryShelvedItems).not.toHaveBeenCalled();
expect(sync.syncLink).not.toHaveBeenCalled();
});
it('runs the bindings that are on and reports the switched-off one by its code', async () => {
const links = [link(1, 'paperless'), link(2, 'nextcloud')];
const { mcp, sync } = makeMcp({ links, off: [1] });
const body = payload(await mcp.syncNow({ tripId: 3 }, ctx));
expect(body.runs).toEqual([
{ linkId: 1, provider: 'paperless', state: 'disabled', errorCode: 'provider_disabled' },
{ linkId: 2, provider: 'nextcloud', ...RUN },
]);
expect(sync.retryShelvedItems.mock.calls).toEqual([[2]]);
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
it('asks for the cheap incremental run unless the caller says otherwise', async () => {
const { mcp, sync } = makeMcp({ links: [link(1)] });
await mcp.syncNow({ tripId: 3 }, ctx);
expect(sync.syncLink).toHaveBeenCalledWith(link(1), { full: false });
await mcp.syncNow({ tripId: 3, full: true }, ctx);
expect(sync.syncLink).toHaveBeenLastCalledWith(link(1), { full: true });
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,939 @@
/**
* The Paperless-ngx adapter: PAPERLESS-001..095.
*
* Two things are pinned here. What the client puts on the wire: the pinned API
* version (an unpinned Accept header is how the `checksum` field silently moved
* out of the document serializer between two `docker pull`s), the token header,
* the `original=true` on every download, and the one JSON object the custom
* fields have to arrive in. And what the adapter makes of the answers: the
* error code for each way a self-hosted instance disappoints, the newest
* revision's hash rather than the root one, and the document id surviving a
* replace although the consume task names a different one.
*
* `safeFetch` and the capped readers are mocked: the SSRF guard does a real DNS
* lookup and the reader wants a real stream. The readCappedJson stub still
* parses the bytes it is handed, so "Paperless answered an HTML login page"
* stays a genuine parse failure instead of a mock returning undefined on
* command. The behaviour asserted here was measured against a live
* Paperless-ngx 3.1.3 first; see BEFUND-paperless.md in the test lab.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { Readable } from 'node:stream';
const {
safeFetchMock,
readCappedJsonMock,
readCappedTextMock,
discardBodyMock,
FakeSsrfBlockedError,
} = vi.hoisted(() => {
// Declared inside the hoisted block: a class at module level is not hoisted
// with the vi.mock factory and would be read before it exists.
class FakeSsrfBlockedError extends Error {
constructor(message: string) {
super(message);
this.name = 'SsrfBlockedError';
}
}
return {
safeFetchMock: vi.fn(),
readCappedJsonMock: vi.fn(),
readCappedTextMock: vi.fn(),
discardBodyMock: vi.fn(),
FakeSsrfBlockedError,
};
});
vi.mock('../../../../src/utils/ssrfGuard', () => ({
safeFetch: safeFetchMock,
SsrfBlockedError: FakeSsrfBlockedError,
}));
vi.mock('../../../../src/utils/cappedFetch', () => ({
readCappedJson: readCappedJsonMock,
readCappedText: readCappedTextMock,
discardBody: discardBodyMock,
}));
import {
PaperlessClient,
PaperlessError,
apiBase,
buildMultipart,
currentVersion,
sha256OrNull,
DOCUMENT_PAGE_SIZE,
} from '../../../../src/nest/doc-sync/providers/paperless.client';
import {
PaperlessDocumentProvider,
PAPERLESS_WEBHOOK_SECRET_HEADER,
TRIP_UID_FIELD_NAME,
documentFileName,
normalizeMime,
parseScopeKey,
titleFromFileName,
} from '../../../../src/nest/doc-sync/providers/paperless.provider';
import { PROVIDER_MAX_PAGES } from '../../../../src/nest/doc-sync/doc-sync.constants';
import type {
DocResult,
DocumentConnectionRef,
DocumentScopeRef,
PushRequest,
} from '../../../../src/nest/doc-sync/document-provider';
import { docFailed } from '../../../../src/nest/doc-sync/document-provider';
const TOKEN = 'ba17f0c4d2e94f118a6c0d3b7e5a9c21f4d6b8e0';
const BASE = 'https://papers.example.org';
const CONN: DocumentConnectionRef = {
connectionId: 4,
createdAt: '2026-09-01 08:00:00',
ownerId: 9,
baseUrl: BASE,
secrets: { api_token: TOKEN },
settings: {},
allowInsecureTls: false,
};
const SCOPE: DocumentScopeRef = {
linkId: 7,
tripId: 42,
scopeKey: 'tag:3',
remoteRootId: '3',
remoteRootPath: '/documents?tags__id__all=3',
cursor: null,
};
const HASH_A = 'a'.repeat(64);
const HASH_B = 'b'.repeat(64);
interface Reply {
ok: boolean;
status: number;
headers: Headers;
body: ReadableStream<Uint8Array> | null;
text(): Promise<string>;
}
interface Recorded {
url: URL;
init: RequestInit;
options?: { rejectUnauthorized?: boolean };
body: string;
}
let calls: Recorded[] = [];
let routes: Map<string, Reply[]>;
function reply(body: unknown, status = 200, headers: Record<string, string> = {}): Reply {
const raw = typeof body === 'string' ? body : JSON.stringify(body);
return {
ok: status >= 200 && status < 300,
status,
headers: new Headers(headers),
body: null,
text: async () => raw,
};
}
/** A streaming answer, for the download path. */
function streamReply(bytes: string, headers: Record<string, string> = {}): Reply {
return {
ok: true,
status: 200,
headers: new Headers(headers),
body: new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(bytes));
controller.close();
},
}),
text: async () => bytes,
};
}
/**
* Stub one route. Several replies are consumed in order and the last one
* repeats, which is what the pagination cases want.
*/
function on(route: string, ...replies: Reply[]): void {
routes.set(route, replies);
}
function requests(route: string): Recorded[] {
return calls.filter((call) => `${(call.init.method ?? 'GET').toUpperCase()} ${call.url.pathname}` === route);
}
function lastRequest(route: string): Recorded {
const matching = requests(route);
const last = matching[matching.length - 1];
if (last === undefined) throw new Error(`no request was made to ${route}`);
return last;
}
function headerOf(call: Recorded, name: string): string | undefined {
const headers = call.init.headers as Record<string, string> | undefined;
return headers?.[name];
}
function tagRow(id: number, name: string, documentCount = 0): unknown {
return { id, name, slug: name, color: '#a6cee3', document_count: documentCount, owner: 2 };
}
function docRow(over: Record<string, unknown> = {}): unknown {
return {
id: 11,
title: 'Hotelrechnung Kyoto',
tags: [3],
mime_type: 'application/pdf',
original_file_name: 'rechnung.pdf',
modified: '2026-09-18T17:30:00.000000+02:00',
added: '2026-09-18T17:29:00.000000+02:00',
deleted_at: null,
custom_fields: [{ field: 1, value: 'trip-uid-42' }],
versions: [{ id: 11, checksum: HASH_A, added: '2026-09-18T15:29:00.000000Z', is_root: true }],
...over,
};
}
function page(results: unknown[], next: string | null = null, count?: number): unknown {
return { count: count ?? results.length, next, previous: null, results };
}
function taskRow(over: Record<string, unknown> = {}): unknown {
return {
id: 3,
task_id: 'task-uuid-1',
status: 'success',
result_data: { document_id: 11 },
related_document_ids: [11],
...over,
};
}
function pushRequest(over: Partial<PushRequest> = {}): PushRequest {
const bytes = Buffer.from('%PDF-1.4 hotel\n');
return {
body: Readable.from(bytes),
fileName: 'rechnung.pdf',
mimeType: 'application/pdf',
size: bytes.length,
sha256: HASH_A,
mtimeSeconds: 1_789_000_000,
trekDocUid: 'doc-uid-1',
trekTripUid: 'trip-uid-42',
...over,
};
}
// `!result.success` does not narrow: the server workspace compiles with
// `strict: false`, so a boolean discriminant stops discriminating. docFailed is
// the predicate the production code uses for the same reason.
function expectFail(result: DocResult<unknown>): { code: string; detail?: string; status?: number } {
if (!docFailed(result)) throw new Error('expected a failure, got success');
return result.error;
}
function expectOk<T>(result: DocResult<T>): T {
if (docFailed(result)) {
throw new Error(`expected success, got ${result.error.code}: ${result.error.detail ?? ''}`);
}
return result.data;
}
let client: PaperlessClient;
let provider: PaperlessDocumentProvider;
beforeEach(() => {
calls = [];
routes = new Map();
client = new PaperlessClient();
provider = new PaperlessDocumentProvider(client);
safeFetchMock.mockReset();
readCappedJsonMock.mockReset();
readCappedTextMock.mockReset();
discardBodyMock.mockReset();
readCappedJsonMock.mockImplementation(async (res: { text(): Promise<string> }) => {
const text = await res.text();
try {
return JSON.parse(text) as unknown;
} catch {
return undefined;
}
});
readCappedTextMock.mockImplementation(async (res: { text(): Promise<string> }) => ({
text: await res.text(),
truncated: false,
}));
safeFetchMock.mockImplementation(
async (url: string, init: RequestInit, options?: { rejectUnauthorized?: boolean }) => {
const parsed = new URL(url);
const method = (init.method ?? 'GET').toUpperCase();
const body = typeof init.body === 'string'
? init.body
: Buffer.isBuffer(init.body)
? init.body.toString('latin1')
: '';
calls.push({ url: parsed, init, options, body });
const queue = routes.get(`${method} ${parsed.pathname}`);
if (queue === undefined || queue.length === 0) {
throw new Error(`no stubbed reply for ${method} ${parsed.pathname}`);
}
return (queue.length > 1 ? queue.shift() : queue[0]) as unknown as Response;
},
);
});
describe('PaperlessClient: the request it builds', () => {
it('PAPERLESS-001: appends /api to whatever the operator pasted', () => {
expect(apiBase(BASE)).toBe(`${BASE}/api`);
expect(apiBase(`${BASE}/`)).toBe(`${BASE}/api`);
expect(apiBase(`${BASE}/api`)).toBe(`${BASE}/api`);
expect(apiBase(`${BASE}/api/`)).toBe(`${BASE}/api`);
expect(apiBase(' https://home.example.org/paperless/ ')).toBe('https://home.example.org/paperless/api');
expect(apiBase('http://192.168.178.9:8000')).toBe('http://192.168.178.9:8000/api');
});
it('PAPERLESS-002: sends the token as a Token header and pins the API version', async () => {
on('GET /api/ui_settings/', reply({ user: { username: 'admin' }, settings: {}, permissions: [] }));
await provider.probe(CONN);
const call = lastRequest('GET /api/ui_settings/');
expect(headerOf(call, 'Authorization')).toBe(`Token ${TOKEN}`);
expect(headerOf(call, 'Accept')).toBe('application/json; version=10');
});
it('PAPERLESS-003: never puts the token in the query string', async () => {
on('GET /api/ui_settings/', reply({ user: { username: 'admin' }, settings: {}, permissions: [] }));
on('GET /api/tags/', reply(page([tagRow(3, 'japan')])));
on('GET /api/documents/', reply(page([docRow()])));
await provider.probe(CONN);
await provider.listScopes(CONN);
await provider.list(CONN, SCOPE);
expect(calls.length).toBeGreaterThanOrEqual(3);
for (const call of calls) {
expect(call.url.search).not.toContain(TOKEN);
expect(call.url.search).not.toMatch(/token|api_key/i);
}
});
it('PAPERLESS-004: maps allowInsecureTls onto rejectUnauthorized and bounds the call', async () => {
on('GET /api/ui_settings/', reply({ user: { username: 'admin' }, settings: {}, permissions: [] }));
await provider.probe(CONN);
await provider.probe({ ...CONN, allowInsecureTls: true });
expect(calls[0]?.options).toEqual({ rejectUnauthorized: true });
expect(calls[1]?.options).toEqual({ rejectUnauthorized: false });
expect(calls[0]?.init.signal).toBeInstanceOf(AbortSignal);
});
it('PAPERLESS-005: reads the profile that does not echo the token back', async () => {
on('GET /api/ui_settings/', reply({
user: { username: 'admin' },
settings: { version: '3.1.3' },
permissions: ['add_workflow', 'add_document'],
}));
const probed = expectOk(await provider.probe(CONN));
expect(probed.account).toBe('admin');
expect(requests('GET /api/profile/')).toHaveLength(0);
});
});
describe('PaperlessProvider: probing a connection', () => {
it('PAPERLESS-010: reports the self-registering webhook when the token may create workflows', async () => {
on('GET /api/ui_settings/', reply({
user: { username: 'admin' },
settings: { version: '3.1.3' },
permissions: ['add_workflow'],
}));
const probed = expectOk(await provider.probe(CONN));
expect(probed.capabilities.push).toBe('webhook-self-registered');
expect(probed.capabilities.stableId).toBe(true);
expect(probed.capabilities.contentHashInListing).toBe(true);
expect(probed.capabilities.replaceInPlace).toBe(true);
expect(probed.capabilities.remoteTrash).toBe(true);
});
it('PAPERLESS-011: downgrades to a manual webhook when the token may not create one', async () => {
on('GET /api/ui_settings/', reply({
user: { username: 'reisender' },
settings: { version: '3.1.3' },
permissions: ['view_document', 'add_document'],
}));
const probed = expectOk(await provider.probe(CONN));
expect(probed.capabilities.push).toBe('webhook-manual');
// The capability the CORE reads stays the provider's own answer; only the
// probe reflects this token's narrower rights.
expect(provider.capabilities(CONN).push).toBe('webhook-self-registered');
});
it('PAPERLESS-012: a wrong token is unauthorized, a forbidden one is forbidden', async () => {
on('GET /api/ui_settings/', reply({ detail: 'Invalid token.' }, 401));
expect(expectFail(await provider.probe(CONN)).code).toBe('unauthorized');
on('GET /api/ui_settings/', reply({ detail: 'no' }, 403));
expect(expectFail(await provider.probe(CONN)).code).toBe('forbidden');
});
it('PAPERLESS-013: an HTML login page is a provider error, not a crash', async () => {
on('GET /api/ui_settings/', reply('<!doctype html><title>Sign in</title>'));
const error = expectFail(await provider.probe(CONN));
expect(error.code).toBe('provider_error');
expect(error.detail).toContain('Paperless itself');
});
it('PAPERLESS-014: a connection without a token never reaches the network', async () => {
const error = expectFail(await provider.probe({ ...CONN, secrets: {} }));
expect(error.code).toBe('unauthorized');
expect(calls).toHaveLength(0);
});
it('PAPERLESS-015: the SSRF guard, a timeout and a self-signed certificate each get their own code', async () => {
safeFetchMock.mockImplementationOnce(async () => {
throw new FakeSsrfBlockedError('Requests to loopback and link-local addresses are not allowed');
});
expect(expectFail(await provider.probe(CONN)).code).toBe('ssrf_blocked');
safeFetchMock.mockImplementationOnce(async () => {
const err = new Error('The operation was aborted due to timeout');
err.name = 'TimeoutError';
throw err;
});
expect(expectFail(await provider.probe(CONN)).code).toBe('timeout');
safeFetchMock.mockImplementationOnce(async () => {
const cause = Object.assign(new Error('self-signed certificate'), {
code: 'DEPTH_ZERO_SELF_SIGNED_CERT',
});
throw Object.assign(new TypeError('fetch failed'), { cause });
});
expect(expectFail(await provider.probe(CONN)).code).toBe('tls_untrusted');
safeFetchMock.mockImplementationOnce(async () => {
throw Object.assign(new TypeError('fetch failed'), {
cause: Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' }),
});
});
expect(expectFail(await provider.probe(CONN)).code).toBe('unreachable');
});
});
describe('PaperlessProvider: scopes are tags', () => {
it('PAPERLESS-020: a tag becomes a scope key, a label and the instance filter URL', async () => {
on('GET /api/tags/', reply(page([tagRow(3, 'japan-2026', 4), tagRow(9, 'privat')])));
const options = expectOk(await provider.listScopes(CONN));
expect(options).toEqual([
{ scopeKey: 'tag:3', label: 'japan-2026', remoteRootId: '3', remoteRootPath: '/documents?tags__id__all=3' },
{ scopeKey: 'tag:9', label: 'privat', remoteRootId: '9', remoteRootPath: '/documents?tags__id__all=9' },
]);
});
it('PAPERLESS-021: follows the pages of a long tag list', async () => {
on(
'GET /api/tags/',
reply(page([tagRow(1, 'a')], 'https://papers.example.org/api/tags/?page=2', 2)),
reply(page([tagRow(2, 'b')], null, 2)),
);
const options = expectOk(await provider.listScopes(CONN));
expect(options.map((option) => option.scopeKey)).toEqual(['tag:1', 'tag:2']);
expect(requests('GET /api/tags/')).toHaveLength(2);
expect(requests('GET /api/tags/')[1]?.url.searchParams.get('page')).toBe('2');
});
it('PAPERLESS-022: a search narrows server-side, an empty one is not sent at all', async () => {
on('GET /api/tags/', reply(page([tagRow(3, 'japan-2026')])));
await provider.listScopes(CONN, ' japan ');
await provider.listScopes(CONN, ' ');
expect(requests('GET /api/tags/')[0]?.url.searchParams.get('name__icontains')).toBe('japan');
expect(requests('GET /api/tags/')[1]?.url.searchParams.has('name__icontains')).toBe(false);
});
it('PAPERLESS-023: creating a tag that exists is a conflict, never a silent adoption', async () => {
on('POST /api/tags/', reply({ error: 'Object violates owner / name unique constraint' }, 400));
expect(expectFail(await provider.createScope(CONN, 'japan-2026')).code).toBe('conflict');
});
it('PAPERLESS-024: a tag that is gone is scope_missing, not a bare 404', async () => {
on('GET /api/tags/3/', reply({ detail: 'No Tag matches the given query.' }, 404));
const error = expectFail(await provider.resolveScope(CONN, SCOPE));
expect(error.code).toBe('scope_missing');
expect(error.status).toBe(404);
});
it('PAPERLESS-025: another provider’s scope key is refused before any request', async () => {
for (const scopeKey of ['path:/trek', 'tag:', 'tag:abc', 'tag:-1', 'fileid:437']) {
const error = expectFail(await provider.resolveScope(CONN, { ...SCOPE, scopeKey }));
expect(error.code).toBe('scope_missing');
}
expect(calls).toHaveLength(0);
expect(parseScopeKey('tag:12')).toBe(12);
});
});
describe('PaperlessProvider: listing a scope', () => {
it('PAPERLESS-030: maps a document onto the core’s shape', async () => {
on('GET /api/documents/', reply(page([docRow()])));
const listed = expectOk(await provider.list(CONN, SCOPE));
expect(listed.documents).toEqual([
{
remoteId: '11',
name: 'Hotelrechnung Kyoto.pdf',
size: null,
mimeType: 'application/pdf',
remoteVersion: '2026-09-18T17:30:00.000000+02:00',
contentHash: HASH_A,
remoteModifiedAt: '2026-09-18T17:30:00.000000+02:00',
isDeleted: false,
},
]);
const call = lastRequest('GET /api/documents/');
expect(call.url.searchParams.get('tags__id__all')).toBe('3');
expect(call.url.searchParams.get('ordering')).toBe('id');
expect(call.url.searchParams.get('page_size')).toBe(String(DOCUMENT_PAGE_SIZE));
});
it('PAPERLESS-031: a checksum that is not sha256 is reported as no hash at all', async () => {
on('GET /api/documents/', reply(page([
docRow({ versions: [{ id: 1, checksum: 'a91fe3f8e2cf4b2b5560b2222d65d60c', added: null, is_root: true }] }),
])));
const listed = expectOk(await provider.list(CONN, SCOPE));
expect(listed.documents[0]?.contentHash).toBeNull();
expect(sha256OrNull('A'.repeat(64))).toBe('a'.repeat(64));
expect(sha256OrNull('zz')).toBeNull();
});
it('PAPERLESS-032: after a replace the hash is the newest revision, not the root one', async () => {
on('GET /api/documents/', reply(page([
docRow({
versions: [
{ id: 12, checksum: HASH_B, added: '2026-09-18T15:40:00.000000Z', is_root: false },
{ id: 11, checksum: HASH_A, added: '2026-09-18T15:29:00.000000Z', is_root: true },
],
}),
])));
const listed = expectOk(await provider.list(CONN, SCOPE));
expect(listed.documents[0]?.contentHash).toBe(HASH_B);
// Independent of the order the instance happens to send them in.
expect(currentVersion({
id: 1,
title: '',
originalFileName: null,
mimeType: null,
modified: null,
added: null,
deletedAt: null,
tagIds: [],
versions: [
{ id: 11, checksum: HASH_A, added: '2026-09-18T15:29:00.000000Z', isRoot: true },
{ id: 12, checksum: HASH_B, added: '2026-09-18T15:40:00.000000Z', isRoot: false },
],
})?.checksum).toBe(HASH_B);
});
it('PAPERLESS-033: a listing that hits the page cap says so and claims nothing about the cursor', async () => {
const endless = reply(page([docRow()], 'https://papers.example.org/api/documents/?page=2', 9999));
on('GET /api/documents/', endless);
const listed = expectOk(await provider.list(CONN, { ...SCOPE, cursor: 'v1:9999:x' }));
expect(listed.truncated).toBe(true);
expect(listed.cursorUnchanged).toBe(false);
expect(requests('GET /api/documents/')).toHaveLength(PROVIDER_MAX_PAGES);
});
it('PAPERLESS-034: an unchanged scope is recognised, a changed one is not', async () => {
on('GET /api/documents/', reply(page([docRow()])));
const first = expectOk(await provider.list(CONN, SCOPE));
expect(first.cursorUnchanged).toBe(false);
const second = expectOk(await provider.list(CONN, { ...SCOPE, cursor: first.cursor }));
expect(second.cursorUnchanged).toBe(true);
// A document leaving the tag moves no timestamp: only the count notices.
on('GET /api/documents/', reply(page([], null, 0)));
const third = expectOk(await provider.list(CONN, { ...SCOPE, cursor: first.cursor }));
expect(third.cursorUnchanged).toBe(false);
expect(third.documents).toHaveLength(0);
});
it('PAPERLESS-035: a document without a title still gets a usable name', async () => {
on('GET /api/documents/', reply(page([
docRow({ id: 5, title: '', original_file_name: 'scan_2026.PDF' }),
docRow({ id: 6, title: '', original_file_name: null, mime_type: 'image/png' }),
docRow({ id: 7, title: 'Bordkarte.pdf', original_file_name: 'bordkarte.pdf' }),
])));
const listed = expectOk(await provider.list(CONN, SCOPE));
expect(listed.documents.map((doc) => doc.name)).toEqual([
'scan_2026.PDF',
'document-6.png',
// An extension the title already carries is not doubled.
'Bordkarte.pdf',
]);
expect(documentFileName({
id: 8,
title: 'Notiz',
originalFileName: null,
mimeType: 'text/plain',
modified: null,
added: null,
deletedAt: null,
tagIds: [],
versions: [],
})).toBe('Notiz.txt');
});
it('PAPERLESS-036: an answer that is not a listing is a provider error', async () => {
on('GET /api/documents/', reply({ unexpected: true }));
expect(expectFail(await provider.list(CONN, SCOPE)).code).toBe('provider_error');
});
});
describe('PaperlessProvider: fetching bytes', () => {
it('PAPERLESS-040: asks for the original file and hands back the stream with its version', async () => {
on('GET /api/documents/11/', reply(docRow()));
on('GET /api/documents/11/download/', streamReply('%PDF-1.4 hotel', {
'content-length': '14',
'content-type': 'application/pdf',
}));
const fetched = expectOk(await provider.fetch(CONN, SCOPE, '11'));
expect(lastRequest('GET /api/documents/11/download/').url.searchParams.get('original')).toBe('true');
expect(fetched.size).toBe(14);
expect(fetched.mimeType).toBe('application/pdf');
expect(fetched.remoteVersion).toBe('2026-09-18T17:30:00.000000+02:00');
const chunks: Buffer[] = [];
for await (const chunk of fetched.body) chunks.push(Buffer.from(chunk));
expect(Buffer.concat(chunks).toString()).toBe('%PDF-1.4 hotel');
});
it('PAPERLESS-041: a missing content-length is an unknown size, never zero', async () => {
on('GET /api/documents/11/', reply(docRow()));
on('GET /api/documents/11/download/', streamReply('%PDF-1.4 hotel'));
const fetched = expectOk(await provider.fetch(CONN, SCOPE, '11'));
expect(fetched.size).toBeNull();
});
it('PAPERLESS-042: a deleted document is not_found, and a nonsense id never leaves the process', async () => {
on('GET /api/documents/11/', reply({ detail: 'No Document matches the given query.' }, 404));
expect(expectFail(await provider.fetch(CONN, SCOPE, '11')).code).toBe('not_found');
const before = calls.length;
expect(expectFail(await provider.fetch(CONN, SCOPE, '../../etc/passwd')).code).toBe('not_found');
expect(calls).toHaveLength(before);
});
it('PAPERLESS-043: a download without a body is a provider error, not an empty file', async () => {
on('GET /api/documents/11/', reply(docRow()));
on('GET /api/documents/11/download/', reply(''));
expect(expectFail(await provider.fetch(CONN, SCOPE, '11'))).toMatchObject({
code: 'provider_error',
detail: 'Paperless sent no body for the document',
});
});
it('PAPERLESS-044: a download announcing more than TREK transfers is refused before it is read', async () => {
on('GET /api/documents/11/', reply(docRow()));
on('GET /api/documents/11/download/', streamReply('%PDF', { 'content-length': String(3 * 1024 ** 3) }));
expect(expectFail(await provider.fetch(CONN, SCOPE, '11'))).toMatchObject({
code: 'too_large',
detail: `content_length=${3 * 1024 ** 3}`,
});
expect(discardBodyMock).toHaveBeenCalledTimes(1);
});
});
describe('PaperlessProvider: pushing bytes', () => {
function stubSuccessfulCreate(): void {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow()])));
on('GET /api/documents/11/', reply(docRow()));
}
it('PAPERLESS-050: an unaccepted type is refused before a single byte is uploaded', async () => {
const error = expectFail(await provider.push(CONN, SCOPE, pushRequest({
fileName: 'tour.gpx',
mimeType: 'application/gpx+xml',
})));
expect(error.code).toBe('unsupported_type');
expect(calls).toHaveLength(0);
});
it('PAPERLESS-050b: a stored Content-Type with parameters is still the type it names', async () => {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow()])));
on('GET /api/documents/11/', reply(docRow({ mime_type: 'text/plain' })));
expectOk(await provider.push(CONN, SCOPE, pushRequest({
fileName: 'notiz.txt',
mimeType: 'Text/Plain; charset=utf-8',
})));
expect(lastRequest('POST /api/documents/post_document/').body).toContain('Content-Type: text/plain\r\n');
expect(normalizeMime('application/pdf')).toBe('application/pdf');
});
it('PAPERLESS-051: an upload carries the tag, the title and the custom field in one request', async () => {
stubSuccessfulCreate();
const pushed = expectOk(await provider.push(CONN, SCOPE, pushRequest()));
expect(pushed).toEqual({
remoteId: '11',
remoteVersion: '2026-09-18T17:30:00.000000+02:00',
remoteModifiedAt: '2026-09-18T17:30:00.000000+02:00',
// Paperless stores duplicate bytes twice, so nothing here may claim a
// deduplication happened.
deduplicated: false,
});
const upload = lastRequest('POST /api/documents/post_document/');
expect(headerOf(upload, 'Content-Type')).toMatch(/^multipart\/form-data; boundary=/);
// Content-Length is undici's business; setting it makes it refuse the call.
expect(headerOf(upload, 'Content-Length')).toBeUndefined();
expect(upload.body).toContain('name="tags"\r\n\r\n3');
expect(upload.body).toContain('name="title"\r\n\r\nrechnung');
expect(upload.body).toContain('name="custom_fields"\r\n\r\n{"1":"trip-uid-42"}');
expect(upload.body).toContain('filename="rechnung.pdf"');
expect(upload.body).toContain('%PDF-1.4 hotel');
});
it('PAPERLESS-052: the trip anchor field is created when the instance has none', async () => {
on('GET /api/custom_fields/', reply(page([])));
on('POST /api/custom_fields/', reply({ id: 4, name: TRIP_UID_FIELD_NAME, data_type: 'string' }));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow()])));
on('GET /api/documents/11/', reply(docRow()));
expectOk(await provider.push(CONN, SCOPE, pushRequest()));
expect(lastRequest('POST /api/custom_fields/').body).toContain(TRIP_UID_FIELD_NAME);
expect(lastRequest('POST /api/documents/post_document/').body).toContain('{"4":"trip-uid-42"}');
});
it('PAPERLESS-053: a custom field list from a build that ignores the filter is not trusted', async () => {
on('GET /api/custom_fields/', reply(page([
{ id: 8, name: 'rechnungsnummer', data_type: 'string' },
{ id: 9, name: 'TREK_TRIP_ID', data_type: 'string' },
])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow()])));
on('GET /api/documents/11/', reply(docRow()));
expectOk(await provider.push(CONN, SCOPE, pushRequest()));
expect(lastRequest('POST /api/documents/post_document/').body).toContain('{"9":"trip-uid-42"}');
expect(requests('POST /api/custom_fields/')).toHaveLength(0);
});
it('PAPERLESS-054: a consume that fails is a provider error, a duplicate one a conflict', async () => {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow({
status: 'failure',
related_document_ids: [],
result_data: { error_type: 'ConsumerError', error_message: 'kaputt.pdf: InputFileError' },
})])));
const failed = expectFail(await provider.push(CONN, SCOPE, pushRequest()));
expect(failed.code).toBe('provider_error');
expect(failed.detail).toContain('InputFileError');
on('GET /api/tasks/', reply(page([taskRow({
status: 'FAILURE',
related_document_ids: [],
result_data: { error_message: 'It is a duplicate of Rechnung (#4)' },
})])));
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('conflict');
});
it('PAPERLESS-055: the task filter is not trusted either, so a foreign task is ignored', async () => {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow({ task_id: 'someone-elses-task', status: 'success' })])));
vi.spyOn(client, 'awaitConsume');
// The only way out is the poll budget, so the client is asked directly with
// a short one rather than holding the test for two minutes.
const creds = { baseUrl: BASE, token: TOKEN, allowInsecureTls: false };
await expect(
client.awaitConsume(creds, 'task-uuid-1', { intervalMs: 1, timeoutMs: 5 }),
).rejects.toMatchObject({ code: 'timeout' });
});
it('PAPERLESS-056: an upload whose task outlives the budget is found again by its checksum', async () => {
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
);
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
// One document with the same bytes in another tag, one in ours: the tagged
// one is the upload we just made.
on('GET /api/documents/', reply(page([
docRow({ id: 21, tags: [99] }),
docRow({ id: 11, tags: [3] }),
])));
on('GET /api/documents/11/', reply(docRow()));
const pushed = expectOk(await provider.push(CONN, SCOPE, pushRequest()));
expect(pushed.remoteId).toBe('11');
expect(lastRequest('GET /api/documents/').url.searchParams.get('checksum__iexact')).toBe(HASH_A);
});
it('PAPERLESS-057a: a document with the same bytes but another tag is not adopted', async () => {
// The tag goes on at upload, so only a tagged document can be the one this
// push made. Taking the untagged twin instead put a stranger's document in
// the trip, and delete-through would have trashed it on a later run.
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
);
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/documents/', reply(page([docRow({ id: 21, tags: [99] })])));
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('timeout');
expect(requests('GET /api/documents/21/')).toHaveLength(0);
});
it('PAPERLESS-057: when the checksum finds nothing either, the timeout is reported', async () => {
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
);
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/documents/', reply(page([])));
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('timeout');
});
it('PAPERLESS-058: a replace keeps the id it was given, not the one the task names', async () => {
on('POST /api/documents/11/update_version/', reply('"task-uuid-2"'));
// update_version files the new revision under a fresh internal id that
// answers 404 on its own. Adopting it would break the mapping.
on('GET /api/tasks/', reply(page([taskRow({ task_id: 'task-uuid-2', related_document_ids: [77] })])));
on('GET /api/documents/11/', reply(docRow({
modified: '2026-09-18T18:00:00.000000+02:00',
versions: [
{ id: 12, checksum: HASH_A, added: '2026-09-18T16:00:00.000000Z', is_root: false },
{ id: 11, checksum: HASH_B, added: '2026-09-18T15:29:00.000000Z', is_root: true },
],
})));
const pushed = expectOk(await provider.push(CONN, SCOPE, pushRequest({ remoteId: '11' })));
expect(pushed.remoteId).toBe('11');
expect(pushed.remoteVersion).toBe('2026-09-18T18:00:00.000000+02:00');
expect(requests('POST /api/documents/post_document/')).toHaveLength(0);
expect(requests('GET /api/documents/77/')).toHaveLength(0);
});
it('PAPERLESS-059: a file larger than it claimed is refused without being uploaded', async () => {
const error = expectFail(await provider.push(CONN, SCOPE, pushRequest({ size: 4 })));
expect(error.code).toBe('too_large');
expect(requests('POST /api/documents/post_document/')).toHaveLength(0);
});
it('PAPERLESS-060: a type the instance refuses comes back as the same code as the local check', async () => {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply({ document: ['File type text/xml not supported'] }, 400));
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('unsupported_type');
});
it('PAPERLESS-061: bytes that arrived differently are reported and the new document is trashed', async () => {
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/tasks/', reply(page([taskRow()])));
on('GET /api/documents/11/', reply(docRow({
versions: [{ id: 11, checksum: HASH_B, added: '2026-09-18T15:29:00.000000Z', is_root: true }],
})));
on('DELETE /api/documents/11/', reply('', 204));
const error = expectFail(await provider.push(CONN, SCOPE, pushRequest({ sha256: HASH_A })));
expect(error.code).toBe('checksum_mismatch');
expect(requests('DELETE /api/documents/11/')).toHaveLength(1);
});
it('PAPERLESS-062: a mismatch after a replace leaves the existing document alone', async () => {
on('POST /api/documents/11/update_version/', reply('"task-uuid-2"'));
on('GET /api/tasks/', reply(page([taskRow({ task_id: 'task-uuid-2' })])));
on('GET /api/documents/11/', reply(docRow({
versions: [{ id: 12, checksum: HASH_B, added: '2026-09-18T16:00:00.000000Z', is_root: false }],
})));
const error = expectFail(await provider.push(CONN, SCOPE, pushRequest({ remoteId: '11', sha256: HASH_A })));
expect(error.code).toBe('checksum_mismatch');
expect(requests('DELETE /api/documents/11/')).toHaveLength(0);
});
});
describe('PaperlessProvider: renaming and trashing', () => {
it('PAPERLESS-070: renaming sets the title without the extension it would add back', async () => {
on('PATCH /api/documents/11/', reply(docRow({ modified: '2026-09-18T19:00:00.000000+02:00' })));
const renamed = expectOk(await provider.rename(CONN, SCOPE, '11', ' Hotelrechnung Kyoto.pdf '));
expect(lastRequest('PATCH /api/documents/11/').body).toBe('{"title":"Hotelrechnung Kyoto"}');
expect(renamed.remoteVersion).toBe('2026-09-18T19:00:00.000000+02:00');
// An extension this adapter would never have added stays part of the name.
expect(titleFromFileName('Vertrag 2026.2')).toBe('Vertrag 2026.2');
expect(titleFromFileName('bordkarte.PDF')).toBe('bordkarte');
});
it('PAPERLESS-071: renaming a document that is gone is not_found', async () => {
on('PATCH /api/documents/11/', reply({ detail: 'No Document matches the given query.' }, 404));
expect(expectFail(await provider.rename(CONN, SCOPE, '11', 'neu.pdf')).code).toBe('not_found');
});
it('PAPERLESS-080: trashing is a soft delete, and doing it twice is still success', async () => {
on('DELETE /api/documents/11/', reply('', 204), reply({ detail: 'No Document matches the given query.' }, 404));
expectOk(await provider.trash(CONN, SCOPE, '11'));
expectOk(await provider.trash(CONN, SCOPE, '11'));
expect(requests('DELETE /api/documents/11/')).toHaveLength(2);
});
});
describe('PaperlessProvider: the self-registered webhook', () => {
it('PAPERLESS-090: subscribes both triggers, filtered on the tag, with the secret in a header', async () => {
on('POST /api/workflows/', reply({ id: 6 }, 201));
const registered = expectOk(
await provider.registerWebhook(CONN, SCOPE, 'https://trek.example.org/api/docsync/hook/7', 'sh4red'),
);
expect(registered.subscriptionId).toBe('6');
const sent = JSON.parse(lastRequest('POST /api/workflows/').body) as {
triggers: Array<{ type: number; filter_has_tags: number[]; sources: number[] }>;
actions: Array<{ type: number; webhook: Record<string, unknown> }>;
};
expect(sent.triggers.map((trigger) => trigger.type)).toEqual([2, 3]);
for (const trigger of sent.triggers) {
expect(trigger.filter_has_tags).toEqual([3]);
// Including the web UI, or a document dropped in by hand fires nothing.
expect(trigger.sources).toEqual([1, 2, 3, 4]);
}
expect(sent.actions[0]?.type).toBe(4);
expect(sent.actions[0]?.webhook).toMatchObject({
url: 'https://trek.example.org/api/docsync/hook/7',
use_params: true,
as_json: true,
include_document: false,
params: { linkId: '7', tripId: '42' },
headers: { [PAPERLESS_WEBHOOK_SECRET_HEADER]: 'sh4red' },
});
// The secret travels in a header, never in the URL or the body.
expect(lastRequest('POST /api/workflows/').url.search).not.toContain('sh4red');
expect(JSON.stringify(sent.actions[0]?.webhook.params)).not.toContain('sh4red');
});
it('PAPERLESS-091: a callback URL Paperless cannot store is refused here, not upstream', async () => {
const long = `https://trek.example.org/api/docsync/hook/${'x'.repeat(240)}`;
const error = expectFail(await provider.registerWebhook(CONN, SCOPE, long, 's'));
expect(error.code).toBe('provider_error');
expect(error.detail).toContain('limit 256');
expect(calls).toHaveLength(0);
});
it('PAPERLESS-092: unsubscribing twice is success, and a nonsense id never leaves the process', async () => {
on('DELETE /api/workflows/6/', reply('', 204), reply({ detail: 'Not found.' }, 404));
expectOk(await provider.unregisterWebhook(CONN, '6'));
expectOk(await provider.unregisterWebhook(CONN, '6'));
const before = calls.length;
expect(expectFail(await provider.unregisterWebhook(CONN, 'workflow-6')).code).toBe('not_found');
expect(calls).toHaveLength(before);
});
});
describe('the multipart body', () => {
it('PAPERLESS-095: a file name cannot break out of the Content-Disposition header', () => {
const { body, contentType } = buildMultipart(
[{ name: 'tags', value: '3' }],
{
fileName: 'a";\r\nX-Injected: 1\r\n\r\nevil.pdf',
mimeType: 'application/pdf',
bytes: Buffer.from('bytes'),
},
);
const text = body.toString('latin1');
expect(text).not.toContain('X-Injected: 1\r\n');
// Quote, CR and LF each become an underscore; everything else survives, so
// the name stays recognisable in the archive.
expect(text).toContain('filename="a_;__X-Injected: 1____evil.pdf"');
const boundary = /boundary=(.+)$/.exec(contentType)?.[1] ?? '';
expect(boundary.length).toBeGreaterThan(16);
expect(text.endsWith(`--${boundary}--\r\n`)).toBe(true);
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,317 @@
/**
* The transport the four document clients share.
*
* The clients' own suites pin what each product puts on the wire; this one
* pins the pieces they all stand on, because a regression here breaks five
* providers at once: which failures read as a timeout, a certificate or a dead
* host, that the timeout and the self-signed switch reach the fetch, and that
* a download can neither announce nor stream its way past its ceiling.
*
* Only `safeFetch` is mocked. `cappedFetch` runs for real, so releasing an
* oversized body is the release that ships.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const { safeFetchMock, SsrfBlockedErrorMock } = vi.hoisted(() => {
class SsrfBlockedError extends Error {
constructor(message: string) {
super(message);
this.name = 'SsrfBlockedError';
}
}
return { safeFetchMock: vi.fn(), SsrfBlockedErrorMock: SsrfBlockedError };
});
vi.mock('../../../../src/utils/ssrfGuard', () => ({
safeFetch: safeFetchMock,
SsrfBlockedError: SsrfBlockedErrorMock,
}));
import {
classifyTransportFailure,
declaredLength,
guardDownload,
providerFetch,
statusErrorCode,
type TransportFailure,
} from '../../../../src/nest/doc-sync/providers/provider-http';
/** `TypeError: fetch failed` with the real reason underneath, the way undici reports it. */
function wrapped(cause: Error): Error {
return new TypeError('fetch failed', { cause });
}
function coded(message: string, code: string): Error {
return Object.assign(new Error(message), { code });
}
function named(message: string, name: string): Error {
return Object.assign(new Error(message), { name });
}
class TransportError extends Error {
constructor(readonly failure: TransportFailure) {
super(`transport: ${failure.code}`);
}
}
const OPTIONS = {
timeoutMs: 5000,
allowInsecureTls: false,
onTransportFailure: (failure: TransportFailure) => new TransportError(failure),
};
/** A body that hands out `chunk`-sized pieces until `total` bytes, recording a cancel. */
function source(chunk: number, total: number): { stream: ReadableStream<Uint8Array>; cancelled: () => boolean } {
let sent = 0;
let cancelled = false;
const stream = new ReadableStream<Uint8Array>({
pull(controller) {
if (sent >= total) {
controller.close();
return;
}
sent += chunk;
controller.enqueue(new Uint8Array(chunk).fill(97));
},
cancel() {
cancelled = true;
},
});
return { stream, cancelled: () => cancelled };
}
function download(body: ReadableStream<Uint8Array> | null, headers: Record<string, string> = {}): Response {
return { ok: true, status: 200, headers: new Headers(headers), body } as unknown as Response;
}
class TooLarge extends Error {
constructor(readonly declared: number) {
super('too large');
}
}
const GUARD = {
maxBytes: 4096,
tooLarge: (declared: number) => new TooLarge(declared),
noBody: () => new Error('no body'),
};
async function drain(stream: NodeJS.ReadableStream): Promise<Buffer> {
const chunks: Buffer[] = [];
for await (const chunk of stream) chunks.push(Buffer.from(chunk as Uint8Array));
return Buffer.concat(chunks);
}
beforeEach(() => {
safeFetchMock.mockReset();
});
describe('classifyTransportFailure', () => {
it('PROVIDER-HTTP-001: the SSRF guard is its own verdict and keeps its reason', () => {
expect(classifyTransportFailure(new SsrfBlockedErrorMock('Requests to loopback are not allowed'))).toEqual({
code: 'ssrf_blocked',
detail: 'Requests to loopback are not allowed',
});
});
it.each([
['TimeoutError, as AbortSignal.timeout rejects', named('The operation was aborted due to timeout', 'TimeoutError')],
['AbortError', named('This operation was aborted', 'AbortError')],
['TimeoutError under undici\'s wrapper', wrapped(named('aborted', 'TimeoutError'))],
['UND_ERR_HEADERS_TIMEOUT', wrapped(coded('Headers Timeout Error', 'UND_ERR_HEADERS_TIMEOUT'))],
['UND_ERR_BODY_TIMEOUT', wrapped(coded('Body Timeout Error', 'UND_ERR_BODY_TIMEOUT'))],
['ABORT_ERR', wrapped(coded('The operation was aborted', 'ABORT_ERR'))],
])('PROVIDER-HTTP-002: a timeout, spelled as %s', (_label, error) => {
expect(classifyTransportFailure(error).code).toBe('timeout');
});
it.each([
'DEPTH_ZERO_SELF_SIGNED_CERT',
'SELF_SIGNED_CERT_IN_CHAIN',
// The private-CA case: the chain is fine, the issuer is simply unknown here.
'UNABLE_TO_GET_ISSUER_CERT_LOCALLY',
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
'CERT_HAS_EXPIRED',
'CERT_SIGNATURE_FAILURE',
'ERR_TLS_CERT_ALTNAME_INVALID',
'HOSTNAME_MISMATCH',
// A leaf without serverAuth, signed by a CA this host does trust.
'INVALID_PURPOSE',
'INVALID_CA',
'PATH_LENGTH_EXCEEDED',
])('PROVIDER-HTTP-003: %s is an untrusted certificate', (code) => {
expect(classifyTransportFailure(wrapped(coded('certificate verdict', code))).code).toBe('tls_untrusted');
});
it('PROVIDER-HTTP-004: a protocol failure in the handshake is not offered as a certificate problem', () => {
// https against a plain http port. The self-signed switch cannot fix it.
const error = wrapped(coded('wrong version number', 'ERR_SSL_WRONG_VERSION_NUMBER'));
expect(classifyTransportFailure(error).code).toBe('unreachable');
});
it('PROVIDER-HTTP-005: everything else is a host that cannot be reached, with the whole chain as detail', () => {
const error = wrapped(coded('connect ECONNREFUSED 192.168.1.9:5001', 'ECONNREFUSED'));
expect(classifyTransportFailure(error)).toEqual({
code: 'unreachable',
detail: 'fetch failed: connect ECONNREFUSED 192.168.1.9:5001',
});
expect(classifyTransportFailure('socket hang up')).toEqual({ code: 'unreachable', detail: 'socket hang up' });
});
it('PROVIDER-HTTP-006: the cause chain is followed five levels and no further', () => {
let error: Error = coded('self-signed certificate', 'DEPTH_ZERO_SELF_SIGNED_CERT');
for (let level = 0; level < 4; level++) error = wrapped(error);
expect(classifyTransportFailure(error).code).toBe('tls_untrusted');
expect(classifyTransportFailure(wrapped(error)).code).toBe('unreachable');
});
it('PROVIDER-HTTP-007: a host that never takes the connection is unreachable, not slow', () => {
// undici gives up on the handshake after ten seconds, before the request's
// own timeout does: a NAS that is switched off, or a firewall that drops the SYN.
const error = wrapped(
Object.assign(new Error('Connect Timeout Error'), { name: 'ConnectTimeoutError', code: 'UND_ERR_CONNECT_TIMEOUT' }),
);
expect(classifyTransportFailure(error).code).toBe('unreachable');
});
});
describe('providerFetch', () => {
it('PROVIDER-HTTP-010: hands the request to the SSRF guard with a timeout and the certificate check on', async () => {
const answer = { ok: true, status: 200 };
safeFetchMock.mockResolvedValue(answer);
const body = new URLSearchParams({ a: '1' });
const response = await providerFetch(
'https://nas.example.org:5001/webapi/entry.cgi',
{ method: 'POST', headers: { Accept: 'application/json' }, body, duplex: 'half' } as RequestInit,
OPTIONS,
);
expect(response).toBe(answer);
const [url, init, options] = safeFetchMock.mock.calls[0] as [string, RequestInit & { duplex?: string }, unknown];
expect(url).toBe('https://nas.example.org:5001/webapi/entry.cgi');
expect(init).toMatchObject({ method: 'POST', headers: { Accept: 'application/json' }, body, duplex: 'half' });
expect(init.signal).toBeInstanceOf(AbortSignal);
expect(options).toEqual({ rejectUnauthorized: true });
});
it('PROVIDER-HTTP-011: relaxes the certificate check only when the connection says so', async () => {
safeFetchMock.mockResolvedValue({ ok: true, status: 200 });
await providerFetch('https://nas.example.org', {}, { ...OPTIONS, allowInsecureTls: true });
expect(safeFetchMock.mock.calls[0][2]).toEqual({ rejectUnauthorized: false });
});
it('PROVIDER-HTTP-012: an instance that never answers is cut off at the timeout', async () => {
safeFetchMock.mockImplementation(
(_url: string, init: RequestInit) =>
new Promise((_resolve, reject) => {
init.signal?.addEventListener('abort', () => reject(init.signal?.reason));
}),
);
const pending = providerFetch('https://nas.example.org', {}, { ...OPTIONS, timeoutMs: 20 });
await expect(pending).rejects.toBeInstanceOf(TransportError);
await expect(pending).rejects.toMatchObject({ failure: { code: 'timeout' } });
});
it('PROVIDER-HTTP-013: a refusal becomes the client\'s own error, built from the classified failure', async () => {
safeFetchMock.mockRejectedValue(new SsrfBlockedErrorMock('Requests to link-local addresses are not allowed'));
const onTransportFailure = vi.fn((failure: TransportFailure) => new TransportError(failure));
const pending = providerFetch('http://169.254.169.254/', {}, { ...OPTIONS, onTransportFailure });
await expect(pending).rejects.toMatchObject({
failure: { code: 'ssrf_blocked', detail: 'Requests to link-local addresses are not allowed' },
});
expect(onTransportFailure).toHaveBeenCalledTimes(1);
});
});
describe('statusErrorCode', () => {
it('PROVIDER-HTTP-020: maps the statuses every product answers alike', () => {
const expected: Array<[number, string]> = [
[401, 'unauthorized'],
[403, 'forbidden'],
[404, 'not_found'],
[409, 'conflict'],
[413, 'too_large'],
[415, 'unsupported_type'],
[429, 'rate_limited'],
[507, 'quota_exceeded'],
[500, 'provider_error'],
[418, 'provider_error'],
];
for (const [status, code] of expected) expect(statusErrorCode(status), `http ${status}`).toBe(code);
});
it('PROVIDER-HTTP-021: a client\'s own reading of a status wins, and its fallback answers the rest', () => {
expect(statusErrorCode(412, { 412: 'conflict' })).toBe('conflict');
expect(statusErrorCode(404, { 404: 'scope_missing' })).toBe('scope_missing');
expect(statusErrorCode(418, {}, 'unknown')).toBe('unknown');
expect(statusErrorCode(401, {}, 'unknown')).toBe('unauthorized');
});
});
describe('declaredLength', () => {
it('PROVIDER-HTTP-030: an absent or unreadable header is an unknown size, never zero', () => {
expect(declaredLength(null)).toBeNull();
expect(declaredLength('')).toBeNull();
expect(declaredLength(' ')).toBeNull();
expect(declaredLength('many')).toBeNull();
expect(declaredLength('-1')).toBeNull();
expect(declaredLength('0')).toBe(0);
expect(declaredLength('4096')).toBe(4096);
});
});
describe('guardDownload', () => {
it('PROVIDER-HTTP-040: streams a body within the limit and reports what it announced', async () => {
const { stream } = source(1024, 3072);
const guarded = guardDownload(download(stream, { 'content-length': '3072' }), GUARD);
expect(guarded.size).toBe(3072);
expect((await drain(guarded.body)).length).toBe(3072);
});
it('PROVIDER-HTTP-041: refuses an announced length over the limit before reading, and lets the socket go', () => {
const { stream, cancelled } = source(1024, 8192);
let thrown: unknown;
try {
guardDownload(download(stream, { 'content-length': '8192' }), GUARD);
} catch (error: unknown) {
thrown = error;
}
expect(thrown).toBeInstanceOf(TooLarge);
expect((thrown as TooLarge).declared).toBe(8192);
expect(cancelled()).toBe(true);
});
it('PROVIDER-HTTP-042: a body that announces nothing and runs past the limit fails mid-stream', async () => {
const { stream, cancelled } = source(1024, 64 * 1024);
const guarded = guardDownload(download(stream), GUARD);
expect(guarded.size).toBeNull();
await expect(drain(guarded.body)).rejects.toThrow(/exceeded 4096 bytes/);
await vi.waitFor(() => expect(cancelled()).toBe(true));
});
it('PROVIDER-HTTP-043: a body that lies about its length is held to the limit all the same', async () => {
const { stream } = source(1024, 64 * 1024);
const guarded = guardDownload(download(stream, { 'content-length': '100' }), GUARD);
await expect(drain(guarded.body)).rejects.toThrow(/exceeded/);
});
it('PROVIDER-HTTP-044: a consumer that stops reading releases the source', async () => {
const { stream, cancelled } = source(1024, 64 * 1024);
const guarded = guardDownload(download(stream), { ...GUARD, maxBytes: 1024 * 1024 });
guarded.body.destroy();
await vi.waitFor(() => expect(cancelled()).toBe(true));
});
it('PROVIDER-HTTP-045: an answer without a body is the client\'s own error', () => {
expect(() => guardDownload(download(null, { 'content-length': '10' }), GUARD)).toThrow('no body');
});
});
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,906 @@
/**
* The WebDAV adapter for Nextcloud and OpenCloud: what it puts on the wire, and
* what it makes of what comes back.
*
* The cases here are the ones that cost real debugging against the two
* products, so they are pinned rather than described: Nextcloud's `OC-FileId`
* header is not its `oc:fileid` and a naive adapter re-uploads every file
* forever; `OC-Checksum: SHA256` is fine on Nextcloud and a hard 400 on
* OpenCloud; a PROPFIND answers one propstat block per status and the 404 block
* is not data; an OpenCloud drive describes its own WebDAV URL on a host the
* admin never entered.
*
* `safeFetch` and the capped readers are mocked, for the reason
* dawarich.client.test.ts gives: the SSRF guard does a real DNS lookup and the
* readers want a real stream. The responses are genuine `Response` objects, so
* headers, status and body behave as they do in production.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
const { safeFetchMock } = vi.hoisted(() => ({ safeFetchMock: vi.fn() }));
vi.mock('../../../../src/utils/ssrfGuard', () => ({
safeFetch: safeFetchMock,
SsrfBlockedError: class SsrfBlockedError extends Error {
constructor(message: string) {
super(message);
this.name = 'SsrfBlockedError';
}
},
}));
vi.mock('../../../../src/utils/cappedFetch', async (importOriginal) => ({
...(await importOriginal<typeof import('../../../../src/utils/cappedFetch')>()),
readCappedText: async (res: Response) => {
const text = await res.text();
return { text: text.slice(0, capBytes), truncated: text.length > capBytes };
},
readCappedJson: async (res: Response) => {
try {
return JSON.parse(await res.text()) as unknown;
} catch {
return undefined;
}
},
}));
import { Readable } from 'node:stream';
import { SsrfBlockedError } from '../../../../src/utils/ssrfGuard';
import {
NextcloudDocumentProvider,
OpencloudDocumentProvider,
flavorOf,
} from '../../../../src/nest/doc-sync/providers/webdav.provider';
import { WebdavClient } from '../../../../src/nest/doc-sync/providers/webdav.client';
import type {
DocumentConnectionRef,
DocumentScopeRef,
PushRequest,
} from '../../../../src/nest/doc-sync/document-provider';
/** Raised by the truncation test only; everything else stays under the cap. */
let capBytes = 1_000_000;
const NC_ORIGIN = 'https://cloud.example.org';
const OC_ORIGIN = 'https://opencloud.example.org';
const NC_ROOT = '/remote.php/dav/files/admin';
const OC_DRIVE = 'storage-1$space-9';
const OC_ROOT = `/dav/spaces/${OC_DRIVE}`;
const ncConn: DocumentConnectionRef = {
connectionId: 7,
createdAt: '2026-09-01 08:00:00',
ownerId: 3,
baseUrl: NC_ORIGIN,
secrets: { app_password: 'app-pw' },
settings: { login_name: 'admin', base_path: '/TREK' },
allowInsecureTls: false,
};
const ocConn: DocumentConnectionRef = {
connectionId: 8,
createdAt: '2026-09-01 08:00:00',
ownerId: 3,
baseUrl: OC_ORIGIN,
secrets: { app_token: 'app-token' },
settings: { username: 'admin' },
allowInsecureTls: true,
};
const ncScope: DocumentScopeRef = {
linkId: 1,
tripId: 42,
scopeKey: 'fileid:60',
remoteRootId: '60',
remoteRootPath: '/TREK/trip 42',
cursor: null,
};
const ocScope: DocumentScopeRef = {
linkId: 2,
tripId: 42,
scopeKey: `drive:${OC_DRIVE}`,
remoteRootId: OC_DRIVE,
remoteRootPath: `${OC_ORIGIN}${OC_ROOT}`,
cursor: null,
};
interface Recorded {
url: string;
method: string;
headers: Record<string, string>;
body: string | undefined;
rejectUnauthorized: boolean | undefined;
}
let calls: Recorded[] = [];
/** `METHOD /path` → the answer, or a function of the request. */
type Answer = Response | ((req: Recorded) => Response);
let routes: Map<string, Answer[]>;
function route(key: string, ...answers: Answer[]): void {
routes.set(key, answers);
}
function reply(body: string | null, init: ResponseInit = {}): Response {
return new Response(body, init);
}
/** A PROPFIND answer built from the props each entry declares. */
function multistatus(
entries: Array<{
href: string;
collection?: boolean;
etag?: string;
fileid?: string;
size?: number;
type?: string;
modified?: string;
checksums?: string;
}>,
): Response {
const body = entries
.map((entry) => {
const found: string[] = [];
const missing: string[] = [];
found.push(`<d:getetag>${entry.etag ?? '"etag"'}</d:getetag>`);
found.push(`<d:getlastmodified>${entry.modified ?? 'Fri, 18 Sep 2026 15:15:54 GMT'}</d:getlastmodified>`);
found.push(entry.collection ? '<d:resourcetype><d:collection/></d:resourcetype>' : '<d:resourcetype/>');
if (entry.fileid) found.push(`<oc:fileid>${entry.fileid}</oc:fileid>`);
if (entry.size === undefined) missing.push('<d:getcontentlength/>');
else found.push(`<d:getcontentlength>${entry.size}</d:getcontentlength>`);
if (entry.type === undefined) missing.push('<d:getcontenttype/>');
else found.push(`<d:getcontenttype>${entry.type}</d:getcontenttype>`);
if (entry.checksums === undefined) missing.push('<oc:checksums/>');
else found.push(`<oc:checksums><oc:checksum>${entry.checksums}</oc:checksum></oc:checksums>`);
const blocks = [`<d:propstat><d:prop>${found.join('')}</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat>`];
if (missing.length > 0) {
blocks.push(
`<d:propstat><d:prop>${missing.join('')}</d:prop><d:status>HTTP/1.1 404 Not Found</d:status></d:propstat>`,
);
}
return `<d:response><d:href>${entry.href}</d:href>${blocks.join('')}</d:response>`;
})
.join('');
return reply(
`<?xml version="1.0"?><d:multistatus xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns">${body}</d:multistatus>`,
{ status: 207, headers: { 'content-type': 'application/xml' } },
);
}
function drive(id: string, name: string, driveType: string, webDavUrl: string): unknown {
return { id, name, driveType, root: { id, webDavUrl } };
}
function requests(method?: string): Recorded[] {
return method ? calls.filter((call) => call.method === method) : calls;
}
function pathOf(call: Recorded): string {
return new URL(call.url).pathname;
}
function pushRequest(overrides: Partial<PushRequest> = {}): PushRequest {
return {
body: Readable.from([Buffer.from('bytes')]),
fileName: 'reiseplan.pdf',
mimeType: 'application/pdf',
size: 5,
sha256: 'a'.repeat(64),
mtimeSeconds: 1_700_000_000,
trekDocUid: 'doc-1',
trekTripUid: 'trip-1',
...overrides,
};
}
let nextcloud: NextcloudDocumentProvider;
let opencloud: OpencloudDocumentProvider;
beforeEach(() => {
capBytes = 1_000_000;
calls = [];
routes = new Map();
const client = new WebdavClient();
nextcloud = new NextcloudDocumentProvider(client);
opencloud = new OpencloudDocumentProvider(client);
safeFetchMock.mockImplementation(
async (url: string, init: RequestInit, options?: { rejectUnauthorized?: boolean }) => {
const headers: Record<string, string> = {};
new Headers(init.headers).forEach((value, key) => {
headers[key] = value;
});
const record: Recorded = {
url,
method: init.method ?? 'GET',
headers,
body: typeof init.body === 'string' ? init.body : undefined,
rejectUnauthorized: options?.rejectUnauthorized,
};
calls.push(record);
const key = `${record.method} ${new URL(url).pathname}`;
const queued = routes.get(key);
if (!queued || queued.length === 0) {
return reply('<d:error/>', { status: 404 });
}
const answer = queued.length > 1 ? queued.shift() : queued[0];
if (!answer) return reply(null, { status: 404 });
return typeof answer === 'function' ? answer(record) : answer.clone();
},
);
});
describe('flavour detection', () => {
it('reads the product off the connection rather than off the registered id', () => {
expect(flavorOf(ncConn)).toBe('nextcloud');
expect(flavorOf(ocConn)).toBe('opencloud');
expect(flavorOf({ ...ncConn, settings: {}, secrets: {} })).toBeNull();
});
it('answers the same capabilities whichever class asks, because the connection decides', () => {
expect(nextcloud.capabilities(ocConn).push).toBe('none');
expect(opencloud.capabilities(ncConn).push).toBe('webhook-manual');
});
it('never claims a content hash it cannot stand behind', () => {
const caps = nextcloud.capabilities(ncConn);
expect(caps.contentHashInListing).toBe(false);
expect(caps.stableId).toBe(true);
expect(caps.remoteTrash).toBe(true);
expect(caps.replaceInPlace).toBe(true);
});
});
describe('probe', () => {
it('checks the WebDAV home and upgrades push when the account may manage webhooks', async () => {
route(`PROPFIND ${NC_ROOT}`, multistatus([{ href: NC_ROOT, collection: true, fileid: '1' }]));
route('GET /ocs/v2.php/apps/webhook_listeners/api/v1/webhooks', reply('{"ocs":{"data":[]}}'));
const result = await nextcloud.probe(ncConn);
expect(result).toMatchObject({ success: true, data: { account: 'admin' } });
expect(result.success && result.data.capabilities.push).toBe('webhook-self-registered');
expect(calls[0].headers.authorization).toBe(`Basic ${Buffer.from('admin:app-pw').toString('base64')}`);
});
it('keeps push at webhook-manual when the webhook endpoint is closed to this account', async () => {
route(`PROPFIND ${NC_ROOT}`, multistatus([{ href: NC_ROOT, collection: true, fileid: '1' }]));
route('GET /ocs/v2.php/apps/webhook_listeners/api/v1/webhooks', reply('{}', { status: 403 }));
const result = await nextcloud.probe(ncConn);
expect(result.success && result.data.capabilities.push).toBe('webhook-manual');
});
it('separates a wrong login name from a wrong password', async () => {
route(`PROPFIND ${NC_ROOT}`, reply('<d:error/>', { status: 404 }));
const missing = await nextcloud.probe(ncConn);
expect(missing).toMatchObject({ success: false, error: { code: 'not_found' } });
expect(missing.success === false && missing.error.detail).toContain('admin');
calls = [];
route(`PROPFIND ${NC_ROOT}`, reply('<d:error/>', { status: 401 }));
expect(await nextcloud.probe(ncConn)).toMatchObject({ success: false, error: { code: 'unauthorized' } });
});
it('names the OpenCloud account from Graph and passes the TLS opt-in through', async () => {
route('GET /graph/v1.0/me', reply('{"onPremisesSamAccountName":"admin","displayName":"Admin"}'));
const result = await opencloud.probe(ocConn);
expect(result).toMatchObject({ success: true, data: { account: 'admin' } });
expect(result.success && result.data.capabilities.push).toBe('none');
expect(calls[0].rejectUnauthorized).toBe(false);
});
it('refuses a half-filled connection without going near the network', async () => {
const result = await nextcloud.probe({ ...ncConn, secrets: {}, settings: { login_name: 'admin' } });
expect(result).toMatchObject({ success: false, error: { code: 'unauthorized' } });
expect(calls).toHaveLength(0);
});
});
describe('scopes', () => {
it('offers the folders under the base path and keys them by oc:fileid', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK`,
multistatus([
{ href: `${NC_ROOT}/TREK/`, collection: true, fileid: '59' },
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60' },
{ href: `${NC_ROOT}/TREK/notes.txt`, fileid: '61', size: 3, type: 'text/plain' },
]),
);
const result = await nextcloud.listScopes(ncConn);
expect(result).toEqual({
success: true,
data: [{ scopeKey: 'fileid:60', label: 'trip 42', remoteRootId: '60', remoteRootPath: '/TREK/trip 42' }],
});
});
it('answers an empty picker rather than an error when the base folder does not exist yet', async () => {
route(`PROPFIND ${NC_ROOT}/TREK`, reply('<d:error/>', { status: 404 }));
expect(await nextcloud.listScopes(ncConn)).toEqual({ success: true, data: [] });
});
it('filters the picker by the query', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK`,
multistatus([
{ href: `${NC_ROOT}/TREK/`, collection: true, fileid: '59' },
{ href: `${NC_ROOT}/TREK/Japan/`, collection: true, fileid: '60' },
{ href: `${NC_ROOT}/TREK/Peru/`, collection: true, fileid: '61' },
]),
);
const result = await nextcloud.listScopes(ncConn, 'jap');
expect(result.success && result.data.map((option) => option.label)).toEqual(['Japan']);
});
it('re-bases an OpenCloud space on the entered origin and drops the virtual share drive', async () => {
route(
'GET /graph/v1.0/me/drives',
reply(
JSON.stringify({
value: [
drive('shares$1', 'Shares', 'virtual', 'https://internal.lan:9200/dav/spaces/shares$1'),
drive(OC_DRIVE, 'TREK Trip 42', 'project', `https://internal.lan:9200${OC_ROOT}`),
],
}),
),
);
const result = await opencloud.listScopes(ocConn);
expect(result).toEqual({
success: true,
data: [
{
scopeKey: `drive:${OC_DRIVE}`,
label: 'TREK Trip 42',
remoteRootId: OC_DRIVE,
remoteRootPath: `${OC_ORIGIN}${OC_ROOT}`,
},
],
});
});
it('creates the base folder when it is missing and reads the new id back', async () => {
route(
`MKCOL ${NC_ROOT}/TREK/Japan%202026`,
reply(null, { status: 409 }),
reply(null, { status: 201 }),
);
route(`MKCOL ${NC_ROOT}/TREK`, reply(null, { status: 201 }));
route(
`PROPFIND ${NC_ROOT}/TREK/Japan%202026`,
multistatus([{ href: `${NC_ROOT}/TREK/Japan%202026/`, collection: true, fileid: '73' }]),
);
const result = await nextcloud.createScope(ncConn, 'Japan 2026');
expect(result).toMatchObject({ success: true, data: { scopeKey: 'fileid:73', remoteRootPath: '/TREK/Japan 2026' } });
expect(requests('MKCOL').map(pathOf)).toEqual([
`${NC_ROOT}/TREK/Japan%202026`,
`${NC_ROOT}/TREK`,
`${NC_ROOT}/TREK/Japan%202026`,
]);
});
it('refuses to hand back a folder somebody else already made', async () => {
route(`MKCOL ${NC_ROOT}/TREK/Japan`, reply(null, { status: 405 }));
expect(await nextcloud.createScope(ncConn, 'Japan')).toMatchObject({
success: false,
error: { code: 'conflict' },
});
});
it('replaces the characters Nextcloud refuses instead of failing the whole folder', async () => {
route(`MKCOL ${NC_ROOT}/TREK/Japan-%202026`, reply(null, { status: 201 }));
route(
`PROPFIND ${NC_ROOT}/TREK/Japan-%202026`,
multistatus([{ href: `${NC_ROOT}/TREK/Japan-%202026/`, collection: true, fileid: '74' }]),
);
const result = await nextcloud.createScope(ncConn, 'Japan: 2026');
expect(result.success && result.data.label).toBe('Japan- 2026');
});
it('creates an OpenCloud space as a project drive', async () => {
route(
'POST /graph/v1.0/drives',
reply(JSON.stringify(drive('storage-1$new', 'Japan', 'project', `https://internal.lan:9200/dav/spaces/storage-1$new`)), {
status: 201,
}),
);
const result = await opencloud.createScope(ocConn, 'Japan');
expect(result).toMatchObject({ success: true, data: { scopeKey: 'drive:storage-1$new' } });
expect(JSON.parse(calls[0].body ?? '{}')).toEqual({ name: 'Japan', driveType: 'project' });
});
it('calls a folder with the same path but a different id scope_missing', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042`,
multistatus([{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '99' }]),
);
const result = await nextcloud.resolveScope(ncConn, ncScope);
expect(result).toMatchObject({ success: false, error: { code: 'scope_missing' } });
expect(result.success === false && result.error.detail).toContain('99');
});
it('reports a deleted space as scope_missing rather than as a broken connection', async () => {
route(`GET /graph/v1.0/drives/${encodeURIComponent(OC_DRIVE)}`, reply('{"error":{}}', { status: 404 }));
expect(await opencloud.resolveScope(ocConn, ocScope)).toMatchObject({
success: false,
error: { code: 'scope_missing' },
});
});
it('rejects a scope key that belongs to another provider', async () => {
const result = await nextcloud.list(ncConn, { ...ncScope, scopeKey: 'drive:storage$x' });
expect(result).toMatchObject({ success: false, error: { code: 'scope_missing' } });
expect(calls).toHaveLength(0);
});
});
describe('list', () => {
const listing = () =>
multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root-1"' },
{
href: `${NC_ROOT}/TREK/trip%2042/sub/`,
collection: true,
fileid: '62',
etag: '"sub-1"',
},
{
href: `${NC_ROOT}/TREK/trip%2042/pass.pdf`,
fileid: '61',
etag: '"doc-1"',
size: 1234,
type: 'application/pdf',
modified: 'Tue, 14 Nov 2023 22:13:20 GMT',
checksums: `SHA256:${'b'.repeat(64)}`,
},
]);
it('enumerates documents only, never the root and never a sub-folder', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, listing());
const result = await nextcloud.list(ncConn, ncScope);
expect(result.success && result.data.documents).toEqual([
{
remoteId: '61',
name: 'pass.pdf',
size: 1234,
mimeType: 'application/pdf',
remoteVersion: '"doc-1"',
contentHash: 'b'.repeat(64),
remoteModifiedAt: '2023-11-14T22:13:20.000Z',
isDeleted: false,
},
]);
expect(result.success && result.data.cursor).toBe('"root-1"');
expect(result.success && result.data.truncated).toBe(false);
});
it('skips the walk entirely when the root ETag has not moved', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root-1"' },
]));
const result = await nextcloud.list(ncConn, { ...ncScope, cursor: '"root-1"' });
expect(result).toEqual({
success: true,
data: { documents: [], cursor: '"root-1"', cursorUnchanged: true, truncated: false },
});
expect(calls).toHaveLength(1);
expect(calls[0].headers.depth).toBe('0');
});
it('walks after the cursor probe shows a different root ETag', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042`,
multistatus([{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root-2"' }]),
listing(),
);
const result = await nextcloud.list(ncConn, { ...ncScope, cursor: '"root-1"' });
expect(result.success && result.data.cursorUnchanged).toBe(false);
expect(calls.map((call) => call.headers.depth)).toEqual(['0', '1']);
});
it('reads a checksum only when it really is a SHA256, so OpenCloud reports none', async () => {
route(
`PROPFIND ${OC_ROOT}`,
multistatus([
{ href: `${OC_ROOT}/`, collection: true, fileid: `${OC_DRIVE}!root`, etag: '"root"' },
{
href: `${OC_ROOT}/pass.pdf`,
fileid: `${OC_DRIVE}!doc`,
etag: '"doc"',
size: 10,
type: 'application/pdf',
checksums: 'SHA1:2117 MD5:eebe ADLER32:5f9f',
},
]),
);
const result = await opencloud.list(ocConn, ocScope);
expect(result.success && result.data.documents[0]).toMatchObject({
remoteId: `${OC_DRIVE}!doc`,
contentHash: null,
});
});
it('reports a listing cut off at the size cap instead of failing the run', async () => {
capBytes = 620;
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, listing());
const result = await nextcloud.list(ncConn, ncScope);
expect(result.success && result.data.truncated).toBe(true);
expect(result.success && result.data.documents.length).toBeLessThanOrEqual(1);
});
it('turns a vanished folder into scope_missing rather than an empty trip', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, reply('<d:error/>', { status: 404 }));
expect(await nextcloud.list(ncConn, ncScope)).toMatchObject({
success: false,
error: { code: 'scope_missing' },
});
});
it('keeps a document without an oc:fileid addressable instead of dropping it', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042`,
multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root"' },
{ href: `${NC_ROOT}/TREK/trip%2042/pass.pdf`, etag: '"doc"', size: 1, type: 'application/pdf' },
]),
);
const result = await nextcloud.list(ncConn, ncScope);
expect(result.success && result.data.documents[0].remoteId).toBe('path:pass.pdf');
});
});
describe('push', () => {
const uploaded = (req: Recorded) =>
reply(null, {
status: 201,
headers: { 'oc-fileid': '00000062oc9fy1g5e2cj', 'oc-etag': `"${req.headers['x-oc-mtime']}"` },
});
it('pins the modification time and the checksum, and normalises the id Nextcloud answers with', async () => {
route(`PUT ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, uploaded);
route(`PROPPATCH ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, etag: '"x"' },
]));
const result = await nextcloud.push(ncConn, ncScope, pushRequest());
const put = requests('PUT')[0];
expect(put.headers['x-oc-mtime']).toBe('1700000000');
expect(put.headers['oc-checksum']).toBe(`SHA256:${'a'.repeat(64)}`);
expect(put.headers['content-length']).toBe('5');
// The header spells the id 00000062oc9fy1g5e2cj; the listing spells it 62,
// and a mismatch here is a duplicate document on every run.
expect(result).toMatchObject({
success: true,
data: { remoteId: '62', remoteModifiedAt: '2023-11-14T22:13:20.000Z', deduplicated: false },
});
});
it('never sends a SHA256 checksum to OpenCloud, which answers 400 to one', async () => {
route(`PUT ${OC_ROOT}/reiseplan.pdf`, reply(null, { status: 201, headers: { 'oc-fileid': 'd1', 'oc-etag': '"e1"' } }));
route(`PROPPATCH ${OC_ROOT}/reiseplan.pdf`, multistatus([{ href: `${OC_ROOT}/reiseplan.pdf`, etag: '"e1"' }]));
await opencloud.push(ocConn, ocScope, pushRequest());
expect(requests('PUT')[0].headers['oc-checksum']).toBeUndefined();
expect(requests('PUT')[0].headers['x-oc-mtime']).toBe('1700000000');
});
it('writes the TREK anchors as dead properties without letting a refusal undo the upload', async () => {
route(`PUT ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, uploaded);
route(`PROPPATCH ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, reply('<d:error/>', { status: 403 }));
const result = await nextcloud.push(ncConn, ncScope, pushRequest());
expect(result.success).toBe(true);
expect(requests('PROPPATCH')[0].body).toContain('doc-1');
expect(requests('PROPPATCH')[0].body).toContain('trip-1');
});
it('asks for the id and version a proxy stripped from the response', async () => {
route(`PUT ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, reply(null, { status: 201 }));
route(`PROPPATCH ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, etag: '"x"' },
]));
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`,
multistatus([{ href: `${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, fileid: '62', etag: '"stored"', size: 5 }]),
);
const result = await nextcloud.push(ncConn, ncScope, pushRequest());
expect(result).toMatchObject({ success: true, data: { remoteId: '62', remoteVersion: '"stored"' } });
});
it('sends If-Match when the core knows which version it is replacing, and reports 412 as a conflict', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042`,
multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root"' },
{ href: `${NC_ROOT}/TREK/trip%2042/pass.pdf`, fileid: '61', etag: '"doc-1"', size: 1, type: 'application/pdf' },
]),
);
route(`PUT ${NC_ROOT}/TREK/trip%2042/pass.pdf`, reply('<d:error/>', { status: 412 }));
const result = await nextcloud.push(
ncConn,
ncScope,
pushRequest({ remoteId: '61', expectedRemoteVersion: '"doc-1"', fileName: 'pass.pdf' }),
);
expect(requests('PUT')[0].headers['if-match']).toBe('"doc-1"');
expect(result).toMatchObject({ success: false, error: { code: 'conflict', status: 412 } });
});
it('refuses to recreate a document that disappeared upstream', async () => {
route(
`PROPFIND ${NC_ROOT}/TREK/trip%2042`,
multistatus([{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root"' }]),
);
const result = await nextcloud.push(ncConn, ncScope, pushRequest({ remoteId: '61' }));
expect(result).toMatchObject({ success: false, error: { code: 'not_found' } });
expect(requests('PUT')).toHaveLength(0);
});
it('maps a full quota to quota_exceeded rather than a generic failure', async () => {
route(`PUT ${NC_ROOT}/TREK/trip%2042/reiseplan.pdf`, reply('<d:error/>', { status: 507 }));
expect(await nextcloud.push(ncConn, ncScope, pushRequest())).toMatchObject({
success: false,
error: { code: 'quota_exceeded' },
});
});
});
describe('fetch, rename and trash', () => {
const folder = () =>
multistatus([
{ href: `${NC_ROOT}/TREK/trip%2042/`, collection: true, fileid: '60', etag: '"root"' },
{
href: `${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`,
fileid: '61',
etag: '"doc-1"',
size: 5,
type: 'application/pdf',
},
]);
it('streams a document back and reports the version it came with', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(
`GET ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`,
reply('the bytes', { headers: { 'content-length': '9', 'content-type': 'application/pdf', etag: '"doc-1"' } }),
);
const result = await nextcloud.fetch(ncConn, ncScope, '61');
expect(result.success).toBe(true);
if (!result.success) return;
const chunks: Buffer[] = [];
for await (const chunk of result.data.body) chunks.push(Buffer.from(chunk));
expect(Buffer.concat(chunks).toString()).toBe('the bytes');
expect(result.data).toMatchObject({ size: 9, mimeType: 'application/pdf', remoteVersion: '"doc-1"' });
});
it('re-uses the listing it already walked instead of re-walking per document', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(`GET ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`, reply('x', { headers: { etag: '"doc-1"' } }));
await nextcloud.list(ncConn, ncScope);
await nextcloud.fetch(ncConn, ncScope, '61');
expect(requests('PROPFIND')).toHaveLength(1);
});
it('refuses a download announcing more than TREK transfers, and one that arrives without a body', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(
`GET ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`,
reply('x', { headers: { 'content-length': String(3 * 1024 ** 3) } }),
reply(null, { status: 200 }),
);
expect(await nextcloud.fetch(ncConn, ncScope, '61')).toMatchObject({
success: false,
error: { code: 'too_large', status: 200 },
});
expect(await nextcloud.fetch(ncConn, ncScope, '61')).toMatchObject({
success: false,
error: { code: 'provider_error', status: 200 },
});
});
it('says not_found for an id the folder no longer holds', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
expect(await nextcloud.fetch(ncConn, ncScope, '999')).toMatchObject({
success: false,
error: { code: 'not_found' },
});
});
it('renames inside the same folder and refuses to overwrite a namesake', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(`MOVE ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`, reply(null, { status: 201, headers: { 'oc-etag': '"doc-2"' } }));
const result = await nextcloud.rename(ncConn, ncScope, '61', 'Reisepass 2026.pdf');
const move = requests('MOVE')[0];
expect(move.headers.destination).toBe(`${NC_ORIGIN}${NC_ROOT}/TREK/trip%2042/Reisepass%202026.pdf`);
expect(move.headers.overwrite).toBe('F');
expect(result).toEqual({ success: true, data: { remoteVersion: '"doc-2"' } });
});
it('reports a rename onto an existing name as a conflict', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(`MOVE ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`, reply('<d:error/>', { status: 412 }));
expect(await nextcloud.rename(ncConn, ncScope, '61', 'anders.pdf')).toMatchObject({
success: false,
error: { code: 'conflict' },
});
});
it('deletes into the recycle bin and treats an already-deleted document as done', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, folder());
route(`DELETE ${NC_ROOT}/TREK/trip%2042/pass%20scan.pdf`, reply(null, { status: 204 }), reply('<d:error/>', { status: 404 }));
expect(await nextcloud.trash(ncConn, ncScope, '61')).toEqual({ success: true, data: undefined });
expect(await nextcloud.trash(ncConn, ncScope, '61')).toEqual({ success: true, data: undefined });
});
});
describe('webhooks', () => {
const OCS = '/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks';
it('registers one unfiltered subscription per event, form-encoded, with the secret as a header', async () => {
route(`GET ${OCS}`, reply('{"ocs":{"data":[]}}'));
let next = 10;
route(`POST ${OCS}`, () => reply(JSON.stringify({ ocs: { data: { id: next++ } } })));
const result = await nextcloud.registerWebhook(ncConn, ncScope, 'https://trek.example/hook', 's3cret');
expect(result).toEqual({ success: true, data: { subscriptionId: '10,11,12,13' } });
const bodies = requests('POST').map((call) => new URLSearchParams(call.body ?? ''));
expect(requests('POST')[0].headers['content-type']).toBe('application/x-www-form-urlencoded');
expect(bodies[0].get('httpMethod')).toBe('POST');
expect(bodies[0].get('uri')).toBe('https://trek.example/hook');
expect(bodies[0].get('authMethod')).toBe('header');
// The header the endpoint actually reads. It used to be registered as
// `X-TREK-Docsync-Signature` while doc-sync-webhook.controller looks for
// `x-trek-docsync-secret`, so every webhook Nextcloud sent was dropped
// without a trace: the subscription existed and the calls arrived.
expect(bodies[0].get('authData[x-trek-docsync-secret]')).toBe('s3cret');
// A $regex filter is not merely ignored upstream: it throws inside the
// listener and stops delivery for every other webhook on that event.
expect(bodies.every((body) => [...body.keys()].every((key) => !key.startsWith('eventFilter')))).toBe(true);
expect(bodies.map((body) => body.get('event'))).toEqual([
'OCP\\Files\\Events\\Node\\NodeCreatedEvent',
'OCP\\Files\\Events\\Node\\NodeWrittenEvent',
'OCP\\Files\\Events\\Node\\NodeDeletedEvent',
'OCP\\Files\\Events\\Node\\NodeRenamedEvent',
]);
});
it('answers a non-admin account with forbidden and leaves the link polling', async () => {
route(`GET ${OCS}`, reply('{}', { status: 403 }));
const result = await nextcloud.registerWebhook(ncConn, ncScope, 'https://trek.example/hook', 's');
expect(result).toMatchObject({ success: false, error: { code: 'forbidden' } });
expect(requests('POST')).toHaveLength(0);
});
it('rolls back the subscriptions it already made when a later one fails', async () => {
route(`GET ${OCS}`, reply('{"ocs":{"data":[]}}'));
route(
`POST ${OCS}`,
reply(JSON.stringify({ ocs: { data: { id: 10 } } })),
reply(JSON.stringify({ ocs: { data: { id: 11 } } })),
reply('<ocs/>', { status: 500 }),
);
route(`DELETE ${OCS}/10`, reply(null, { status: 200 }));
route(`DELETE ${OCS}/11`, reply(null, { status: 200 }));
const result = await nextcloud.registerWebhook(ncConn, ncScope, 'https://trek.example/hook', 's');
expect(result).toMatchObject({ success: false, error: { code: 'provider_error' } });
expect(requests('DELETE').map(pathOf)).toEqual([`${OCS}/10`, `${OCS}/11`]);
});
it('tells an OpenCloud admin there is nothing to subscribe to instead of failing the connection', async () => {
const result = await opencloud.registerWebhook(ocConn, ocScope, 'https://trek.example/hook', 's');
expect(result).toMatchObject({ success: false, error: { code: 'forbidden' } });
expect(calls).toHaveLength(0);
});
it('removes every subscription the id list names, tolerating one that is already gone', async () => {
route(`DELETE ${OCS}/10`, reply(null, { status: 200 }));
route(`DELETE ${OCS}/11`, reply('<ocs/>', { status: 404 }));
expect(await nextcloud.unregisterWebhook(ncConn, '10,11')).toEqual({ success: true, data: undefined });
expect(requests('DELETE')).toHaveLength(2);
});
});
describe('failure classification', () => {
it('separates a timeout, an untrusted certificate and a blocked host', async () => {
const timeout = Object.assign(new TypeError('fetch failed'), {
cause: Object.assign(new Error('The operation was aborted'), { name: 'TimeoutError' }),
});
safeFetchMock.mockRejectedValueOnce(timeout);
expect(await nextcloud.probe(ncConn)).toMatchObject({ success: false, error: { code: 'timeout' } });
const tls = Object.assign(new TypeError('fetch failed'), {
cause: Object.assign(new Error('self-signed certificate'), { code: 'DEPTH_ZERO_SELF_SIGNED_CERT' }),
});
safeFetchMock.mockRejectedValueOnce(tls);
expect(await nextcloud.probe(ncConn)).toMatchObject({ success: false, error: { code: 'tls_untrusted' } });
safeFetchMock.mockRejectedValueOnce(new SsrfBlockedError('Requests to loopback are not allowed'));
expect(await nextcloud.probe(ncConn)).toMatchObject({ success: false, error: { code: 'ssrf_blocked' } });
safeFetchMock.mockRejectedValueOnce(new TypeError('fetch failed'));
expect(await nextcloud.probe(ncConn)).toMatchObject({ success: false, error: { code: 'unreachable' } });
});
it('keeps the upstream text in detail and out of the code', async () => {
route(`PROPFIND ${NC_ROOT}`, reply('<s:message>Username or password was incorrect</s:message>', { status: 401 }));
const result = await nextcloud.probe(ncConn);
expect(result).toMatchObject({ success: false, error: { code: 'unauthorized' } });
expect(result.success === false && result.error.detail).toContain('incorrect');
});
it('refuses a body that is not a multistatus instead of reading it as an empty folder', async () => {
route(`PROPFIND ${NC_ROOT}/TREK/trip%2042`, reply('<html><body>Please log in</body></html>', { status: 207 }));
expect(await nextcloud.list(ncConn, ncScope)).toMatchObject({
success: false,
error: { code: 'provider_error' },
});
});
});
@@ -0,0 +1,118 @@
/**
* RoadtripPlanService against the worker's real SQLite schema.
*
* The planning SELECT is the one place the server reads a visit's times for the road
* trip, and the tests beside the MCP tools hand the service its rows by hand, so a
* column the statement never selects would go unnoticed there. These run the statement.
*/
import { db } from '../../../src/db/database';
import { DatabaseService } from '../../../src/nest/database/database.service';
import { RoadtripPlanService } from '../../../src/nest/roadtrip/roadtrip-plan.service';
import { createDay, createDayAssignment, createPlace, createTrip, createUser } from '../../helpers/factories';
import { resetTestDb } from '../../helpers/test-db';
import { beforeEach, describe, expect, it, vi } from 'vitest';
/** Every leg an hour and 60 km, whatever it joins. */
function hourlyRouter() {
return {
profiles: () => ['driving'],
route: vi.fn(async (_user: number, _trip: number, _day: number, points: { lat: number; lng: number }[]) => ({
parts: points.slice(1).map(() => ({ distance: 60000, duration: 3600 })),
avoidMissed: [],
leg: {
line: points.map((p) => [p.lat, p.lng]),
vias: [],
seg: {
from: [points[0].lat, points[0].lng],
to: [points[points.length - 1].lat, points[points.length - 1].lng],
mid: [points[0].lat, points[0].lng],
distance: 60000 * (points.length - 1),
duration: 3600 * (points.length - 1),
mode: 'driving',
distanceText: '',
drivingText: '',
walkingText: '',
},
},
})),
};
}
function setup() {
const { user } = createUser(db);
const trip = createTrip(db, user.id);
const day = createDay(db, trip.id);
const visits = ['Hamburg', 'Lueneburg', 'Celle'].map((name, i) => {
const place = createPlace(db, trip.id, { name, lat: 53 - i * 0.3, lng: 10 });
db.prepare('UPDATE places SET duration_minutes = ? WHERE id = ?').run(i === 0 ? 0 : 30, place.id);
return createDayAssignment(db, day.id, place.id);
});
db.prepare("UPDATE day_assignments SET assignment_time = '09:00' WHERE id = ?").run(visits[0].id);
const plans = new RoadtripPlanService(
new DatabaseService(db),
{ getUserSettings: () => ({}) } as never,
{ read: () => ({}) } as never,
hourlyRouter() as never,
{ listForTrip: () => [], tracksForTrip: () => [] } as never,
{ list: () => [] } as never,
);
return { user, trip, visits, plans };
}
beforeEach(() => {
resetTestDb(db);
});
describe('a visit end time on the road trip', () => {
it('is read from the visit, and from the place when the visit has none', () => {
const { user, trip, visits, plans } = setup();
db.prepare("UPDATE day_assignments SET assignment_end_time = '14:00' WHERE id = ?").run(visits[1].id);
db.prepare(
"UPDATE places SET end_time = '18:00' WHERE id = (SELECT place_id FROM day_assignments WHERE id = ?)",
).run(visits[2].id);
const context = plans.context(trip.id, user.id);
expect(context.visits.map((v) => [v.name, v.end_time])).toEqual([
['Hamburg', null],
['Lueneburg', '14:00'],
['Celle', '18:00'],
]);
});
it('is when the drive leaves the stop, in place of its stay', async () => {
const { user, trip, visits, plans } = setup();
db.prepare("UPDATE day_assignments SET assignment_end_time = '14:00' WHERE id = ?").run(visits[1].id);
const { calculated } = await plans.calculate(trip.id, user.id);
const day = calculated.days[0];
expect(day.stops[1]).toMatchObject({ name: 'Lueneburg', leaveAt: '14:00' });
expect(day.schedule.entries.map((e) => [e.arrival, e.departure])).toEqual([
['09:00', '09:00'],
['10:00', '14:00'],
['15:00', '15:30'],
]);
expect(day.schedule.warnings).toEqual([]);
});
it('is reported when the drive gets there after it', async () => {
const { user, trip, visits, plans } = setup();
db.prepare("UPDATE day_assignments SET assignment_end_time = '09:30' WHERE id = ?").run(visits[1].id);
const { calculated } = await plans.calculate(trip.id, user.id);
expect(calculated.days[0].schedule.warnings).toEqual([{ index: 1, code: 'missedLeave', minutes: 30 }]);
expect(calculated.days[0].schedule.entries[1]).toMatchObject({ arrival: '10:00', departure: '10:00' });
});
it('leaves a stop without one to its stay', async () => {
const { user, trip, plans } = setup();
const { calculated } = await plans.calculate(trip.id, user.id);
expect(calculated.days[0].schedule.entries[1]).toMatchObject({ arrival: '10:00', departure: '10:30' });
expect(calculated.days[0].stops[1].leaveAt).toBeNull();
});
});
@@ -176,6 +176,24 @@ describe('browser-independent roadtrip calculation', () => {
const plan = await s.plans.calculate(10, 5);
expect(plan.omittedVisits).toEqual([2]);
});
it('hands calculate_roadtrip the end time of a visit as the moment the drive leaves it', async () => {
const s = setup();
// Stop 1 is left at 07:30 and the road takes two hours, so stop 2 is reached half an
// hour after the 09:00 it was meant to be left at. Stop 3 is left at noon, whatever
// its stay says.
Object.assign(s.visits[1], { end_time: '09:00' });
Object.assign(s.visits[2], { end_time: '12:00' });
const mcp = new RoadtripPlanningMcp(s.plans, {} as never, {} as never);
const answer = await mcp.calculate(
{ tripId: 10, includeGeometry: false, settings: { roadtrip_day_start: '', roadtrip_day_end: '' } },
ctx,
);
const body = JSON.parse(answer.content[0].text as string);
const [day] = body.days;
expect(day.stops.map((stop: { leaveAt: string | null }) => stop.leaveAt)).toEqual([undefined, '09:00', '12:00']);
expect(day.schedule.entries.map((e: { departure: string }) => e.departure)).toEqual(['07:30', '09:30', '12:00']);
expect(day.schedule.warnings).toEqual([{ index: 1, code: 'missedLeave', minutes: 30 }]);
});
it('keeps explicit end-day visits and manual boundaries in the shared planning path', async () => {
const s = setup();
s.visits[0].end_day = 1;
+1
View File
@@ -82,6 +82,7 @@ export default defineConfig({
'src/nest/database/**/*.ts': { statements: 99, branches: 100, functions: 99, lines: 99 },
'src/nest/day-notes/**/*.ts': { statements: 92, branches: 83, functions: 99, lines: 99 },
'src/nest/days/**/*.ts': { statements: 92, branches: 82, functions: 98, lines: 97 },
'src/nest/doc-sync/**/*.ts': { statements: 91, branches: 79, functions: 96, lines: 95 },
'src/nest/feeds/**/*.ts': { statements: 91, branches: 83, functions: 83, lines: 91 },
'src/nest/files/**/*.ts': { statements: 97, branches: 95, functions: 99, lines: 98 },
'src/nest/geo/**/*.ts': { statements: 99, branches: 95, functions: 99, lines: 99 },