mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-23 07:26:30 +00:00
Merge pull request #2391 from liketrek/feat/document-providers
feat(documents): two-way sync with Paperless, Papra, Nextcloud, OpenCloud and Synology
This commit is contained in:
@@ -42,6 +42,7 @@ export const MCP_GATED_ADDON_IDS: readonly AddonId[] = [
|
||||
ADDON_IDS.COLLAB,
|
||||
ADDON_IDS.COLLECTIONS,
|
||||
ADDON_IDS.DAWARICH,
|
||||
ADDON_IDS.DOCUMENTS,
|
||||
ADDON_IDS.JOURNEY,
|
||||
ADDON_IDS.PACKING,
|
||||
ADDON_IDS.ROADTRIP,
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
/**
|
||||
* The five document providers and the fields their connection form asks for.
|
||||
*
|
||||
* Shared by the migration that introduces the tables and by `seeds.ts`, so a
|
||||
* fresh install and an upgraded one end up with the same rows. Both call sites
|
||||
* use INSERT OR IGNORE, which is what lets an operator rename or re-sort a row
|
||||
* without a later boot undoing it.
|
||||
*
|
||||
* `label` and `hint` are i18n key suffixes, never display text: the client
|
||||
* resolves them as `docsync.<label>`, the same way the photo provider form
|
||||
* resolves `memories.<label>`.
|
||||
*
|
||||
* Nextcloud and OpenCloud are separate rows although one WebDAV adapter serves
|
||||
* both: the fields differ (a Nextcloud login name and app password against a
|
||||
* folder, an OpenCloud username and app token against a space), and an operator
|
||||
* scanning the addon shelf should find the product they actually run.
|
||||
*/
|
||||
|
||||
interface ProviderRow {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
icon: string;
|
||||
sort_order: number;
|
||||
}
|
||||
|
||||
interface ProviderFieldRow {
|
||||
provider_id: string;
|
||||
field_key: string;
|
||||
label: string;
|
||||
input_type: 'text' | 'url' | 'password' | 'checkbox';
|
||||
placeholder: string | null;
|
||||
hint: string | null;
|
||||
required: 0 | 1;
|
||||
secret: 0 | 1;
|
||||
sort_order: number;
|
||||
}
|
||||
|
||||
const PROVIDERS: ProviderRow[] = [
|
||||
{
|
||||
id: 'paperless',
|
||||
name: 'Paperless-ngx',
|
||||
description: 'Two-way document sync with a Paperless-ngx instance, scoped by tag',
|
||||
icon: 'FileText',
|
||||
sort_order: 0,
|
||||
},
|
||||
{
|
||||
id: 'papra',
|
||||
name: 'Papra',
|
||||
description: 'Two-way document sync with a Papra organisation, scoped by tag',
|
||||
icon: 'FileText',
|
||||
sort_order: 1,
|
||||
},
|
||||
{
|
||||
id: 'nextcloud',
|
||||
name: 'Nextcloud',
|
||||
description: 'Two-way document sync with a Nextcloud folder over WebDAV',
|
||||
icon: 'Cloud',
|
||||
sort_order: 2,
|
||||
},
|
||||
{
|
||||
id: 'opencloud',
|
||||
name: 'OpenCloud',
|
||||
description: 'Two-way document sync with an OpenCloud space over WebDAV',
|
||||
icon: 'Cloud',
|
||||
sort_order: 3,
|
||||
},
|
||||
{
|
||||
id: 'synologydrive',
|
||||
name: 'Synology Drive',
|
||||
description: 'Two-way document sync with a folder on a Synology NAS',
|
||||
icon: 'HardDrive',
|
||||
sort_order: 4,
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* `allow_insecure_tls` defaults to 0 on every provider, unlike the Synology
|
||||
* photo provider, where skipping verification is the stored default. A LAN
|
||||
* instance with a self-signed certificate is common enough to need the switch
|
||||
* and not common enough to justify shipping it on.
|
||||
*/
|
||||
const FIELDS: ProviderFieldRow[] = [
|
||||
// Paperless-ngx: token auth, tag scope.
|
||||
{ provider_id: 'paperless', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://paperless.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
|
||||
{ provider_id: 'paperless', field_key: 'api_token', label: 'providerApiToken', input_type: 'password', placeholder: 'API token', hint: 'hintPaperlessToken', required: 1, secret: 1, sort_order: 1 },
|
||||
{ provider_id: 'paperless', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 2 },
|
||||
|
||||
// Papra: bearer key, and an organisation the key must already belong to.
|
||||
{ provider_id: 'papra', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://papra.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
|
||||
{ provider_id: 'papra', field_key: 'api_key', label: 'providerApiKey', input_type: 'password', placeholder: 'ppapi_…', hint: 'hintPapraKey', required: 1, secret: 1, sort_order: 1 },
|
||||
{ provider_id: 'papra', field_key: 'organization_id', label: 'providerOrganization', input_type: 'text', placeholder: 'org_…', hint: 'hintPapraOrg', required: 1, secret: 0, sort_order: 2 },
|
||||
{ provider_id: 'papra', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 3 },
|
||||
|
||||
// Nextcloud: an app password, never the account password. The latter breaks
|
||||
// the moment the account turns on 2FA or logs in through OIDC.
|
||||
{ provider_id: 'nextcloud', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://cloud.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
|
||||
{ provider_id: 'nextcloud', field_key: 'login_name', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: 'hintNextcloudLogin', required: 1, secret: 0, sort_order: 1 },
|
||||
{ provider_id: 'nextcloud', field_key: 'app_password', label: 'providerAppPassword', input_type: 'password', placeholder: 'app password', hint: 'hintNextcloudAppPassword', required: 1, secret: 1, sort_order: 2 },
|
||||
{ provider_id: 'nextcloud', field_key: 'base_path', label: 'providerBasePath', input_type: 'text', placeholder: '/TREK', hint: 'hintBasePath', required: 0, secret: 0, sort_order: 3 },
|
||||
{ provider_id: 'nextcloud', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 4 },
|
||||
|
||||
// OpenCloud: auth-app token, spaces instead of folders.
|
||||
{ provider_id: 'opencloud', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://opencloud.example.com', hint: null, required: 1, secret: 0, sort_order: 0 },
|
||||
{ provider_id: 'opencloud', field_key: 'username', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: null, required: 1, secret: 0, sort_order: 1 },
|
||||
{ provider_id: 'opencloud', field_key: 'app_token', label: 'providerAppToken', input_type: 'password', placeholder: 'app token', hint: 'hintOpenCloudToken', required: 1, secret: 1, sort_order: 2 },
|
||||
{ provider_id: 'opencloud', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 3 },
|
||||
|
||||
// Synology: DSM has no app tokens at all, so this is a real account. The OTP
|
||||
// is used once to obtain a device token, which is what gets stored.
|
||||
{ provider_id: 'synologydrive', field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://nas.example.com:5001', hint: 'hintSynologyUrl', required: 1, secret: 0, sort_order: 0 },
|
||||
{ provider_id: 'synologydrive', field_key: 'username', label: 'providerUsername', input_type: 'text', placeholder: 'username', hint: 'hintSynologyUser', required: 1, secret: 0, sort_order: 1 },
|
||||
{ provider_id: 'synologydrive', field_key: 'password', label: 'providerPassword', input_type: 'password', placeholder: 'password', hint: null, required: 1, secret: 1, sort_order: 2 },
|
||||
{ provider_id: 'synologydrive', field_key: 'otp_code', label: 'providerOTP', input_type: 'text', placeholder: '123456', hint: 'hintSynologyOtp', required: 0, secret: 1, sort_order: 3 },
|
||||
{ provider_id: 'synologydrive', field_key: 'base_path', label: 'providerBasePath', input_type: 'text', placeholder: '/trek', hint: 'hintBasePath', required: 0, secret: 0, sort_order: 4 },
|
||||
{ provider_id: 'synologydrive', field_key: 'allow_insecure_tls', label: 'allowInsecureTls', input_type: 'checkbox', placeholder: null, hint: null, required: 0, secret: 0, sort_order: 5 },
|
||||
];
|
||||
|
||||
export function seedDocumentProviders(db: Database.Database): void {
|
||||
const insertProvider = db.prepare(
|
||||
'INSERT OR IGNORE INTO document_providers (id, name, description, icon, enabled, sort_order) VALUES (?, ?, ?, ?, 0, ?)',
|
||||
);
|
||||
for (const p of PROVIDERS) insertProvider.run(p.id, p.name, p.description, p.icon, p.sort_order);
|
||||
|
||||
const insertField = db.prepare(
|
||||
`INSERT OR IGNORE INTO document_provider_fields
|
||||
(provider_id, field_key, label, input_type, placeholder, hint, required, secret, sort_order)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
for (const f of FIELDS) {
|
||||
insertField.run(f.provider_id, f.field_key, f.label, f.input_type, f.placeholder, f.hint, f.required, f.secret, f.sort_order);
|
||||
}
|
||||
}
|
||||
|
||||
export const DOCUMENT_PROVIDER_SEED_IDS = PROVIDERS.map((p) => p.id);
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ROADTRIP_PREFERENCE_KEYS } from '@trek/shared';
|
||||
import { readEnv } from '../app-config';
|
||||
import { encrypt_api_key } from '../nest/common/crypto/apiKeyCrypto';
|
||||
import { seedDocumentProviders } from './document-provider-seed';
|
||||
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'fs';
|
||||
@@ -4949,6 +4950,202 @@ function runMigrations(db: Database.Database): void {
|
||||
'CREATE INDEX IF NOT EXISTS idx_journey_entries_source_assignment ON journey_entries(source_place_id, source_assignment_id)',
|
||||
);
|
||||
},
|
||||
|
||||
/*
|
||||
* Document providers, part 1 of 3: the registry (#214).
|
||||
*
|
||||
* Deliberately its own pair of tables rather than a `kind` column on
|
||||
* `photo_providers`. The client filters on `type === 'photo_provider'`, the
|
||||
* Journey cascade runs `UPDATE photo_providers SET enabled = 0` with no
|
||||
* WHERE clause, and migrations are append-only. Reusing those tables would
|
||||
* change the behaviour of three existing paths, which is exactly what the
|
||||
* no-breaking-changes rule forbids. The field columns follow
|
||||
* `photo_provider_fields` except for `settings_key` and `payload_key`.
|
||||
* Those map a photo field onto a settings key and a request key; a document
|
||||
* field is stored under its own `field_key` in one JSON column, and the
|
||||
* connect form takes its fields from /docsync/providers rather than from
|
||||
* the generic settings form, so nothing would ever read them.
|
||||
*/
|
||||
() => {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS document_providers (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
icon TEXT DEFAULT 'FileText',
|
||||
enabled INTEGER DEFAULT 0,
|
||||
sort_order INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS document_provider_fields (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider_id TEXT NOT NULL REFERENCES document_providers(id) ON DELETE CASCADE,
|
||||
field_key TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
input_type TEXT NOT NULL DEFAULT 'text',
|
||||
placeholder TEXT,
|
||||
hint TEXT,
|
||||
required INTEGER DEFAULT 0,
|
||||
secret INTEGER DEFAULT 0,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
UNIQUE(provider_id, field_key)
|
||||
);
|
||||
`);
|
||||
seedDocumentProviders(db);
|
||||
},
|
||||
|
||||
/*
|
||||
* Document providers, part 2 of 3: the connection, and the trip binding.
|
||||
*
|
||||
* The connection carries a `trip_id`, and that is the one place this design
|
||||
* departs from every integration already in the repo. Immich, Synology
|
||||
* Photos, AirTrail and Dawarich all hang off a user, and even
|
||||
* `trip_album_links` carries a `user_id`; photos become visible to the rest
|
||||
* of a trip only through an opt-in `shared` flag. None of that can satisfy
|
||||
* "everyone on the trip sees the same documents": it would make a
|
||||
* document's visibility depend on whose credentials fetched it. So the trip
|
||||
* admin binds the trip once, the server talks to the provider under that
|
||||
* single identity, and TREK's own membership decides who sees what.
|
||||
*
|
||||
* `owner_user_id` stays separate from `trip_id` so the credential holder is
|
||||
* always explicit: when that person leaves the trip the binding goes to
|
||||
* `orphaned` rather than silently continuing to use an ex-member's token.
|
||||
*
|
||||
* Secrets live in one encrypted JSON blob instead of per-provider columns:
|
||||
* a sixth provider then needs no migration, and the key rotation in
|
||||
* scripts/migrate-encryption.ts stays one line instead of a field list that
|
||||
* someone will forget, and a forgotten column does not survive a rotation.
|
||||
*/
|
||||
() => {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS document_connections (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
|
||||
provider_id TEXT NOT NULL REFERENCES document_providers(id) ON DELETE CASCADE,
|
||||
owner_user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
base_url TEXT NOT NULL,
|
||||
secrets TEXT,
|
||||
settings TEXT NOT NULL DEFAULT '{}',
|
||||
allow_insecure_tls INTEGER NOT NULL DEFAULT 0,
|
||||
capabilities TEXT,
|
||||
last_probe_at TEXT,
|
||||
last_probe_state TEXT NOT NULL DEFAULT 'never',
|
||||
last_probe_error TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(trip_id, provider_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_connections_trip ON document_connections(trip_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_connections_owner ON document_connections(owner_user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trip_document_links (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
|
||||
connection_id INTEGER NOT NULL REFERENCES document_connections(id) ON DELETE CASCADE,
|
||||
provider_id TEXT NOT NULL,
|
||||
remote_scope_key TEXT NOT NULL,
|
||||
remote_root_id TEXT,
|
||||
remote_root_path TEXT,
|
||||
remote_label TEXT NOT NULL DEFAULT '',
|
||||
direction TEXT NOT NULL DEFAULT 'both',
|
||||
delete_policy TEXT NOT NULL DEFAULT 'unlink',
|
||||
conflict_policy TEXT NOT NULL DEFAULT 'manual',
|
||||
sync_enabled INTEGER NOT NULL DEFAULT 1,
|
||||
webhook_token TEXT,
|
||||
webhook_secret TEXT,
|
||||
webhook_subscription_id TEXT,
|
||||
remote_cursor TEXT,
|
||||
last_sync_at TEXT,
|
||||
last_sync_state TEXT NOT NULL DEFAULT 'never',
|
||||
last_sync_error TEXT,
|
||||
failure_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT,
|
||||
created_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(trip_id, connection_id, remote_scope_key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_trip_document_links_trip ON trip_document_links(trip_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_trip_document_links_due ON trip_document_links(sync_enabled, next_attempt_at);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_trip_document_links_token
|
||||
ON trip_document_links(webhook_token) WHERE webhook_token IS NOT NULL;
|
||||
`);
|
||||
},
|
||||
|
||||
/*
|
||||
* Document providers, part 3 of 3: the pairing and its sync state.
|
||||
*
|
||||
* `content_sha256` and `pushed_sha256` are two columns on purpose. The
|
||||
* first is the bytes both sides last agreed on, the second is what TREK
|
||||
* itself last uploaded. Collapsing them into one is precisely the mistake
|
||||
* that builds an echo loop: a webhook fires for TREK's own write, the core
|
||||
* cannot tell it from a stranger's edit, and the file bounces.
|
||||
*
|
||||
* `remote_missing_at` records that something vanished upstream instead of
|
||||
* acting on it, the rule Dawarich already follows with `source_missing_at`.
|
||||
* An unmounted share answers with an empty listing, and reading that as
|
||||
* "everything was deleted" would empty a trip.
|
||||
*
|
||||
* `file_id ON DELETE SET NULL` keeps a tombstone behind after a document is
|
||||
* permanently deleted in TREK, so the next run does not cheerfully download
|
||||
* it again.
|
||||
*/
|
||||
() => {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS document_sync_items (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
link_id INTEGER NOT NULL REFERENCES trip_document_links(id) ON DELETE CASCADE,
|
||||
trip_id INTEGER NOT NULL REFERENCES trips(id) ON DELETE CASCADE,
|
||||
file_id INTEGER REFERENCES trip_files(id) ON DELETE SET NULL,
|
||||
trek_doc_uid TEXT NOT NULL,
|
||||
remote_id TEXT,
|
||||
remote_name TEXT,
|
||||
remote_version TEXT,
|
||||
remote_size INTEGER,
|
||||
remote_modified_at TEXT,
|
||||
content_sha256 TEXT,
|
||||
pushed_sha256 TEXT,
|
||||
state TEXT NOT NULL DEFAULT 'pending',
|
||||
error_code TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT,
|
||||
remote_missing_at TEXT,
|
||||
first_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_seen_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
synced_at TEXT
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_remote
|
||||
ON document_sync_items(link_id, remote_id) WHERE remote_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_file
|
||||
ON document_sync_items(link_id, file_id) WHERE file_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_document_sync_items_uid
|
||||
ON document_sync_items(link_id, trek_doc_uid);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_sync_items_trip_state ON document_sync_items(trip_id, state);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_sync_items_hash ON document_sync_items(link_id, content_sha256);
|
||||
CREATE INDEX IF NOT EXISTS idx_document_sync_items_due ON document_sync_items(link_id, next_attempt_at);
|
||||
`);
|
||||
},
|
||||
|
||||
/*
|
||||
* Document providers: when TREK itself put a provider copy in the bin.
|
||||
*
|
||||
* A file deleted in TREK under the `trash` policy takes its provider copy
|
||||
* with it. Taken back out of TREK's trash, it has to go up again; a copy
|
||||
* somebody else deleted in the meantime must not. Both leave the same gap
|
||||
* in a listing, so the difference is written down when TREK acts rather
|
||||
* than guessed at later.
|
||||
*
|
||||
* No backfill from the binding's current policy: that policy may not be
|
||||
* the one the deletion ran under, and reading it back is the retroactive
|
||||
* mistake this column exists to avoid. A row left NULL is treated like a
|
||||
* copy somebody else deleted, which flags it instead of uploading it.
|
||||
*/
|
||||
() => {
|
||||
const hasColumn = db
|
||||
.prepare("SELECT 1 FROM pragma_table_info('document_sync_items') WHERE name = 'remote_trashed_at'")
|
||||
.get();
|
||||
if (!hasColumn) db.exec('ALTER TABLE document_sync_items ADD COLUMN remote_trashed_at TEXT');
|
||||
},
|
||||
];
|
||||
|
||||
if (currentVersion < migrations.length) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import crypto from 'crypto';
|
||||
import { readEnv } from '../app-config';
|
||||
import { seedDocumentProviders } from './document-provider-seed';
|
||||
|
||||
// bcrypt cost factor for the seeded admin password — kept in sync with authService.
|
||||
const BCRYPT_COST = 12;
|
||||
@@ -164,6 +165,13 @@ function seedAddons(db: Database.Database): void {
|
||||
for (const f of providerFields) {
|
||||
insertProviderField.run(f.provider_id, f.field_key, f.label, f.input_type, f.placeholder, f.hint, f.required, f.secret, f.settings_key, f.payload_key, f.sort_order);
|
||||
}
|
||||
|
||||
// Document providers live in their own pair of tables (see the migration
|
||||
// for why they are not a `kind` column on photo_providers). Seeded from the
|
||||
// same helper the migration uses, so a fresh install and an upgraded one
|
||||
// agree.
|
||||
seedDocumentProviders(db);
|
||||
|
||||
console.log('Default addons seeded');
|
||||
} catch (err: unknown) {
|
||||
console.error('Error seeding addons:', err instanceof Error ? err.message : err);
|
||||
|
||||
@@ -598,6 +598,28 @@ export class AdminService {
|
||||
fieldsByProvider.set(field.provider_id, arr);
|
||||
}
|
||||
|
||||
// Document providers are the Documents addon's shelf rows, the same way
|
||||
// photo providers are Journey's. Hidden on a managed instance for the same
|
||||
// reason: Paperless, Papra, Nextcloud, OpenCloud and a Synology NAS are all
|
||||
// servers the admin runs at home, and a hosted TREK has no route to one.
|
||||
//
|
||||
// They carry no `config` routes, unlike photo providers: the credentials do
|
||||
// not belong to a user here but to a trip, so they are entered in the trip
|
||||
// rather than in settings. The admin decides only whether a provider may be
|
||||
// offered at all.
|
||||
const docProviders = this.db.all<{
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string | null;
|
||||
icon: string;
|
||||
enabled: number;
|
||||
sort_order: number;
|
||||
}>(`
|
||||
SELECT id, name, description, icon, enabled, sort_order
|
||||
FROM document_providers
|
||||
ORDER BY sort_order, id
|
||||
`).filter(() => !readEnv().managed.enabled);
|
||||
|
||||
return [
|
||||
...addons.map((a) => ({
|
||||
...a,
|
||||
@@ -628,6 +650,17 @@ export class AdminService {
|
||||
})),
|
||||
sort_order: p.sort_order,
|
||||
})),
|
||||
...docProviders.map((p) => ({
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
description: p.description,
|
||||
type: 'document_provider',
|
||||
icon: p.icon,
|
||||
enabled: !!p.enabled,
|
||||
config: {},
|
||||
fields: [],
|
||||
sort_order: p.sort_order,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -635,7 +668,8 @@ export class AdminService {
|
||||
type ProviderRow = { id: string; name: string; description?: string | null; icon: string; enabled: number; sort_order: number };
|
||||
const addon = this.db.get<Addon>('SELECT * FROM addons WHERE id = ?', id);
|
||||
const provider = this.db.get<ProviderRow>('SELECT * FROM photo_providers WHERE id = ?', id);
|
||||
if (!addon && !provider) return { error: 'Addon not found', status: 404 };
|
||||
const docProvider = this.db.get<ProviderRow>('SELECT * FROM document_providers WHERE id = ?', id);
|
||||
if (!addon && !provider && !docProvider) return { error: 'Addon not found', status: 404 };
|
||||
|
||||
// The whole addon, not just its config: on a centrally administered install
|
||||
// the operator owns the endpoint, the model and the per-document cost, so
|
||||
@@ -652,6 +686,13 @@ export class AdminService {
|
||||
return { error: 'Enable the Journey addon first', status: 409 };
|
||||
}
|
||||
|
||||
// Same rule one shelf down: a document provider only exists to serve the
|
||||
// file manager, so switching one on under a disabled Documents addon would
|
||||
// advertise a sync nothing can reach.
|
||||
if (docProvider && data.enabled === true && !this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS)) {
|
||||
return { error: 'Enable the Documents addon first', status: 409 };
|
||||
}
|
||||
|
||||
this.db.transaction(() => {
|
||||
if (addon) {
|
||||
if (data.enabled !== undefined) {
|
||||
@@ -660,6 +701,10 @@ export class AdminService {
|
||||
// resurface the moment journey returns, which nobody switched on.
|
||||
if (id === ADDON_IDS.JOURNEY && !data.enabled)
|
||||
this.db.run('UPDATE photo_providers SET enabled = 0');
|
||||
// Documents off takes its providers with it, for the reason above: a
|
||||
// row left enabled would resurface the moment the addon returns.
|
||||
if (id === ADDON_IDS.DOCUMENTS && !data.enabled)
|
||||
this.db.run('UPDATE document_providers SET enabled = 0');
|
||||
}
|
||||
if (data.config !== undefined) {
|
||||
// The AI-parsing addon holds an API key — encrypt it at rest and preserve
|
||||
@@ -670,9 +715,12 @@ export class AdminService {
|
||||
: data.config;
|
||||
this.db.run('UPDATE addons SET config = ? WHERE id = ?', JSON.stringify(configToStore), id);
|
||||
}
|
||||
} else {
|
||||
} else if (provider) {
|
||||
if (data.enabled !== undefined)
|
||||
this.db.run('UPDATE photo_providers SET enabled = ? WHERE id = ?', data.enabled ? 1 : 0, id);
|
||||
} else {
|
||||
if (data.enabled !== undefined)
|
||||
this.db.run('UPDATE document_providers SET enabled = ? WHERE id = ?', data.enabled ? 1 : 0, id);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ import { PhotosModule } from './photos/photos.module';
|
||||
import { MemoriesModule } from './memories/memories.module';
|
||||
import { AirtrailModule } from './integrations/airtrail.module';
|
||||
import { DawarichModule } from './integrations/dawarich.module';
|
||||
import { DocSyncModule } from './doc-sync/doc-sync.module';
|
||||
import { JourneyModule } from './journey/journey.module';
|
||||
import { CollectionsModule } from './collections/collections.module';
|
||||
import { ShareModule } from './share/share.module';
|
||||
@@ -81,7 +82,7 @@ import { RealtimeGatewayModule } from './realtime/realtime-gateway.module';
|
||||
* migrated.
|
||||
*/
|
||||
@Module({
|
||||
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
|
||||
imports: [AppConfigModule, DatabaseModule, RealtimeModule, RealtimeGatewayModule, SchedulingModule, McpModule.forRoot({ accessPolicy: trekMcpAccessPolicy, validateAccess: trekMcpValidateAccess }), HealthModule, PlatformModule, McpTransportModule, WeatherModule, PublicApiModule, HelpModule, AirportsModule, ConfigModule, SystemNoticesModule, GeoModule, MapsModule, PlaceEnrichmentModule, PlaceShadowModule, RouteUsageModule, CategoriesModule, TagsModule, NotificationsModule, AtlasModule, VacayModule, SchoolHolidaysModule, PackingModule, TodoModule, BudgetModule, ReservationsModule, DaysModule, DayNotesModule, AccommodationsModule, AssignmentsModule, PlacesModule, RoadtripModule, TripsModule, CollabModule, FilesModule, DocSyncModule, PhotosModule, MemoriesModule, AirtrailModule, DawarichModule, JourneyModule, CollectionsModule, ShareModule, TripInviteModule, TransitModule, FeedsModule, SettingsModule, StorageModule, BackupModule, AuthModule, OidcModule, OauthModule, AdminModule, AddonsModule, AuditModule, PermissionsModule, PluginsModule, BookingImportModule, ReservationImportModule, LlmParseModule, ManagedExtModule],
|
||||
providers: [
|
||||
// Default-deny: a route is authenticated unless it carries @Public() or
|
||||
// @OptionalAuth(), or declares its own @UseGuards chain. Protection used to
|
||||
|
||||
@@ -167,8 +167,17 @@ export class AssignmentOpsController {
|
||||
if (!this.assignments.getAssignmentForTrip(id, tripId)) {
|
||||
throw new HttpException({ error: 'Assignment not found' }, 404);
|
||||
}
|
||||
const assignment = this.assignments.updateTime(id, body.place_time, body.end_time);
|
||||
const { assignment, reordered, vias } = this.assignments.updateTime(id, body.place_time, body.end_time);
|
||||
this.assignments.broadcast(tripId, 'assignment:updated', { assignment }, socketId);
|
||||
// The whole day when a start moved stops, or collaborators apply the one row
|
||||
// they were sent to their old order and end up with a third one.
|
||||
//
|
||||
// Both to every socket, the writer's included, so the order and the vias pinned
|
||||
// to it arrive together. The planner holds its vias in memory and would route the
|
||||
// new anchors on the old order until its reload of the day came back, and a save
|
||||
// replayed from the offline queue has no reload after it at all.
|
||||
if (reordered) this.assignments.broadcast(tripId, 'assignment:reordered', reordered, undefined);
|
||||
if (vias) this.assignments.broadcast(tripId, 'roadtripVia:changed', vias, undefined);
|
||||
this.assignments.reconcile(tripId, socketId);
|
||||
return { assignment };
|
||||
}
|
||||
|
||||
@@ -130,12 +130,15 @@ export class AssignmentsMcp {
|
||||
if (!this.guards.hasTripPermission('day_edit', tripId, ctx.userId)) return permissionDenied();
|
||||
const existing = this.assignments.getAssignmentForTrip(assignmentId, tripId);
|
||||
if (!existing) return errorResult('Assignment not found.');
|
||||
const assignment = this.assignments.updateTime(
|
||||
const { assignment, reordered, vias } = this.assignments.updateTime(
|
||||
assignmentId,
|
||||
place_time !== undefined ? place_time : existing.assignment_time,
|
||||
end_time !== undefined ? end_time : existing.assignment_end_time
|
||||
);
|
||||
// Same three events as PUT /assignments/:id/time.
|
||||
this.guards.safeBroadcast(tripId, 'assignment:updated', { assignment });
|
||||
if (reordered) this.guards.safeBroadcast(tripId, 'assignment:reordered', reordered);
|
||||
if (vias) this.guards.safeBroadcast(tripId, 'roadtripVia:changed', vias);
|
||||
this.assignments.reconcile(tripId);
|
||||
return ok({ assignment });
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
|
||||
import { chronoOrder, type RoadtripVia, type TrekWsPayload, type TrekWsTripEventName } from '@trek/shared';
|
||||
import { isEmptyReanchoring, reanchorByStopOrder, type AnchoredVia } from '@trek/shared/roadtrip';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import { DatabaseService, type TripAccess } from '../database/database.service';
|
||||
import { PermissionsService } from '../permissions/permissions.service';
|
||||
@@ -10,6 +11,39 @@ import { JourneyDomainService } from '../journey/journey-domain.service';
|
||||
|
||||
type Trip = TripAccess;
|
||||
|
||||
/** One stop of a day as the time sort reads it. */
|
||||
interface DayStopRow {
|
||||
id: number;
|
||||
order_index: number;
|
||||
effective_time: string | null;
|
||||
located: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* What saving a time changed besides the stop itself, so each caller can tell the
|
||||
* trip. `reordered` carries the day's whole order and `vias` the day's re-pinned
|
||||
* vias; both stay null when the save left every stop where it was.
|
||||
*/
|
||||
export interface AssignmentTimeUpdate {
|
||||
assignment: ReturnType<AssignmentsService['getAssignmentWithPlace']>;
|
||||
reordered: { dayId: number; orderedIds: number[] } | null;
|
||||
vias: { dayId: number; vias: RoadtripVia[] } | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Where the time sort puts a value it cannot read as a clock time: after every real
|
||||
* time, which is where the '99:99' sentinel has always put a time without a colon. A
|
||||
* legacy "morning" keeps sorting where it did. The client reads such a value as no
|
||||
* time at all; that difference is older than this rule and left alone.
|
||||
*/
|
||||
const UNREADABLE_TIME = 99 * 60 + 99;
|
||||
|
||||
function sortMinutes(time: string | null): number | null {
|
||||
if (!time) return null;
|
||||
const clock = /(?:^|T)(\d{1,2}):(\d{2})/.exec(time);
|
||||
return clock ? Number(clock[1]) * 60 + Number(clock[2]) : UNREADABLE_TIME;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assignments domain service — owns the day-assignment SQL (relocated from the
|
||||
* legacy services/assignmentService.ts, then hardened: every multi-statement
|
||||
@@ -206,48 +240,136 @@ export class AssignmentsService {
|
||||
`, assignmentId);
|
||||
}
|
||||
|
||||
updateTime(id: string | number, placeTime: unknown, endTime: unknown) {
|
||||
this.dbs.transaction(() => {
|
||||
/**
|
||||
* Saves a visit's own start and end, and puts the day back in time order when the
|
||||
* start changed. The rule is the planner's (`chronoOrder`): an untimed stop stays
|
||||
* behind the stop it followed. This used to append every untimed stop after the
|
||||
* timed ones, so one start time pulled a stop planned last to the top of the day.
|
||||
*
|
||||
* The rule is shared, what it reads is not. This sorts the day's stops alone, with a
|
||||
* booked night timed by its check-in. The planner sorts day notes and bookings in
|
||||
* between them and never draws the night's row. An untimed stop behind a timed note
|
||||
* or train takes that item's time there and the previous stop's time here, so on
|
||||
* such a day the order stored and the order drawn can differ. The old sort did the
|
||||
* same.
|
||||
*/
|
||||
updateTime(id: string | number, placeTime: unknown, endTime: unknown): AssignmentTimeUpdate {
|
||||
const sorted = this.dbs.transaction(() => {
|
||||
const stored = this.dbs.get<{ day_id: number; start: string | null }>(`
|
||||
SELECT da.day_id, COALESCE(da.assignment_time, p.place_time, acc.check_in) AS start
|
||||
FROM day_assignments da
|
||||
JOIN places p ON da.place_id = p.id
|
||||
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
|
||||
WHERE da.id = ?
|
||||
`, id);
|
||||
|
||||
// Falsy times (null, undefined, '') all clear the override — an empty
|
||||
// string is a clear, not a stored value.
|
||||
this.dbs.run('UPDATE day_assignments SET assignment_time = ?, assignment_end_time = ? WHERE id = ?',
|
||||
placeTime || null, endTime || null, id);
|
||||
|
||||
// Auto-sort: reorder timed assignments chronologically within the day
|
||||
if (placeTime) {
|
||||
const assignment = this.dbs.get<{ day_id: number }>('SELECT day_id FROM day_assignments WHERE id = ?', id);
|
||||
if (assignment) {
|
||||
// A booked night's hour lives on the booking, not on the stop: nobody types a
|
||||
// time into a hotel row, they type a check-in. Left out of this, the night
|
||||
// counted as untimed and stayed wherever it had been dropped, so pinning an
|
||||
// afternoon stop sorted that one and left the hotel sitting in front of or
|
||||
// behind it by accident.
|
||||
const dayAssignments = this.dbs.all<{ id: number; effective_time: string | null }>(`
|
||||
SELECT da.id, COALESCE(da.assignment_time, p.place_time, acc.check_in) as effective_time
|
||||
FROM day_assignments da
|
||||
JOIN places p ON da.place_id = p.id
|
||||
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
|
||||
WHERE da.day_id = ?
|
||||
ORDER BY da.order_index ASC
|
||||
`, assignment.day_id);
|
||||
|
||||
// Separate timed and untimed, sort timed by time
|
||||
const timed = dayAssignments.filter(a => a.effective_time).sort((a, b) => {
|
||||
const ta = a.effective_time!.includes(':') ? a.effective_time! : '99:99';
|
||||
const tb = b.effective_time!.includes(':') ? b.effective_time! : '99:99';
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
const untimed = dayAssignments.filter(a => !a.effective_time);
|
||||
|
||||
// Interleave: timed in chronological order, untimed keep relative position
|
||||
const reordered = [...timed, ...untimed];
|
||||
const update = this.dbs.prepare('UPDATE day_assignments SET order_index = ? WHERE id = ?');
|
||||
reordered.forEach((a, i) => update.run(i, a.id));
|
||||
}
|
||||
}
|
||||
// Only a start that moved sorts. An end is a label. A start sent again as it
|
||||
// stood (the place form saving an End, the stay dialog taking one off, an MCP
|
||||
// call that names only the end) leaves the day the way the traveller left it,
|
||||
// which can be out of time order on purpose. Compared the way the sort reads
|
||||
// it, so a visit given the time its place already had moves nothing either. A
|
||||
// cleared start leaves the day alone too.
|
||||
if (!placeTime || !stored) return null;
|
||||
if (sortMinutes(String(placeTime)) === sortMinutes(stored.start)) return null;
|
||||
return this.sortDayByTime(stored.day_id);
|
||||
});
|
||||
|
||||
return this.getAssignmentWithPlace(Number(id));
|
||||
return {
|
||||
assignment: this.getAssignmentWithPlace(Number(id)),
|
||||
reordered: sorted ? { dayId: sorted.dayId, orderedIds: sorted.orderedIds } : null,
|
||||
vias: sorted?.viasMoved ? { dayId: sorted.dayId, vias: this.listDayVias(sorted.dayId) } : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts one day in time order. Writes nothing when it already is, which is the usual
|
||||
* case: most starts are typed in the order the day is planned.
|
||||
*/
|
||||
private sortDayByTime(dayId: number): { dayId: number; orderedIds: number[]; viasMoved: boolean } | null {
|
||||
// A booked night's hour lives on the booking, not on the stop: nobody types a
|
||||
// time into a hotel row, they type a check-in. Left out of this, the night
|
||||
// counted as untimed and stayed wherever it had been dropped, so pinning an
|
||||
// afternoon stop sorted that one and left the hotel sitting in front of or
|
||||
// behind it by accident.
|
||||
const rows = this.dbs.all<DayStopRow>(`
|
||||
SELECT da.id, da.order_index, COALESCE(da.assignment_time, p.place_time, acc.check_in) as effective_time,
|
||||
(p.lat IS NOT NULL AND p.lng IS NOT NULL) as located
|
||||
FROM day_assignments da
|
||||
JOIN places p ON da.place_id = p.id
|
||||
LEFT JOIN day_accommodations acc ON acc.id = da.accommodation_id
|
||||
WHERE da.day_id = ?
|
||||
ORDER BY da.order_index ASC, da.created_at ASC, da.id ASC
|
||||
`, dayId);
|
||||
|
||||
const sorted = chronoOrder(rows, row => sortMinutes(row.effective_time));
|
||||
if (sorted.every((row, i) => row === rows[i])) return null;
|
||||
|
||||
// Numbered from 0, the way a drag stores a day (`reorderAssignments`). The order
|
||||
// goes out as a list of ids and every client numbers it by position, so keys kept
|
||||
// with their gaps would put the day notes and bookings that sort between stops in
|
||||
// one place for the writer, who reads the day back, and in another for everyone
|
||||
// else. Only a stop whose key changes is written.
|
||||
const update = this.dbs.prepare('UPDATE day_assignments SET order_index = ? WHERE id = ?');
|
||||
sorted.forEach((row, i) => {
|
||||
if (row.order_index !== i) update.run(i, row.id);
|
||||
});
|
||||
|
||||
return { dayId, orderedIds: sorted.map(row => row.id), viasMoved: this.reanchorVias(dayId, rows, sorted) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps every drawn road behind the stop it was drawn after, the rule the planner
|
||||
* applies when stops are dragged (`reanchorByStopOrder`). A via is pinned to a
|
||||
* POSITION among the day's located stops, so a sort that moves a stop would
|
||||
* otherwise hand the vias behind it to other legs. Inside the sort's transaction:
|
||||
* an order without its vias is a road the traveller never drew.
|
||||
*
|
||||
* No sequence renumbering, unlike RoadtripService.reanchor: a reorder maps each leg
|
||||
* onto a different one, so two legs' vias never end up on the same leg.
|
||||
*/
|
||||
private reanchorVias(dayId: number, before: DayStopRow[], after: DayStopRow[]): boolean {
|
||||
const located = (rows: DayStopRow[]) => rows.filter(row => row.located).map(row => row.id);
|
||||
const previousIds = located(before);
|
||||
const nextIds = located(after);
|
||||
// Only stops without coordinates moved. The router never sees those, so every leg
|
||||
// is still the one it was.
|
||||
if (previousIds.every((stopId, i) => stopId === nextIds[i])) return false;
|
||||
|
||||
// A via behind the day's last stop bends the drive into the next day, on a trip
|
||||
// with connected days or a night drive (the planner's `anchorFor` files it there).
|
||||
// `reanchorByStopOrder` gives a last stop no leg and deletes what follows it, which
|
||||
// is right for a stop the sort made last and wrong for one that was last already.
|
||||
const lastAt = previousIds.length - 1;
|
||||
const seam = previousIds[lastAt] === nextIds[lastAt] ? lastAt : null;
|
||||
const vias = this.dbs.all<AnchoredVia>('SELECT id, after_order_index, lat, lng FROM roadtrip_vias WHERE day_id = ?', dayId)
|
||||
.filter(via => via.after_order_index !== seam);
|
||||
const plan = reanchorByStopOrder(vias, previousIds, nextIds);
|
||||
for (const viaId of plan.remove) {
|
||||
this.dbs.run('DELETE FROM roadtrip_vias WHERE id = ? AND day_id = ?', viaId, dayId);
|
||||
}
|
||||
for (const via of plan.vias) {
|
||||
this.dbs.run('UPDATE roadtrip_vias SET after_order_index = ? WHERE id = ? AND day_id = ?', via.after_order_index, via.id, dayId);
|
||||
}
|
||||
return !isEmptyReanchoring(plan);
|
||||
}
|
||||
|
||||
/**
|
||||
* The day's vias in the shape the road trip routes broadcast them. RoadtripService
|
||||
* has this query too, but its module imports this one, so it cannot be injected here.
|
||||
*/
|
||||
private listDayVias(dayId: number): RoadtripVia[] {
|
||||
return this.dbs.all<RoadtripVia>(
|
||||
`SELECT id, day_id, after_order_index, sequence, lat, lng, created_at
|
||||
FROM roadtrip_vias
|
||||
WHERE day_id = ?
|
||||
ORDER BY after_order_index, sequence, id`,
|
||||
dayId,
|
||||
);
|
||||
}
|
||||
|
||||
setEndDay(id: string | number, endDay: boolean) {
|
||||
|
||||
@@ -107,6 +107,11 @@ export const PUBLIC_ROUTE_ALLOW_LIST: string[] = [
|
||||
'DiscoveryController.protectedResource',
|
||||
'DiscoveryController.wellKnownFallback',
|
||||
'DiscoveryController.wellKnownRoot',
|
||||
// A document provider cannot hold a TREK session. The per-binding token in
|
||||
// the path is the credential, a shared secret is checked on top where the
|
||||
// provider can send one, and the handler's only effect is to schedule a sync
|
||||
// run. It never reads the request body as data.
|
||||
'DocSyncWebhookController.nudge',
|
||||
'FeaturesController.features',
|
||||
// The container/uptime probe.
|
||||
'FeaturesController.health',
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import crypto from 'crypto';
|
||||
import { DOCSYNC_SECRET_MASK, type DocsyncConnectionInput, type DocsyncLinkInput } from '@trek/shared';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { checkSsrf } from '../../utils/ssrfGuard';
|
||||
import { DocumentProviderRegistry } from './document-provider.registry';
|
||||
import type { DocumentConnectionRef, DocumentScopeRef, DocResult } from './document-provider';
|
||||
import { docFailed } from './document-provider';
|
||||
import { decryptSecrets, encryptSecrets, maskSecrets, mergeSecrets } from './doc-sync-secrets';
|
||||
|
||||
/**
|
||||
* Connections and trip bindings: everything a human configures, as opposed to
|
||||
* what the reconciler does with it.
|
||||
*
|
||||
* The connection carries a `trip_id`, and that is the one place this design
|
||||
* leaves the pattern every other integration in the repo follows. Immich,
|
||||
* Synology Photos, AirTrail and Dawarich all store credentials per user, and
|
||||
* `trip_album_links` carries a `user_id` so that photos are shared only through
|
||||
* an explicit flag. None of that can deliver "everyone on the trip sees the
|
||||
* same documents": visibility would depend on whose credentials fetched a file,
|
||||
* and a member without their own Paperless account would see nothing.
|
||||
*
|
||||
* So the trip admin binds the trip once, the server talks to the provider under
|
||||
* that single identity, and TREK's own membership decides who sees what. The
|
||||
* provider never learns that TREK has members. `owner_user_id` stays explicit
|
||||
* beside `trip_id` precisely so that the credential holder is nameable: when
|
||||
* they leave the trip the binding goes to `orphaned` rather than quietly
|
||||
* carrying on with an ex-member's token.
|
||||
*/
|
||||
|
||||
export interface ConnectionRow {
|
||||
id: number;
|
||||
trip_id: number;
|
||||
provider_id: string;
|
||||
owner_user_id: number;
|
||||
base_url: string;
|
||||
secrets: string | null;
|
||||
settings: string;
|
||||
allow_insecure_tls: number;
|
||||
capabilities: string | null;
|
||||
last_probe_at: string | null;
|
||||
last_probe_state: string;
|
||||
last_probe_error: string | null;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface LinkRow {
|
||||
id: number;
|
||||
trip_id: number;
|
||||
connection_id: number;
|
||||
provider_id: string;
|
||||
remote_scope_key: string;
|
||||
remote_root_id: string | null;
|
||||
remote_root_path: string | null;
|
||||
remote_label: string;
|
||||
direction: string;
|
||||
delete_policy: string;
|
||||
conflict_policy: string;
|
||||
sync_enabled: number;
|
||||
webhook_token: string | null;
|
||||
webhook_secret: string | null;
|
||||
webhook_subscription_id: string | null;
|
||||
remote_cursor: string | null;
|
||||
last_sync_at: string | null;
|
||||
last_sync_state: string;
|
||||
last_sync_error: string | null;
|
||||
failure_count: number;
|
||||
next_attempt_at: string | null;
|
||||
}
|
||||
|
||||
interface ProviderFieldRow {
|
||||
field_key: string;
|
||||
/** i18n key suffix, never display text: the client resolves `docsync.<label>`. */
|
||||
label: string;
|
||||
input_type: string;
|
||||
placeholder: string | null;
|
||||
hint: string | null;
|
||||
secret: number;
|
||||
required: number;
|
||||
sort_order: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DocSyncConfigService {
|
||||
constructor(
|
||||
private readonly db: DatabaseService,
|
||||
private readonly registry: DocumentProviderRegistry,
|
||||
) {}
|
||||
|
||||
// ── Provider metadata ──────────────────────────────────────────────────────
|
||||
|
||||
/** Only providers the instance admin switched on may be configured. */
|
||||
enabledProviderIds(): string[] {
|
||||
return this.db.connection
|
||||
.prepare('SELECT id FROM document_providers WHERE enabled = 1 ORDER BY sort_order')
|
||||
.all()
|
||||
.map((r) => (r as { id: string }).id);
|
||||
}
|
||||
|
||||
providerFields(providerId: string): ProviderFieldRow[] {
|
||||
// Every column the form needs, not just the three the secret bookkeeping
|
||||
// uses: the client renders the field from this row, and a missing `label`
|
||||
// turns into a literal "docsync.undefined" on screen.
|
||||
return this.db.connection
|
||||
.prepare(
|
||||
`SELECT field_key, label, input_type, placeholder, hint, secret, required, sort_order
|
||||
FROM document_provider_fields WHERE provider_id = ? ORDER BY sort_order`,
|
||||
)
|
||||
.all(providerId) as ProviderFieldRow[];
|
||||
}
|
||||
|
||||
private secretKeys(providerId: string): string[] {
|
||||
return this.providerFields(providerId).filter((f) => f.secret === 1).map((f) => f.field_key);
|
||||
}
|
||||
|
||||
/**
|
||||
* The display name whatever the enabled flag says. A binding outlives the
|
||||
* admin switching its provider off, and the client only gets names for the
|
||||
* providers that are still on, so without this it would show the raw id.
|
||||
*/
|
||||
private providerName(providerId: string): string {
|
||||
const row = this.db.connection
|
||||
.prepare('SELECT name FROM document_providers WHERE id = ?')
|
||||
.get(providerId) as { name: string } | undefined;
|
||||
return row?.name ?? providerId;
|
||||
}
|
||||
|
||||
// ── Connections ────────────────────────────────────────────────────────────
|
||||
|
||||
getConnection(id: number): ConnectionRow | undefined {
|
||||
return this.db.connection.prepare('SELECT * FROM document_connections WHERE id = ?').get(id) as ConnectionRow | undefined;
|
||||
}
|
||||
|
||||
listConnections(tripId: number): ConnectionRow[] {
|
||||
return this.db.connection
|
||||
.prepare('SELECT * FROM document_connections WHERE trip_id = ? ORDER BY id')
|
||||
.all(tripId) as ConnectionRow[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a stored row into what an adapter needs. Secrets are decrypted here
|
||||
* and nowhere else, so there is one place to audit and one place that could
|
||||
* leak them into a log. The ref can write back what the adapter earns, which
|
||||
* is what makes it a saved connection's ref; see `saveEarnedSecret`.
|
||||
*/
|
||||
toRef(row: ConnectionRow): DocumentConnectionRef {
|
||||
let settings: Record<string, string> = {};
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(row.settings || '{}');
|
||||
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
|
||||
if (typeof v === 'string') settings[k] = v;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
settings = {};
|
||||
}
|
||||
return {
|
||||
connectionId: row.id,
|
||||
createdAt: row.created_at,
|
||||
ownerId: row.owner_user_id,
|
||||
baseUrl: row.base_url,
|
||||
secrets: decryptSecrets(row.secrets),
|
||||
settings,
|
||||
allowInsecureTls: row.allow_insecure_tls === 1,
|
||||
saveSecret: (key, value) => this.saveEarnedSecret(row.id, key, value),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep a secret the provider earned itself rather than one typed into the
|
||||
* form, or drop it with null. DSM's device token is the one there is: without
|
||||
* it, a restart would leave every two-factor NAS waiting for its owner to
|
||||
* type a fresh code.
|
||||
*
|
||||
* It lives in the same encrypted blob as the form's secrets, under a key no
|
||||
* form field has, and that is what keeps it out of every response
|
||||
* (`maskSecrets` reports form fields only) and out of the form's reach
|
||||
* (`mergeSecrets` writes form fields only).
|
||||
*
|
||||
* The blob is read back inside the transaction rather than taken from the
|
||||
* caller's ref: a sync run holds its ref for minutes, and writing that
|
||||
* snapshot back would undo a password the owner changed in the meantime.
|
||||
*/
|
||||
saveEarnedSecret(connectionId: number, key: string, value: string | null): void {
|
||||
this.db.transaction(() => {
|
||||
const row = this.getConnection(connectionId);
|
||||
if (!row) return;
|
||||
const secrets = decryptSecrets(row.secrets);
|
||||
if (value === null) delete secrets[key];
|
||||
else secrets[key] = value;
|
||||
this.db.connection
|
||||
.prepare('UPDATE document_connections SET secrets = ? WHERE id = ?')
|
||||
.run(encryptSecrets(secrets), connectionId);
|
||||
});
|
||||
}
|
||||
|
||||
/** What a client may see: no secret values, only whether each one is set. */
|
||||
publicConnection(row: ConnectionRow): Record<string, unknown> {
|
||||
const ref = this.toRef(row);
|
||||
return {
|
||||
id: row.id,
|
||||
tripId: row.trip_id,
|
||||
providerId: row.provider_id,
|
||||
ownerUserId: row.owner_user_id,
|
||||
baseUrl: row.base_url,
|
||||
settings: ref.settings,
|
||||
secrets: maskSecrets(ref.secrets, this.secretKeys(row.provider_id)),
|
||||
allowInsecureTls: row.allow_insecure_tls === 1,
|
||||
capabilities: row.capabilities ? safeParse(row.capabilities) : null,
|
||||
lastProbeAt: row.last_probe_at,
|
||||
lastProbeState: row.last_probe_state,
|
||||
lastProbeError: row.last_probe_error,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a base URL before it is stored.
|
||||
*
|
||||
* A private address is stored WITH a warning rather than refused, because
|
||||
* self-hosting is the normal case here: a Paperless on 192.168.x is the
|
||||
* point of the feature, not an attack. That is the same call airtrail.service
|
||||
* and dawarich.service already made. What is refused is an address that the
|
||||
* SSRF guard rejects for a reason other than being private.
|
||||
*/
|
||||
async validateBaseUrl(raw: string): Promise<DocResult<{ url: string; isPrivate: boolean }>> {
|
||||
const trimmed = raw.trim().replace(/\/+$/, '');
|
||||
if (!trimmed) return { success: false, error: { code: 'unreachable', detail: 'empty url' } };
|
||||
const verdict = await checkSsrf(trimmed);
|
||||
if (!verdict.allowed && !verdict.isPrivate) {
|
||||
return { success: false, error: { code: 'ssrf_blocked', detail: verdict.error } };
|
||||
}
|
||||
return { success: true, data: { url: trimmed, isPrivate: verdict.isPrivate } };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or update the trip's connection for one provider.
|
||||
*
|
||||
* Secrets that arrive blank or masked keep their stored value, so a client
|
||||
* that renders the form from a GET never has to hold the real credential.
|
||||
* A secret the provider earned itself is kept too, whatever the form sends.
|
||||
* DSM's device token is stored with the address and account it was issued
|
||||
* for, so an edit that points the connection somewhere else leaves it
|
||||
* unusable, and the adapter drops it on its next call.
|
||||
*/
|
||||
async upsertConnection(
|
||||
tripId: number,
|
||||
userId: number,
|
||||
input: DocsyncConnectionInput,
|
||||
): Promise<DocResult<ConnectionRow>> {
|
||||
const urlCheck = await this.validateBaseUrl(input.baseUrl);
|
||||
if (docFailed(urlCheck)) return { success: false, error: urlCheck.error };
|
||||
|
||||
const fields = this.providerFields(input.providerId);
|
||||
const secretKeys = fields.filter((f) => f.secret === 1).map((f) => f.field_key);
|
||||
const plainKeys = fields.filter((f) => f.secret !== 1).map((f) => f.field_key);
|
||||
|
||||
const existing = this.db.connection
|
||||
.prepare('SELECT * FROM document_connections WHERE trip_id = ? AND provider_id = ?')
|
||||
.get(tripId, input.providerId) as ConnectionRow | undefined;
|
||||
|
||||
const storedSecrets = existing ? decryptSecrets(existing.secrets) : {};
|
||||
const nextSecrets = mergeSecrets(storedSecrets, input.credentials, secretKeys);
|
||||
|
||||
const settings: Record<string, string> = {};
|
||||
for (const key of plainKeys) {
|
||||
if (key === 'allow_insecure_tls') continue;
|
||||
const submitted = input.credentials[key];
|
||||
if (submitted !== undefined) settings[key] = submitted;
|
||||
else if (existing) {
|
||||
const prev = this.toRef(existing).settings[key];
|
||||
if (prev !== undefined) settings[key] = prev;
|
||||
}
|
||||
}
|
||||
|
||||
for (const f of fields) {
|
||||
if (f.required !== 1) continue;
|
||||
const present = f.secret === 1 ? !!nextSecrets[f.field_key] : !!settings[f.field_key] || f.field_key === 'base_url';
|
||||
if (!present) {
|
||||
return { success: false, error: { code: 'unauthorized', detail: `missing required field ${f.field_key}` } };
|
||||
}
|
||||
}
|
||||
|
||||
const encrypted = encryptSecrets(nextSecrets);
|
||||
const settingsJson = JSON.stringify(settings);
|
||||
const insecure = input.allowInsecureTls ? 1 : 0;
|
||||
|
||||
const row = this.db.transaction(() => {
|
||||
if (existing) {
|
||||
this.db.connection
|
||||
.prepare(
|
||||
`UPDATE document_connections
|
||||
SET base_url = ?, secrets = ?, settings = ?, allow_insecure_tls = ?,
|
||||
owner_user_id = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(urlCheck.data.url, encrypted, settingsJson, insecure, userId, existing.id);
|
||||
return this.getConnection(existing.id) as ConnectionRow;
|
||||
}
|
||||
const info = this.db.connection
|
||||
.prepare(
|
||||
`INSERT INTO document_connections (trip_id, provider_id, owner_user_id, base_url, secrets, settings, allow_insecure_tls)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(tripId, input.providerId, userId, urlCheck.data.url, encrypted, settingsJson, insecure);
|
||||
return this.getConnection(Number(info.lastInsertRowid)) as ConnectionRow;
|
||||
});
|
||||
|
||||
return { success: true, data: row };
|
||||
}
|
||||
|
||||
recordProbe(connectionId: number, state: 'ok' | 'failed', error: string | null, capabilities: unknown): void {
|
||||
this.db.connection
|
||||
.prepare(
|
||||
`UPDATE document_connections
|
||||
SET last_probe_at = CURRENT_TIMESTAMP, last_probe_state = ?, last_probe_error = ?,
|
||||
capabilities = COALESCE(?, capabilities), updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(state, error, capabilities ? JSON.stringify(capabilities) : null, connectionId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deleting a connection leaves every document in place. It unbinds, nothing
|
||||
* more: the same promise Dawarich's disconnect makes, and the only version
|
||||
* of this action that is safe to offer without a confirmation dialog.
|
||||
*/
|
||||
deleteConnection(id: number): void {
|
||||
this.db.connection.prepare('DELETE FROM document_connections WHERE id = ?').run(id);
|
||||
}
|
||||
|
||||
// ── Trip bindings ──────────────────────────────────────────────────────────
|
||||
|
||||
listLinks(tripId: number): LinkRow[] {
|
||||
return this.db.connection
|
||||
.prepare('SELECT * FROM trip_document_links WHERE trip_id = ? ORDER BY id')
|
||||
.all(tripId) as LinkRow[];
|
||||
}
|
||||
|
||||
getLink(id: number): LinkRow | undefined {
|
||||
return this.db.connection.prepare('SELECT * FROM trip_document_links WHERE id = ?').get(id) as LinkRow | undefined;
|
||||
}
|
||||
|
||||
getLinkByToken(token: string): LinkRow | undefined {
|
||||
return this.db.connection
|
||||
.prepare('SELECT * FROM trip_document_links WHERE webhook_token = ?')
|
||||
.get(token) as LinkRow | undefined;
|
||||
}
|
||||
|
||||
toScopeRef(link: LinkRow): DocumentScopeRef {
|
||||
return {
|
||||
linkId: link.id,
|
||||
tripId: link.trip_id,
|
||||
scopeKey: link.remote_scope_key,
|
||||
remoteRootId: link.remote_root_id,
|
||||
remoteRootPath: link.remote_root_path,
|
||||
cursor: link.remote_cursor,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The anchor fields (`remoteRootId`, `remoteRootPath`, `remoteLabel`) are
|
||||
* what the picker was handed by `listScopes` or `createScope` a moment
|
||||
* earlier, sent back by the client. Taking them on trust is fine: only the
|
||||
* trip owner or an instance admin gets here, and an anchor typed in by hand
|
||||
* still goes through the connection's own credential, so it cannot open
|
||||
* anything that credential could not open anyway.
|
||||
*/
|
||||
createLink(tripId: number, userId: number, input: DocsyncLinkInput): DocResult<LinkRow> {
|
||||
const conn = this.getConnection(input.connectionId);
|
||||
if (!conn || conn.trip_id !== tripId) {
|
||||
return { success: false, error: { code: 'not_found', detail: 'connection not found for this trip' } };
|
||||
}
|
||||
const existing = this.db.connection
|
||||
.prepare('SELECT * FROM trip_document_links WHERE trip_id = ? AND connection_id = ? AND remote_scope_key = ?')
|
||||
.get(tripId, input.connectionId, input.scopeKey) as LinkRow | undefined;
|
||||
if (existing) return { success: true, data: existing };
|
||||
|
||||
// A per-link token, so revoking one binding cannot be used to poke another,
|
||||
// and so a leaked webhook URL identifies exactly one trip.
|
||||
const token = crypto.randomBytes(24).toString('base64url');
|
||||
const secret = crypto.randomBytes(24).toString('base64url');
|
||||
|
||||
const info = this.db.connection
|
||||
.prepare(
|
||||
`INSERT INTO trip_document_links
|
||||
(trip_id, connection_id, provider_id, remote_scope_key, remote_root_id, remote_root_path,
|
||||
remote_label, direction, delete_policy, conflict_policy, sync_enabled,
|
||||
webhook_token, webhook_secret, created_by, next_attempt_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)`,
|
||||
)
|
||||
.run(
|
||||
tripId,
|
||||
input.connectionId,
|
||||
conn.provider_id,
|
||||
input.scopeKey,
|
||||
input.remoteRootId ?? null,
|
||||
input.remoteRootPath ?? null,
|
||||
input.remoteLabel ?? '',
|
||||
input.direction,
|
||||
input.deletePolicy,
|
||||
input.conflictPolicy,
|
||||
input.syncEnabled ? 1 : 0,
|
||||
token,
|
||||
String(encryptSecrets({ webhook: secret })),
|
||||
userId,
|
||||
);
|
||||
return { success: true, data: this.getLink(Number(info.lastInsertRowid)) as LinkRow };
|
||||
}
|
||||
|
||||
updateLink(id: number, patch: Partial<DocsyncLinkInput>): LinkRow | undefined {
|
||||
const sets: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
if (patch.direction !== undefined) { sets.push('direction = ?'); values.push(patch.direction); }
|
||||
if (patch.deletePolicy !== undefined) { sets.push('delete_policy = ?'); values.push(patch.deletePolicy); }
|
||||
if (patch.conflictPolicy !== undefined) { sets.push('conflict_policy = ?'); values.push(patch.conflictPolicy); }
|
||||
if (patch.syncEnabled !== undefined) { sets.push('sync_enabled = ?'); values.push(patch.syncEnabled ? 1 : 0); }
|
||||
if (patch.remoteLabel !== undefined) { sets.push('remote_label = ?'); values.push(patch.remoteLabel); }
|
||||
if (sets.length === 0) return this.getLink(id);
|
||||
sets.push('updated_at = CURRENT_TIMESTAMP');
|
||||
this.db.connection.prepare(`UPDATE trip_document_links SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
|
||||
return this.getLink(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unbinding keeps both copies and drops the pairing rows. It never deletes a
|
||||
* document on either side. A user who wants that does it deliberately, in
|
||||
* the system that holds the file.
|
||||
*/
|
||||
deleteLink(id: number): void {
|
||||
this.db.transaction(() => {
|
||||
this.db.connection.prepare('DELETE FROM document_sync_items WHERE link_id = ?').run(id);
|
||||
this.db.connection.prepare('DELETE FROM trip_document_links WHERE id = ?').run(id);
|
||||
});
|
||||
}
|
||||
|
||||
webhookSecret(link: LinkRow): string {
|
||||
return decryptSecrets(link.webhook_secret).webhook ?? '';
|
||||
}
|
||||
|
||||
/** The masked view a client gets, with the webhook URL it may need to paste. */
|
||||
publicLink(link: LinkRow, webhookBaseUrl: string | null): Record<string, unknown> {
|
||||
return {
|
||||
id: link.id,
|
||||
tripId: link.trip_id,
|
||||
connectionId: link.connection_id,
|
||||
providerId: link.provider_id,
|
||||
providerName: this.providerName(link.provider_id),
|
||||
scopeKey: link.remote_scope_key,
|
||||
remoteRootId: link.remote_root_id,
|
||||
remoteRootPath: link.remote_root_path,
|
||||
remoteLabel: link.remote_label,
|
||||
direction: link.direction,
|
||||
deletePolicy: link.delete_policy,
|
||||
conflictPolicy: link.conflict_policy,
|
||||
syncEnabled: link.sync_enabled === 1,
|
||||
lastSyncAt: link.last_sync_at,
|
||||
lastSyncState: link.last_sync_state,
|
||||
lastSyncError: link.last_sync_error,
|
||||
failureCount: link.failure_count,
|
||||
// Shown so a user can paste it into a provider that will not let TREK
|
||||
// subscribe on its own (Papra, and Nextcloud without admin rights).
|
||||
webhookUrl: webhookBaseUrl && link.webhook_token
|
||||
? `${webhookBaseUrl}/api/docsync/webhook/${link.webhook_token}`
|
||||
: null,
|
||||
webhookSecret: link.webhook_secret ? DOCSYNC_SECRET_MASK : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bindings whose credential owner is no longer on the trip.
|
||||
*
|
||||
* Checked on every run rather than hooked to member removal, because a
|
||||
* membership can also disappear through a trip transfer or a direct DB edit,
|
||||
* and a binding that keeps using an ex-member's token is the kind of thing
|
||||
* nobody notices until it is a complaint.
|
||||
*/
|
||||
markOrphanedLinks(): number {
|
||||
const info = this.db.connection
|
||||
.prepare(
|
||||
`UPDATE trip_document_links
|
||||
SET last_sync_state = 'orphaned', sync_enabled = 0, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE sync_enabled = 1
|
||||
AND last_sync_state != 'orphaned'
|
||||
AND connection_id IN (
|
||||
SELECT c.id FROM document_connections c
|
||||
WHERE c.owner_user_id NOT IN (
|
||||
SELECT tm.user_id FROM trip_members tm WHERE tm.trip_id = c.trip_id
|
||||
UNION SELECT t.user_id FROM trips t WHERE t.id = c.trip_id
|
||||
)
|
||||
)`,
|
||||
)
|
||||
.run();
|
||||
return info.changes;
|
||||
}
|
||||
}
|
||||
|
||||
function safeParse(value: string): unknown {
|
||||
try {
|
||||
return JSON.parse(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { DOCSYNC_SECRET_MASK } from '@trek/shared';
|
||||
import { decrypt_api_key, maybe_encrypt_api_key } from '../common/crypto/apiKeyCrypto';
|
||||
|
||||
/**
|
||||
* Credential handling for document connections.
|
||||
*
|
||||
* All secret fields of a connection live in ONE encrypted JSON column rather
|
||||
* than a column per provider. A sixth provider then needs no migration, and the
|
||||
* key rotation in scripts/migrate-encryption.ts stays a single line instead of
|
||||
* a field list somebody will extend and forget. A forgotten column does not
|
||||
* survive a rotation, and the failure only shows up months later as a
|
||||
* connection that cannot authenticate any more.
|
||||
*/
|
||||
|
||||
export type SecretMap = Record<string, string>;
|
||||
|
||||
/** Encrypt the whole secret map for storage. Empty maps store as NULL. */
|
||||
export function encryptSecrets(secrets: SecretMap): string | null {
|
||||
const entries = Object.entries(secrets).filter(([, v]) => typeof v === 'string' && v.length > 0);
|
||||
if (entries.length === 0) return null;
|
||||
return String(maybe_encrypt_api_key(JSON.stringify(Object.fromEntries(entries))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrypt for use. A value that cannot be decrypted returns an empty map rather
|
||||
* than throwing: the caller then reports `unauthorized` and the user re-enters
|
||||
* the credential, which is the recoverable outcome. That is the same choice
|
||||
* synology.service.ts makes for a stale SID: a decryption failure after an
|
||||
* ENCRYPTION_KEY rotation must not turn into a boot-time crash.
|
||||
*/
|
||||
export function decryptSecrets(stored: string | null | undefined): SecretMap {
|
||||
if (!stored) return {};
|
||||
const plain = decrypt_api_key(stored);
|
||||
if (typeof plain !== 'string' || plain.length === 0) return {};
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(plain);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {};
|
||||
const out: SecretMap = {};
|
||||
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
|
||||
if (typeof v === 'string') out[k] = v;
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge submitted form values over the stored ones.
|
||||
*
|
||||
* A secret field that arrives blank, absent, or as the mask keeps its stored
|
||||
* value. That rule is what lets the client render a connection form without
|
||||
* ever holding the secret it was given back, the same contract the photo
|
||||
* provider form and Dawarich already use, so the generic component needs no
|
||||
* special case here.
|
||||
*
|
||||
* A stored key that is no form field is a secret the provider earned itself
|
||||
* (DSM's device token). It is carried over untouched and cannot be set from
|
||||
* here: the form never saw it, so an edit can neither wipe it nor plant one.
|
||||
*/
|
||||
export function mergeSecrets(stored: SecretMap, submitted: Record<string, string>, secretKeys: readonly string[]): SecretMap {
|
||||
const next: SecretMap = { ...stored };
|
||||
for (const key of secretKeys) {
|
||||
const value = submitted[key];
|
||||
if (value === undefined) continue;
|
||||
if (value === '' || value === DOCSYNC_SECRET_MASK) continue;
|
||||
next[key] = value;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a client is allowed to see: whether a secret is set, never its value.
|
||||
* Form fields only, so a secret the provider earned itself is not even
|
||||
* reported as present.
|
||||
*/
|
||||
export function maskSecrets(secrets: SecretMap, secretKeys: readonly string[]): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const key of secretKeys) {
|
||||
if (secrets[key]) out[key] = DOCSYNC_SECRET_MASK;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { Controller, HttpCode, OnModuleDestroy, Param, Post, Req } from '@nestjs/common';
|
||||
import type { Request } from 'express';
|
||||
import crypto from 'crypto';
|
||||
import { Public } from '../auth/public.decorator';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { SETTING_SYNC_ENABLED, WEBHOOK_NUDGE_DEBOUNCE_SECONDS } from './doc-sync.constants';
|
||||
import { DocSyncConfigService, type LinkRow } from './doc-sync-config.service';
|
||||
import { DocSyncService } from './doc-sync.service';
|
||||
|
||||
/**
|
||||
* `/api/docsync/webhook/:token`: the one endpoint a provider calls.
|
||||
*
|
||||
* Deliberately the thinnest thing in the domain. It answers 200 and schedules a
|
||||
* run; it never reads the body as truth. Every provider here has a different
|
||||
* payload, none of them signs it in a way all five share, Paperless gives its
|
||||
* webhook five seconds before it retries, and Nextcloud's payload carries a
|
||||
* path that is different for every member of a share. Treating any of that as
|
||||
* data would mean trusting an unauthenticated stranger's description of what
|
||||
* changed. So the webhook means exactly one thing: look now.
|
||||
*
|
||||
* `@Public` because a provider cannot hold a TREK session. The token in the URL
|
||||
* is the authentication, one per binding, so a leaked URL can only ever nudge
|
||||
* the one trip it belongs to, and nudging is all it can do. Where the provider
|
||||
* supports it, a shared secret is checked as well.
|
||||
*/
|
||||
@Controller('api/docsync/webhook')
|
||||
export class DocSyncWebhookController implements OnModuleDestroy {
|
||||
/**
|
||||
* One pending run per binding, so a burst folds into a single pass.
|
||||
*
|
||||
* Providers fire per document: dropping twenty files into a watched Nextcloud
|
||||
* folder is twenty calls within a second or two. Each one used to start its
|
||||
* own run, which the service's in-flight guard then answered with `busy`,
|
||||
* so nineteen changes were announced and thrown away, and the one run that
|
||||
* did start had begun before most of them landed. Collecting them for a beat
|
||||
* and then running once is both less work and more correct.
|
||||
*/
|
||||
private readonly pending = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
constructor(
|
||||
private readonly config: DocSyncConfigService,
|
||||
private readonly sync: DocSyncService,
|
||||
private readonly db: DatabaseService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* The same switches the scheduler obeys: the Documents addon, the binding's
|
||||
* provider, and the instance-wide kill switch.
|
||||
*
|
||||
* Without them "switch document sync off" meant "stop the poll", while every
|
||||
* provider holding a webhook kept driving full runs: an admin turning the
|
||||
* addon off would have watched it carry on. Checked when the timer fires as
|
||||
* well as on arrival, so a switch thrown during the debounce window still
|
||||
* takes effect.
|
||||
*/
|
||||
private syncIsOn(link: LinkRow): boolean {
|
||||
if (this.sync.isSwitchedOff(link)) return false;
|
||||
const killSwitch = this.db.get<{ value: string }>(
|
||||
'SELECT value FROM app_settings WHERE key = ?', SETTING_SYNC_ENABLED,
|
||||
)?.value;
|
||||
// Unrecognised values mean ON: the setting is absent by default, and only an
|
||||
// explicit 'false' stops the sync. Same rule as the job.
|
||||
return killSwitch !== 'false';
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
for (const timer of this.pending.values()) clearTimeout(timer);
|
||||
this.pending.clear();
|
||||
}
|
||||
|
||||
@Post(':token')
|
||||
@Public('A provider cannot hold a TREK session; the per-link token in the URL is the authentication, and the call can only ever trigger a sync run.')
|
||||
@HttpCode(200)
|
||||
nudge(@Param('token') token: string, @Req() req: Request) {
|
||||
const link = this.config.getLinkByToken(token);
|
||||
// Always 200, even for an unknown token: a 404 here would let anyone probe
|
||||
// which tokens exist, and a provider that gets an error will retry anyway.
|
||||
if (!link || link.sync_enabled !== 1) return { received: true };
|
||||
if (!this.syncIsOn(link)) return { received: true };
|
||||
|
||||
const secret = this.config.webhookSecret(link);
|
||||
if (secret && !this.secretMatches(req, secret)) return { received: true };
|
||||
|
||||
// Fire and forget. Paperless allows five seconds before it counts the call
|
||||
// as failed and retries, and a sync run takes longer than that whenever
|
||||
// there is anything to do, so the answer goes out now and the run happens
|
||||
// after the debounce window, by which time the rest of the burst has
|
||||
// arrived and been folded into this same timer.
|
||||
this.schedule(link.id, () => this.config.getLink(link.id));
|
||||
return { received: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Start one run per binding per window, trailing rather than leading.
|
||||
*
|
||||
* Trailing on purpose: the first call of a burst is the least informed one,
|
||||
* because the provider is usually still writing the rest. The link is looked
|
||||
* up again when the timer fires, so a binding switched off or deleted in the
|
||||
* meantime does not get one last run out of a stale row.
|
||||
*/
|
||||
private schedule(linkId: number, reload: () => ReturnType<DocSyncConfigService['getLink']>, isRetry = false): void {
|
||||
if (this.pending.has(linkId)) return;
|
||||
const timer = setTimeout(() => {
|
||||
this.pending.delete(linkId);
|
||||
const fresh = reload();
|
||||
if (!fresh || fresh.sync_enabled !== 1) return;
|
||||
if (!this.syncIsOn(fresh)) return;
|
||||
void this.sync.syncLink(fresh).then((res) => {
|
||||
// A run that was already in flight answers `busy`, and the changes this
|
||||
// nudge was about may have landed after that run read the folder. Ask
|
||||
// again once rather than waiting out a whole poll interval: once, and
|
||||
// only for `busy`, so this cannot become a loop.
|
||||
if (res?.state === 'busy' && !isRetry) this.schedule(linkId, reload, true);
|
||||
});
|
||||
}, WEBHOOK_NUDGE_DEBOUNCE_SECONDS * 1000);
|
||||
// A pending nudge must not hold the process open at shutdown.
|
||||
if (typeof timer.unref === 'function') timer.unref();
|
||||
this.pending.set(linkId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Accept either a plain shared-secret header (Paperless workflows, Nextcloud
|
||||
* `authMethod: header`) or Papra's standard-webhooks HMAC. Compared in
|
||||
* constant time, and a mismatch is silently ignored rather than reported, so
|
||||
* the endpoint tells a prober nothing either way.
|
||||
*/
|
||||
private secretMatches(req: Request, secret: string): boolean {
|
||||
const header = req.get('x-trek-docsync-secret');
|
||||
if (header && timingSafeEqualStr(header, secret)) return true;
|
||||
|
||||
const sig = req.get('webhook-signature');
|
||||
const id = req.get('webhook-id');
|
||||
const ts = req.get('webhook-timestamp');
|
||||
if (sig && id && ts) {
|
||||
const raw = (req as Request & { rawBody?: Buffer }).rawBody;
|
||||
const body = raw ? raw.toString('utf8') : JSON.stringify(req.body ?? {});
|
||||
const expected = crypto
|
||||
.createHmac('sha256', Buffer.from(secret))
|
||||
.update(`${id}.${ts}.${body}`)
|
||||
.digest('base64');
|
||||
for (const part of sig.split(' ')) {
|
||||
const value = part.startsWith('v1,') ? part.slice(3) : part;
|
||||
if (timingSafeEqualStr(value, expected)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function timingSafeEqualStr(a: string, b: string): boolean {
|
||||
const ab = Buffer.from(a);
|
||||
const bb = Buffer.from(b);
|
||||
if (ab.length !== bb.length) return false;
|
||||
return crypto.timingSafeEqual(ab, bb);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* Tuning constants for the document sync core.
|
||||
*
|
||||
* Kept out of the service so the pure planner can be unit-tested against them
|
||||
* without a container, and so a reviewer can see every limit in one place
|
||||
* instead of hunting for magic numbers.
|
||||
*/
|
||||
|
||||
/** Per-request ceiling for provider calls that return metadata. */
|
||||
export const PROVIDER_TIMEOUT_MS = 15_000;
|
||||
|
||||
/** Transfers get longer, because a 40 MB PDF over a home upstream is not fast. */
|
||||
export const PROVIDER_TRANSFER_TIMEOUT_MS = 120_000;
|
||||
|
||||
/** Metadata responses are JSON; nothing legitimate here is megabytes. */
|
||||
export const PROVIDER_JSON_MAX_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Page cap per listing run, mirroring the reason dawarich.client.ts states: a
|
||||
* provider that keeps answering `next` must not hold a worker forever. Hitting
|
||||
* it sets `truncated`, which the core surfaces rather than silently treating a
|
||||
* partial listing as the whole truth.
|
||||
*/
|
||||
export const PROVIDER_MAX_PAGES = 40;
|
||||
|
||||
/**
|
||||
* The share of a link's known documents that may disappear in one run before
|
||||
* the run refuses to act on any of it.
|
||||
*
|
||||
* An unmounted Synology share, a revoked Nextcloud share and an expired
|
||||
* Paperless token all produce the same thing: a listing far shorter than the
|
||||
* last one. Without this guard the reconciler would read that as a mass
|
||||
* deletion. Below the threshold, vanished documents are recorded as
|
||||
* `remote_missing` and wait for a human either way. This only decides whether
|
||||
* the run trusts the listing enough to record anything at all.
|
||||
*/
|
||||
export const MASS_DELETE_RATIO = 0.4;
|
||||
|
||||
/** Below this count the ratio is meaningless, so the guard only counts. */
|
||||
export const MASS_DELETE_MIN_ITEMS = 5;
|
||||
|
||||
/**
|
||||
* Backoff for a failing link, in seconds, indexed by consecutive failures.
|
||||
* Flat 30 minutes after the sixth, because a provider that has been down for
|
||||
* three hours is a configuration problem, not a blip, and hammering it earns a
|
||||
* DSM auto-block.
|
||||
*/
|
||||
export const LINK_BACKOFF_SECONDS = [60, 300, 900, 1800, 3600, 7200] as const;
|
||||
|
||||
/** Per-document retry curve, shorter: usually a transient upload failure. */
|
||||
export const ITEM_BACKOFF_SECONDS = [30, 120, 600, 3600] as const;
|
||||
|
||||
/** After this many attempts a document stops retrying and waits for a human. */
|
||||
export const ITEM_MAX_ATTEMPTS = 6;
|
||||
|
||||
/**
|
||||
* Consecutive link failures after which the circuit opens and the job stops
|
||||
* scheduling it at all until someone presses "sync now".
|
||||
*/
|
||||
export const LINK_CIRCUIT_OPEN_AFTER = 10;
|
||||
|
||||
/** Default poll interval. Clamped, overridable per instance in app_settings. */
|
||||
export const DEFAULT_POLL_INTERVAL_SECONDS = 300;
|
||||
export const MIN_POLL_INTERVAL_SECONDS = 60;
|
||||
export const MAX_POLL_INTERVAL_SECONDS = 3600;
|
||||
|
||||
/**
|
||||
* How long a webhook may accelerate a link before the poll takes over again.
|
||||
* A webhook only ever means "look now". This keeps a link whose subscription
|
||||
* silently died from going stale forever.
|
||||
*/
|
||||
export const WEBHOOK_NUDGE_DEBOUNCE_SECONDS = 5;
|
||||
|
||||
/** Documents pulled or pushed per run, so one huge trip cannot starve others. */
|
||||
export const MAX_TRANSFERS_PER_RUN = 25;
|
||||
|
||||
/**
|
||||
* What a manual run answers for a binding an admin has switched off, through
|
||||
* its provider or through the Documents addon.
|
||||
*
|
||||
* Not one of the shared DocsyncErrorCodes: those say what went wrong between
|
||||
* TREK and a provider, and here nothing has. It is never written to the binding
|
||||
* either, which is what lets the binding resume as it was left.
|
||||
*/
|
||||
export const PROVIDER_DISABLED = 'provider_disabled';
|
||||
|
||||
/** app_settings keys, read per tick so an admin toggle needs no restart. */
|
||||
export const SETTING_POLL_INTERVAL = 'docsync_poll_interval_seconds';
|
||||
export const SETTING_SYNC_ENABLED = 'docsync_sync_enabled';
|
||||
// There is deliberately no size key here: a document that arrives through a
|
||||
// binding is bound by the same ceiling as one somebody uploads (MAX_FILE_SIZE in
|
||||
// files.constants), checked against the announced size and again while the
|
||||
// bytes stream. A second, sync-only limit would be a number nobody maintains
|
||||
// and a way for the two paths to disagree about what fits.
|
||||
@@ -0,0 +1,439 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
HttpCode,
|
||||
HttpException,
|
||||
Param,
|
||||
Patch,
|
||||
Post,
|
||||
Put,
|
||||
Query,
|
||||
Req,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import type { Request } from 'express';
|
||||
import type { User } from '../../types';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { AddonGuard } from '../addons/addon.guard';
|
||||
import { RequireAddon } from '../addons/require-addon.decorator';
|
||||
import { JwtAuthGuard } from '../auth/jwt-auth.guard';
|
||||
import { CurrentUser } from '../auth/current-user.decorator';
|
||||
import { TripAccessGuard } from '../permissions/trip-access.guard';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import { docFailed } from './document-provider';
|
||||
import { DocSyncConfigService, type LinkRow } from './doc-sync-config.service';
|
||||
import { DocSyncService } from './doc-sync.service';
|
||||
import { DocumentProviderRegistry } from './document-provider.registry';
|
||||
import { PROVIDER_DISABLED } from './doc-sync.constants';
|
||||
import {
|
||||
DocsyncConnectionDto,
|
||||
DocsyncConnectionTestDto,
|
||||
DocsyncLinkDto,
|
||||
DocsyncLinkUpdateDto,
|
||||
DocsyncResolveConflictDto,
|
||||
DocsyncScopeCreateDto,
|
||||
DocsyncSyncNowDto,
|
||||
} from './doc-sync.dto';
|
||||
|
||||
/**
|
||||
* `/api/trips/:tripId/docsync`: a trip's document provider binding.
|
||||
*
|
||||
* Trip-scoped rather than user-scoped, and that is the whole point of the
|
||||
* feature: one connection per trip, which every member reads through. The
|
||||
* routes therefore sit under the trip, not under `/api/integrations`.
|
||||
*
|
||||
* Two shapes of answer, the split Dawarich already established: connection
|
||||
* routes (`test`, `status`) answer 200 even when the provider is unreachable,
|
||||
* with the reason in the body, because a form showing someone their own typo
|
||||
* needs a field to render rather than an exception. Everything else answers the
|
||||
* status the failure deserves.
|
||||
*
|
||||
* Only the trip owner may change a binding (plus an instance admin, who
|
||||
* overrides trip permissions everywhere else too). It hands TREK a credential
|
||||
* that usually reaches the owner's entire document archive, so widening this to
|
||||
* every member would let any member point the trip at a folder the owner never
|
||||
* meant to share. Reading the status is open to all members: they need to know
|
||||
* where their documents are going.
|
||||
*/
|
||||
@Controller('api/trips/:tripId/docsync')
|
||||
@UseGuards(AddonGuard, JwtAuthGuard, TripAccessGuard)
|
||||
@RequireAddon(ADDON_IDS.DOCUMENTS, 'Documents')
|
||||
export class DocSyncController {
|
||||
constructor(
|
||||
private readonly config: DocSyncConfigService,
|
||||
private readonly sync: DocSyncService,
|
||||
private readonly registry: DocumentProviderRegistry,
|
||||
private readonly db: DatabaseService,
|
||||
private readonly realtime: RealtimeService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Tell the trip that a binding came or went.
|
||||
*
|
||||
* Whether a trip is bound decides whether its members see the sync button
|
||||
* at all, and a run only pings when it moved a document. Binding two empty
|
||||
* sides, or unbinding, moves nothing, so a member sitting on the Files tab
|
||||
* kept the old answer until they left it. The whole room, the caller
|
||||
* included: it is a ping to re-read, not a change to apply, and the owner's
|
||||
* own button depends on it too once every provider is switched off.
|
||||
*/
|
||||
private announceBinding(link: Pick<LinkRow, 'id' | 'trip_id'>): void {
|
||||
this.realtime.broadcast(link.trip_id, 'docsync:changed', { linkId: link.id, pulled: 0, pushed: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
* The trip owner or an instance admin, the same rule as `canManageDocSync`
|
||||
* on the client. Checked here rather than through @RequirePermission: the
|
||||
* permission table has no "manage integrations" action and inventing one
|
||||
* would grant it to every trip_member by default, which is the opposite of
|
||||
* what a credential this broad needs.
|
||||
*
|
||||
* The 403 still says "owner": an admin never sees it, and to everybody who
|
||||
* does, the owner is the person to ask.
|
||||
*/
|
||||
private assertCanManage(tripId: string, user: User): void {
|
||||
if (user.role === 'admin') return;
|
||||
const trip = this.db.get<{ user_id: number }>('SELECT user_id FROM trips WHERE id = ?', tripId);
|
||||
if (!trip) throw new HttpException('Trip not found', 404);
|
||||
if (Number(trip.user_id) !== Number(user.id)) {
|
||||
throw new HttpException('Only the trip owner can change document sync', 403);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Providers and status ───────────────────────────────────────────────────
|
||||
|
||||
/** Which providers this instance has switched on, with their form fields. */
|
||||
@Get('providers')
|
||||
providers() {
|
||||
const rows = this.db.connection
|
||||
.prepare('SELECT id, name, description, icon FROM document_providers WHERE enabled = 1 ORDER BY sort_order')
|
||||
.all() as Array<{ id: string; name: string; description: string | null; icon: string }>;
|
||||
return rows.map((p) => ({
|
||||
...p,
|
||||
// A provider row with no registered adapter would render a form that
|
||||
// cannot work, so it is reported rather than hidden.
|
||||
available: !!this.registry.get(p.id),
|
||||
fields: this.config.providerFields(p.id).map((f) => ({ ...f, secret: f.secret === 1, required: f.required === 1 })),
|
||||
}));
|
||||
}
|
||||
|
||||
@Get('status')
|
||||
status(@Param('tripId') tripId: string) {
|
||||
return this.sync.status(Number(tripId));
|
||||
}
|
||||
|
||||
// ── Connections ────────────────────────────────────────────────────────────
|
||||
|
||||
@Get('connections')
|
||||
listConnections(@Param('tripId') tripId: string) {
|
||||
return this.config.listConnections(Number(tripId)).map((c) => this.config.publicConnection(c));
|
||||
}
|
||||
|
||||
@Put('connections')
|
||||
@HttpCode(200)
|
||||
async upsertConnection(
|
||||
@Param('tripId') tripId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncConnectionDto,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
if (!this.config.enabledProviderIds().includes(body.providerId)) {
|
||||
throw new HttpException(`Provider: "${body.providerId}" is not enabled, contact server administrator`, 400);
|
||||
}
|
||||
const res = await this.config.upsertConnection(Number(tripId), Number(user.id), body);
|
||||
if (docFailed(res)) throw new HttpException(res.error.detail || res.error.code, 400);
|
||||
return this.config.publicConnection(res.data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe form values. Always 200: the result is in the body, because this is
|
||||
* a form field, not a failure of the request. The photo providers pin the
|
||||
* same contract with a test their e2e suite marks CRITICAL.
|
||||
*/
|
||||
@Post('connections/test')
|
||||
@HttpCode(200)
|
||||
async testConnection(
|
||||
@Param('tripId') tripId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncConnectionTestDto,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const provider = this.registry.get(body.providerId);
|
||||
if (!provider) return { connected: false, error: 'unknown_provider' };
|
||||
|
||||
const urlCheck = await this.config.validateBaseUrl(body.baseUrl);
|
||||
if (docFailed(urlCheck)) return { connected: false, error: urlCheck.error.code };
|
||||
|
||||
// Probing with the values on screen, merged over whatever is stored, so a
|
||||
// user testing an unchanged connection does not have to retype the secret.
|
||||
const existing = this.config
|
||||
.listConnections(Number(tripId))
|
||||
.find((c) => c.provider_id === body.providerId);
|
||||
// Only for the address the credential was stored against. Merging it into a
|
||||
// probe of an arbitrary baseUrl turns this route into a way to have TREK
|
||||
// post a stored API token at a server of the caller's choosing, which is
|
||||
// exactly what somebody who inherited a trip but not its credentials would
|
||||
// reach for. Same host, same scheme, same port, or the caller types it in.
|
||||
const sameTarget = !!existing && sameOrigin(existing.base_url, urlCheck.data.url);
|
||||
const stored = sameTarget && existing ? this.config.toRef(existing) : null;
|
||||
const fields = this.config.providerFields(body.providerId);
|
||||
const secrets: Record<string, string> = { ...(stored?.secrets ?? {}) };
|
||||
const settings: Record<string, string> = { ...(stored?.settings ?? {}) };
|
||||
for (const f of fields) {
|
||||
const submitted = body.credentials[f.field_key];
|
||||
if (submitted === undefined || submitted === '') continue;
|
||||
if (f.secret === 1) secrets[f.field_key] = submitted;
|
||||
else settings[f.field_key] = submitted;
|
||||
}
|
||||
|
||||
const res = await provider.probe({
|
||||
connectionId: existing?.id ?? 0,
|
||||
createdAt: existing?.created_at ?? '',
|
||||
ownerId: Number(user.id),
|
||||
baseUrl: urlCheck.data.url,
|
||||
secrets,
|
||||
settings,
|
||||
allowInsecureTls: body.allowInsecureTls,
|
||||
});
|
||||
if (docFailed(res)) return { connected: false, error: res.error.code, detail: res.error.detail };
|
||||
if (existing) this.config.recordProbe(existing.id, 'ok', null, res.data.capabilities);
|
||||
return { connected: true, account: res.data.account, capabilities: res.data.capabilities };
|
||||
}
|
||||
|
||||
@Delete('connections/:connectionId')
|
||||
@HttpCode(200)
|
||||
deleteConnection(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('connectionId') connectionId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const conn = this.config.getConnection(Number(connectionId));
|
||||
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
|
||||
// Its bindings go with it (ON DELETE CASCADE), so read them first.
|
||||
const unbound = this.config.listLinks(conn.trip_id).filter((l) => l.connection_id === conn.id);
|
||||
this.config.deleteConnection(conn.id);
|
||||
for (const link of unbound) this.announceBinding(link);
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
// ── Scopes ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** Folders, tags or spaces the trip could be bound to. */
|
||||
@Get('connections/:connectionId/scopes')
|
||||
async listScopes(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('connectionId') connectionId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Query('q') q?: string,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const conn = this.config.getConnection(Number(connectionId));
|
||||
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
|
||||
const provider = this.registry.get(conn.provider_id);
|
||||
if (!provider) throw new HttpException('Provider not available', 400);
|
||||
const res = await provider.listScopes(this.config.toRef(conn), q);
|
||||
if (docFailed(res)) return { scopes: [], error: res.error.code };
|
||||
return { scopes: res.data };
|
||||
}
|
||||
|
||||
@Post('connections/:connectionId/scopes')
|
||||
async createScope(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('connectionId') connectionId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncScopeCreateDto,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const conn = this.config.getConnection(Number(connectionId));
|
||||
if (!conn || conn.trip_id !== Number(tripId)) throw new HttpException('Connection not found', 404);
|
||||
const provider = this.registry.get(conn.provider_id);
|
||||
if (!provider) throw new HttpException('Provider not available', 400);
|
||||
const res = await provider.createScope(this.config.toRef(conn), body.name);
|
||||
if (docFailed(res)) throw new HttpException(res.error.code, 400);
|
||||
return res.data;
|
||||
}
|
||||
|
||||
// ── Bindings ───────────────────────────────────────────────────────────────
|
||||
|
||||
@Get('links')
|
||||
listLinks(@Param('tripId') tripId: string, @Req() req: Request) {
|
||||
const base = publicOrigin(req);
|
||||
return this.config.listLinks(Number(tripId)).map((l) => this.config.publicLink(l, base));
|
||||
}
|
||||
|
||||
@Post('links')
|
||||
@HttpCode(200)
|
||||
async createLink(
|
||||
@Param('tripId') tripId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncLinkDto,
|
||||
@Req() req: Request,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const res = this.config.createLink(Number(tripId), Number(user.id), body);
|
||||
if (docFailed(res)) throw new HttpException(res.error.detail || res.error.code, 400);
|
||||
this.announceBinding(res.data);
|
||||
|
||||
// Subscribe where the provider lets TREK do it itself. A failure here is
|
||||
// not a failure of the binding (polling still carries it), so it is logged
|
||||
// into the link state rather than thrown at the user.
|
||||
const conn = this.config.getConnection(res.data.connection_id);
|
||||
const provider = conn ? this.registry.get(conn.provider_id) : undefined;
|
||||
if (conn && provider?.registerWebhook) {
|
||||
const base = publicOrigin(req);
|
||||
if (base) {
|
||||
const hook = await provider.registerWebhook(
|
||||
this.config.toRef(conn),
|
||||
this.config.toScopeRef(res.data),
|
||||
`${base}/api/docsync/webhook/${res.data.webhook_token}`,
|
||||
this.config.webhookSecret(res.data),
|
||||
);
|
||||
if (hook.success) {
|
||||
this.db.connection
|
||||
.prepare('UPDATE trip_document_links SET webhook_subscription_id = ? WHERE id = ?')
|
||||
.run(hook.data.subscriptionId, res.data.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A first run right away, so the user sees something happen instead of
|
||||
// waiting out a poll interval and wondering whether it worked.
|
||||
void this.sync.syncLink(res.data, { full: true });
|
||||
return this.config.publicLink(this.config.getLink(res.data.id) ?? res.data, publicOrigin(req));
|
||||
}
|
||||
|
||||
@Patch('links/:linkId')
|
||||
@HttpCode(200)
|
||||
updateLink(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('linkId') linkId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncLinkUpdateDto,
|
||||
@Req() req: Request,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const link = this.config.getLink(Number(linkId));
|
||||
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
|
||||
const updated = this.config.updateLink(link.id, body);
|
||||
return this.config.publicLink(updated ?? link, publicOrigin(req));
|
||||
}
|
||||
|
||||
@Delete('links/:linkId')
|
||||
@HttpCode(200)
|
||||
async deleteLink(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('linkId') linkId: string,
|
||||
@CurrentUser() user: User,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const link = this.config.getLink(Number(linkId));
|
||||
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
|
||||
|
||||
const conn = this.config.getConnection(link.connection_id);
|
||||
const provider = conn ? this.registry.get(conn.provider_id) : undefined;
|
||||
if (conn && provider?.unregisterWebhook && link.webhook_subscription_id) {
|
||||
await provider.unregisterWebhook(this.config.toRef(conn), link.webhook_subscription_id);
|
||||
}
|
||||
this.config.deleteLink(link.id);
|
||||
this.announceBinding(link);
|
||||
// Unbinding keeps both copies. Nothing is deleted anywhere.
|
||||
return { success: true, documentsKept: true };
|
||||
}
|
||||
|
||||
@Post('links/:linkId/sync')
|
||||
@HttpCode(200)
|
||||
async syncNow(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('linkId') linkId: string,
|
||||
@Body() body: DocsyncSyncNowDto,
|
||||
) {
|
||||
const link = this.config.getLink(Number(linkId));
|
||||
if (!link || link.trip_id !== Number(tripId)) throw new HttpException('Link not found', 404);
|
||||
// An orphaned binding stays orphaned: its credential belongs to somebody who
|
||||
// is no longer on this trip, and a manual run would use it anyway. The
|
||||
// scheduler already refuses it (sync_enabled is 0), so without this the
|
||||
// button was a way around that.
|
||||
if (link.last_sync_state === 'orphaned') {
|
||||
throw new HttpException({ error: 'This binding lost its owner and has to be reconnected' }, 409);
|
||||
}
|
||||
// Refused before the shelved rows are touched, so a binding an admin
|
||||
// switched off stays exactly as it was and resumes where it stopped. A code
|
||||
// rather than a sentence: the client says it in the reader's language.
|
||||
if (this.sync.isSwitchedOff(link)) {
|
||||
throw new HttpException({ error: PROVIDER_DISABLED }, 409);
|
||||
}
|
||||
// A person asking for a run is also asking for the rows that gave up to be
|
||||
// tried once more; the scheduler gets no such reprieve.
|
||||
this.sync.retryShelvedItems(link.id);
|
||||
return this.sync.syncLink(link, { full: body.full });
|
||||
}
|
||||
|
||||
// ── Documents and conflicts ────────────────────────────────────────────────
|
||||
|
||||
@Get('items')
|
||||
items(@Param('tripId') tripId: string, @Query('state') state?: string) {
|
||||
const rows = state
|
||||
? this.db.connection
|
||||
.prepare(
|
||||
`SELECT i.*, f.original_name AS file_name FROM document_sync_items i
|
||||
LEFT JOIN trip_files f ON f.id = i.file_id
|
||||
WHERE i.trip_id = ? AND i.state = ? ORDER BY i.id DESC LIMIT 500`,
|
||||
)
|
||||
.all(Number(tripId), state)
|
||||
: this.db.connection
|
||||
.prepare(
|
||||
`SELECT i.*, f.original_name AS file_name FROM document_sync_items i
|
||||
LEFT JOIN trip_files f ON f.id = i.file_id
|
||||
WHERE i.trip_id = ? ORDER BY i.id DESC LIMIT 500`,
|
||||
)
|
||||
.all(Number(tripId));
|
||||
return rows;
|
||||
}
|
||||
|
||||
@Post('items/:itemId/resolve')
|
||||
@HttpCode(200)
|
||||
async resolve(
|
||||
@Param('tripId') tripId: string,
|
||||
@Param('itemId') itemId: string,
|
||||
@CurrentUser() user: User,
|
||||
@Body() body: DocsyncResolveConflictDto,
|
||||
) {
|
||||
this.assertCanManage(tripId, user);
|
||||
const ok = await this.sync.resolveConflict(Number(itemId), body.keep, Number(tripId));
|
||||
if (!ok) throw new HttpException('Item is not in conflict', 400);
|
||||
return { success: true };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Whether two URLs address the same server, for deciding if a stored secret may be reused. */
|
||||
function sameOrigin(a: string | null | undefined, b: string): boolean {
|
||||
if (!a) return false;
|
||||
try {
|
||||
const x = new URL(a);
|
||||
const y = new URL(b);
|
||||
return x.protocol === y.protocol && x.host === y.host;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The origin a provider has to call back on.
|
||||
*
|
||||
* Derived from the request rather than from configuration because a self-hosted
|
||||
* TREK is reached under whatever name its operator chose, and asking them to
|
||||
* maintain a second copy of it would get it wrong. Returns null behind a proxy
|
||||
* that strips the host, in which case the webhook is simply not offered and
|
||||
* polling carries the binding.
|
||||
*/
|
||||
function publicOrigin(req: Request): string | null {
|
||||
const host = req.get('x-forwarded-host') || req.get('host');
|
||||
if (!host) return null;
|
||||
const proto = (req.get('x-forwarded-proto') || req.protocol || 'http').split(',')[0].trim();
|
||||
return `${proto}://${host}`;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { createZodDto } from 'nestjs-zod';
|
||||
import {
|
||||
docsyncConnectionInputSchema,
|
||||
docsyncConnectionTestSchema,
|
||||
docsyncLinkInputSchema,
|
||||
docsyncLinkUpdateSchema,
|
||||
docsyncResolveConflictSchema,
|
||||
docsyncScopeCreateSchema,
|
||||
docsyncSyncNowSchema,
|
||||
} from '@trek/shared';
|
||||
|
||||
/**
|
||||
* Zod DTOs for every mutating route in this domain.
|
||||
*
|
||||
* The boot-time ratchet in common/validate-body-contracts.ts refuses to start
|
||||
* if a mutation takes an unwrapped body, so these are not optional decoration.
|
||||
*/
|
||||
export class DocsyncConnectionDto extends createZodDto(docsyncConnectionInputSchema) {}
|
||||
export class DocsyncConnectionTestDto extends createZodDto(docsyncConnectionTestSchema) {}
|
||||
export class DocsyncLinkDto extends createZodDto(docsyncLinkInputSchema) {}
|
||||
export class DocsyncLinkUpdateDto extends createZodDto(docsyncLinkUpdateSchema) {}
|
||||
export class DocsyncScopeCreateDto extends createZodDto(docsyncScopeCreateSchema) {}
|
||||
export class DocsyncResolveConflictDto extends createZodDto(docsyncResolveConflictSchema) {}
|
||||
export class DocsyncSyncNowDto extends createZodDto(docsyncSyncNowSchema) {}
|
||||
@@ -0,0 +1,677 @@
|
||||
import crypto from 'crypto';
|
||||
import path from 'path';
|
||||
import { BLOCKED_EXTENSIONS } from '../files/files.constants';
|
||||
import { MASS_DELETE_MIN_ITEMS, MASS_DELETE_RATIO } from './doc-sync.constants';
|
||||
import type { RemoteDocument } from './document-provider';
|
||||
|
||||
/**
|
||||
* The pure half of the sync core.
|
||||
*
|
||||
* `planReconcile` takes a snapshot of both sides and returns the actions to
|
||||
* take. It touches no database, no HTTP and no container, which is what lets
|
||||
* the hard cases (conflict, echo, mass deletion, rename, a document that
|
||||
* exists on neither side any more) be tested exhaustively as plain data. The
|
||||
* service does nothing but execute the plan.
|
||||
*/
|
||||
|
||||
// ── Identity and incoming names ──────────────────────────────────────────────
|
||||
|
||||
/** Per-document anchor, written into provider metadata where there is room. */
|
||||
export function newTrekDocUid(): string {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
|
||||
/**
|
||||
* Names arriving from a provider are hostile input: they can carry path
|
||||
* separators, control characters, leading dots and NTFS-illegal characters, and
|
||||
* they end up both in a DB column and in a Content-Disposition header.
|
||||
*
|
||||
* The extension is preserved deliberately: it is what TREK's download route
|
||||
* derives the Content-Type from, and what the blocklist check reads.
|
||||
*/
|
||||
export function sanitizeIncomingName(raw: string): string {
|
||||
// Split on separators by hand rather than through path.basename: the win32
|
||||
// flavour reads `a:` as a drive letter and silently drops it, so a perfectly
|
||||
// ordinary `a:b.pdf` from a Linux provider would arrive as `b.pdf`. Only
|
||||
// slashes are directory separators as far as this is concerned; the colon is
|
||||
// dealt with below, as an illegal character.
|
||||
const base = (raw || '').split(/[\\/]/).pop()?.trim() ?? '';
|
||||
const cleaned = base
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\u0000-\u001f\u007f]/g, '')
|
||||
.replace(/[\\/:*?"<>|]/g, '_')
|
||||
.replace(/^\.+/, '')
|
||||
.slice(0, 200);
|
||||
return cleaned || 'document';
|
||||
}
|
||||
|
||||
/**
|
||||
* The blocklist is enforced on incoming provider documents for exactly the same
|
||||
* reason it is enforced on uploads: TREK serves downloads inline with an
|
||||
* extension-derived Content-Type, so an .svg or .html from a Nextcloud folder
|
||||
* would be a stored XSS. A rejected document becomes a visible `rejected_type`
|
||||
* row rather than a silent skip, so the person who put it there can find out.
|
||||
*/
|
||||
export function isBlockedName(name: string): boolean {
|
||||
const ext = path.extname(name).toLowerCase();
|
||||
return !!ext && BLOCKED_EXTENSIONS.includes(ext);
|
||||
}
|
||||
|
||||
/**
|
||||
* The operator's own allowlist, applied to incoming provider documents.
|
||||
*
|
||||
* Same rule as an upload: `*` admits anything the blocklist above still lets
|
||||
* through, and an empty extension is refused rather than waved past, because a
|
||||
* file with no extension is served with a Content-Type TREK had to guess.
|
||||
*/
|
||||
export function isAllowedByOperator(name: string, allowedCsv: string): boolean {
|
||||
const ext = path.extname(name).toLowerCase().replace(/^\./, '');
|
||||
if (!ext) return false;
|
||||
if (allowedCsv.trim() === '*') return true;
|
||||
return allowedCsv
|
||||
.split(',')
|
||||
.map((e) => e.trim().toLowerCase().replace(/^\./, ''))
|
||||
.filter(Boolean)
|
||||
.includes(ext);
|
||||
}
|
||||
|
||||
// ── The plan ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** What the core knows about one pairing before the run. */
|
||||
export interface SyncItemState {
|
||||
id: number;
|
||||
fileId: number | null;
|
||||
trekDocUid: string;
|
||||
remoteId: string | null;
|
||||
remoteVersion: string | null;
|
||||
/** The name the provider had at the last agreed state: the rename arbiter. */
|
||||
remoteName: string | null;
|
||||
/**
|
||||
* Size and modification time of the provider copy as last seen. Only read
|
||||
* where ids are paths, to recognise the same copy after a rename or a move.
|
||||
*/
|
||||
remoteSize: number | null;
|
||||
remoteModifiedAt: string | null;
|
||||
/** The bytes both sides last agreed on. */
|
||||
contentSha256: string | null;
|
||||
/** What TREK itself last uploaded: the echo guard. */
|
||||
pushedSha256: string | null;
|
||||
state: string;
|
||||
attempts: number;
|
||||
/** When the backoff lets this row be tried again; null once it is shelved. */
|
||||
nextAttemptAt: string | null;
|
||||
remoteMissingAt: string | null;
|
||||
/**
|
||||
* When TREK itself put the provider copy in the recycle bin, under the
|
||||
* `trash` policy. A copy binned by TREK and one deleted by somebody else
|
||||
* leave the same gap in a listing; only this tells them apart once the file
|
||||
* comes back out of TREK's trash.
|
||||
*/
|
||||
remoteTrashedAt: string | null;
|
||||
}
|
||||
|
||||
/** What TREK currently holds for this trip. */
|
||||
export interface LocalDocument {
|
||||
fileId: number;
|
||||
name: string;
|
||||
size: number;
|
||||
mimeType: string | null;
|
||||
sha256: string | null;
|
||||
deletedAt: string | null;
|
||||
}
|
||||
|
||||
export type PlanAction =
|
||||
/** Bytes only exist upstream: download and create a trip_files row. */
|
||||
| { kind: 'pull'; remote: RemoteDocument; itemId: number | null }
|
||||
/** Bytes only exist in TREK: upload. */
|
||||
| { kind: 'push'; local: LocalDocument; itemId: number | null; remoteId: string | null }
|
||||
/** Upstream moved on while TREK did not: replace local bytes. */
|
||||
| { kind: 'pull_update'; remote: RemoteDocument; itemId: number }
|
||||
/** TREK moved on while upstream did not: push a new revision. */
|
||||
| { kind: 'push_update'; local: LocalDocument; itemId: number; remoteId: string }
|
||||
/**
|
||||
* Where the id is the path: the paired copy is listed under a new one,
|
||||
* because it was renamed or moved. The pairing follows; nothing else changes.
|
||||
*/
|
||||
| { kind: 'relocate'; itemId: number; remote: RemoteDocument }
|
||||
/** A name changed on one side only. */
|
||||
| { kind: 'rename_remote'; itemId: number; remoteId: string; name: string }
|
||||
| { kind: 'rename_local'; itemId: number; fileId: number; name: string }
|
||||
/** Both sides changed since the agreed state. */
|
||||
| { kind: 'conflict'; itemId: number; remote: RemoteDocument | null; local: LocalDocument | null }
|
||||
/** Present upstream last run, absent now. Recorded, never acted on. */
|
||||
| { kind: 'mark_remote_missing'; itemId: number }
|
||||
/** Deleted in TREK; what happens upstream is the link's delete policy. */
|
||||
| { kind: 'local_deleted'; itemId: number; remoteId: string }
|
||||
/**
|
||||
* Back out of TREK's trash while the provider copy stayed: the pairing counts
|
||||
* again. With `missing`, the copy is gone and TREK did not bin it, so the row
|
||||
* comes back flagged the way a vanished copy is.
|
||||
*/
|
||||
| { kind: 'local_restored'; itemId: number; missing?: true }
|
||||
/** Back out of TREK's trash after TREK binned the copy: unpaired, so it goes up as a new document. */
|
||||
| { kind: 'detach'; itemId: number }
|
||||
/** The copy TREK binned is listed again: somebody restored it at the provider. */
|
||||
| { kind: 'remote_restored'; itemId: number }
|
||||
/** Nothing to do but the row should stop looking stale. */
|
||||
| { kind: 'touch'; itemId: number; remote: RemoteDocument | null };
|
||||
|
||||
export interface ReconcilePlan {
|
||||
actions: PlanAction[];
|
||||
/** True when the listing looked like a mass deletion and was not trusted. */
|
||||
massDeleteGuardTripped: boolean;
|
||||
/** Newly gone upstream: vanished since the last run, or found gone on a restore in TREK. */
|
||||
missingCount: number;
|
||||
}
|
||||
|
||||
export interface ReconcileInput {
|
||||
items: readonly SyncItemState[];
|
||||
remote: readonly RemoteDocument[];
|
||||
local: readonly LocalDocument[];
|
||||
direction: 'both' | 'pull' | 'push';
|
||||
/** A truncated listing must never be read as "the rest was deleted". */
|
||||
remoteTruncated: boolean;
|
||||
/**
|
||||
* The provider says its side has not moved since the stored cursor.
|
||||
*
|
||||
* Every adapter derives that cursor from the scope's contents (a digest over
|
||||
* the documents for Paperless, Papra and Synology, the root ETag for WebDAV),
|
||||
* so the flag means "nothing changed upstream" for all of them. The WebDAV
|
||||
* adapter acts on it by returning an EMPTY list instead of walking the folder,
|
||||
* which is only safe if this is read: otherwise a quiet binding looks like a
|
||||
* folder somebody emptied. It was not read, and the third run of every idle
|
||||
* Nextcloud or OpenCloud binding tripped the mass-delete guard for good.
|
||||
*/
|
||||
remoteUnchanged: boolean;
|
||||
/**
|
||||
* What to do when both sides moved since they last agreed.
|
||||
*
|
||||
* `manual` parks the row for a person. The other two are the reason this
|
||||
* field exists: the binding was asked at setup, the answer was stored, and
|
||||
* nothing ever read it, so every conflict was manual whatever the binding
|
||||
* said, and a person who had chosen a side still had to pick it again.
|
||||
*/
|
||||
conflictPolicy: 'manual' | 'trek_wins' | 'provider_wins';
|
||||
/** Providers without stable ids need the rename heuristic. */
|
||||
stableRemoteIds: boolean;
|
||||
/** How many failures a row gets before it is shelved for a person to look at. */
|
||||
maxAttempts: number;
|
||||
/** Injected so the backoff window is testable; defaults to now. */
|
||||
now?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide what has to happen, from a snapshot of both sides.
|
||||
*
|
||||
* The three-way comparison is the whole point: `contentSha256` is the state
|
||||
* both sides last agreed on, so "changed here" and "changed there" are separate
|
||||
* questions and a change on one side alone is never a conflict. Collapsing that
|
||||
* into a two-way comparison is what makes naive sync engines either lose edits
|
||||
* or ping-pong forever.
|
||||
*/
|
||||
export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
const { remote, local, direction, remoteTruncated, stableRemoteIds, maxAttempts, conflictPolicy } = input;
|
||||
const now = input.now ?? new Date().toISOString().replace('T', ' ').slice(0, 19);
|
||||
const actions: PlanAction[] = [];
|
||||
|
||||
/**
|
||||
* Rows that must not be tried again on this run.
|
||||
*
|
||||
* A provider that refuses a document refuses it every time: a file it reads
|
||||
* as corrupt, a type it does not accept, a quota that is full. Without this
|
||||
* the row is re-planned on every pass, so a single bad document turns into an
|
||||
* upload attempt every cron tick forever; rows were found at 46 attempts
|
||||
* against a limit of 6, because the limit was written to the database and
|
||||
* never read back out.
|
||||
*
|
||||
* Shelved is not dead: a manual run clears the counter (see resetItemAttempts),
|
||||
* which is what "waits for a person" means.
|
||||
*/
|
||||
const blocked = new Set<number>();
|
||||
for (const it of input.items) {
|
||||
if (it.state !== 'error') continue;
|
||||
if (it.attempts >= maxAttempts) { blocked.add(it.id); continue; }
|
||||
if (it.nextAttemptAt !== null && it.nextAttemptAt > now) blocked.add(it.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every action goes through here so one rule covers all of them.
|
||||
*
|
||||
* No exception for `mark_remote_missing`: a row only counts as vanished if it
|
||||
* was `synced` last run, and a row is only shelved if it is in `error`, so the
|
||||
* two can never describe the same row.
|
||||
*/
|
||||
const add = (action: PlanAction): void => {
|
||||
const itemId = 'itemId' in action ? action.itemId : null;
|
||||
if (itemId !== null && blocked.has(itemId)) return;
|
||||
actions.push(action);
|
||||
};
|
||||
|
||||
/**
|
||||
* Both sides changed the bytes. The binding's answer decides, and `manual`
|
||||
* parks it.
|
||||
*
|
||||
* A side that wins takes the whole document, name and bytes together, because
|
||||
* the two halves of one document coming from different sides is not a state
|
||||
* anybody asked for. Direction still has the last word: a pull-only binding
|
||||
* cannot push even when TREK is meant to win, so it parks instead of doing
|
||||
* the opposite of what was chosen.
|
||||
*/
|
||||
const settle = (it: SyncItemState, r: RemoteDocument, l: LocalDocument | null): void => {
|
||||
// Only the two answers that name a winner act; anything else (`manual`, a
|
||||
// value from a future version, a column somebody edited by hand) parks the
|
||||
// row. Overwriting one side is the destructive move, so it needs a yes.
|
||||
const park = (): void => { add({ kind: 'conflict', itemId: it.id, remote: r, local: l }); };
|
||||
if (conflictPolicy === 'provider_wins' && direction !== 'push') {
|
||||
return add({ kind: 'pull_update', remote: r, itemId: it.id });
|
||||
}
|
||||
if (conflictPolicy === 'trek_wins' && direction !== 'pull' && l !== null) {
|
||||
return add({ kind: 'push_update', local: l, itemId: it.id, remoteId: it.remoteId });
|
||||
}
|
||||
return park();
|
||||
};
|
||||
|
||||
/**
|
||||
* Both sides renamed a document whose bytes still agree. Only the name is in
|
||||
* dispute, so the winning side's name is carried over and nothing is sent.
|
||||
*
|
||||
* This went through `settle` once, and `trek_wins` re-uploaded identical
|
||||
* bytes. Where the provider replaces in place (WebDAV, Synology) the upload
|
||||
* also kept the provider's name, so the next run read that as a rename made
|
||||
* upstream and gave TREK's file the name TREK was supposed to win with.
|
||||
*/
|
||||
const settleName = (it: SyncItemState, r: RemoteDocument, l: LocalDocument, incoming: string): void => {
|
||||
if (conflictPolicy === 'provider_wins' && direction !== 'push') {
|
||||
return add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
|
||||
}
|
||||
if (conflictPolicy === 'trek_wins' && direction !== 'pull') {
|
||||
return add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
|
||||
}
|
||||
return add({ kind: 'conflict', itemId: it.id, remote: r, local: l });
|
||||
};
|
||||
|
||||
const remoteById = new Map<string, RemoteDocument>();
|
||||
for (const r of remote) if (!r.isDeleted) remoteById.set(r.remoteId, r);
|
||||
const localById = new Map<number, LocalDocument>();
|
||||
for (const l of local) localById.set(l.fileId, l);
|
||||
|
||||
/**
|
||||
* Where the id is the path, a rename or a move upstream reads as one copy
|
||||
* gone and another one new.
|
||||
*
|
||||
* This used to be patched up in the upstream-only loop with a `touch`, which
|
||||
* went wrong twice. The pairing had already been counted as vanished by then,
|
||||
* so the same run flagged it missing (and counted it towards the mass-delete
|
||||
* guard); and `touch` wrote the provider's new name into the arbiter, so the
|
||||
* next run read the name TREK still had as a rename made in TREK and renamed
|
||||
* the provider's copy back.
|
||||
*
|
||||
* So the pairing is moved first, before anything else looks at it, `reopen`
|
||||
* included: onto a new entry that is recognisably the same copy (see
|
||||
* `sameCopy`). Only the id and the version move. The agreed name stays, so
|
||||
* the pair loop reads the new name the way it reads any rename, and the rest
|
||||
* of the plan sees a document that is listed. A rename then ends exactly as
|
||||
* it does where ids are stable. A rename TREK sent upstream is found again
|
||||
* the same way: the adapter cannot say what the new path is, so the stored id
|
||||
* is stale until here.
|
||||
*
|
||||
* Only a one-to-one match moves anything. Two copies of the same bytes
|
||||
* renamed at once cannot be told apart, and pairing them crosswise puts each
|
||||
* name on the other file; unless one of them kept its name (a move), neither
|
||||
* is moved, and both are handled like any other gap and new document.
|
||||
*
|
||||
* A copy TREK binned itself only gets an entry nobody else fits. Its gap is
|
||||
* TREK's own doing, and letting it make a tie meant that deleting one of two
|
||||
* identical files in TREK stopped a rename of the other from being followed,
|
||||
* which came back as a missing flag plus a second download. It is still
|
||||
* recognised on its own, as a copy restored at the provider somewhere else.
|
||||
*/
|
||||
const followMoves = (list: readonly SyncItemState[]): SyncItemState[] => {
|
||||
const paired = new Set(list.map((it) => it.remoteId));
|
||||
const fresh = remote.filter((r) => !r.isDeleted && !paired.has(r.remoteId));
|
||||
const gone = list.filter((it) => it.remoteId !== null && !remoteById.has(it.remoteId));
|
||||
|
||||
const owner = (r: RemoteDocument): SyncItemState | null => {
|
||||
const fits = gone.filter((o) => sameCopy(o, r));
|
||||
return soleMatch(fits, (o) => keptName(o, r))
|
||||
?? soleMatch(fits.filter((o) => o.remoteTrashedAt === null), (o) => keptName(o, r));
|
||||
};
|
||||
|
||||
const moved = new Map<number, RemoteDocument>();
|
||||
for (const it of gone) {
|
||||
const r = soleMatch(fresh.filter((d) => sameCopy(it, d)), (d) => keptName(it, d));
|
||||
if (!r || owner(r) !== it) continue;
|
||||
// Counted above so a shelved row still makes a tie a tie, but not moved:
|
||||
// `add` would drop the action while the plan went on with the new id.
|
||||
if (blocked.has(it.id)) continue;
|
||||
moved.set(it.id, r);
|
||||
}
|
||||
|
||||
return list.map((it) => {
|
||||
const r = moved.get(it.id);
|
||||
if (!r) return it;
|
||||
add({ kind: 'relocate', itemId: it.id, remote: r });
|
||||
// Same bytes, so the new version marker is not a change: on a path-keyed
|
||||
// provider it differs only because the path is part of it.
|
||||
return { ...it, remoteId: r.remoteId, remoteVersion: r.remoteVersion };
|
||||
});
|
||||
};
|
||||
|
||||
// Restored rows whose copy turned out to be gone: missing, but not vanished.
|
||||
let foundGone = 0;
|
||||
|
||||
/**
|
||||
* A file taken back out of TREK's trash.
|
||||
*
|
||||
* A deletion TREK has acted on is otherwise left alone for good (see the
|
||||
* pair loop), so this is the one way out of `local_deleted`, and which way
|
||||
* depends on the provider copy. If it is listed, the pairing counts again and
|
||||
* the row is judged like any synced one from here on: an edit made upstream
|
||||
* in the meantime comes down. If TREK binned it, TREK took it away and the
|
||||
* person asked for it back, so the pairing is dropped and the file goes up as
|
||||
* a new document through the ordinary push.
|
||||
*
|
||||
* A copy gone by somebody else's hand is flagged as missing rather than
|
||||
* uploaded over their deletion, and flagged right here rather than counted
|
||||
* as vanished. Nothing is recorded about a copy while its row sits in
|
||||
* `local_deleted`, so it may have been gone for weeks, and counting it
|
||||
* towards the mass-delete guard read a handful of such restores as a listing
|
||||
* that shrank: the guard tripped, the run was abandoned before the restores
|
||||
* were written, and the next run found the same plan, for good. Only a
|
||||
* listing that could have shown the copy may say it is gone, so otherwise
|
||||
* the row waits; `needsFullListing` keeps that to a single run.
|
||||
*
|
||||
* The returned row is what the rest of the plan sees, so it has to match
|
||||
* what the executor writes for the action added here.
|
||||
*/
|
||||
const reopen = (it: SyncItemState): SyncItemState => {
|
||||
if (!isBackFromTrash(it, localById)) return it;
|
||||
if (!input.remoteUnchanged && remoteById.has(it.remoteId)) {
|
||||
add({ kind: 'local_restored', itemId: it.id });
|
||||
return { ...it, state: 'synced', remoteMissingAt: null, remoteTrashedAt: null };
|
||||
}
|
||||
if (it.remoteTrashedAt === null) {
|
||||
if (input.remoteUnchanged || remoteTruncated) return it;
|
||||
foundGone += 1;
|
||||
add({ kind: 'local_restored', itemId: it.id, missing: true });
|
||||
return { ...it, state: 'remote_missing', remoteMissingAt: now, remoteTrashedAt: null };
|
||||
}
|
||||
// A pull-only binding cannot upload, and a gap in a truncated listing says
|
||||
// nothing about the copy. The row waits for a run that can decide.
|
||||
if (direction === 'pull' || remoteTruncated) return it;
|
||||
add({ kind: 'detach', itemId: it.id });
|
||||
return { ...it, state: 'pending', remoteId: null, remoteVersion: null, remoteTrashedAt: null };
|
||||
};
|
||||
|
||||
// Upstream unchanged means there is no listing to follow anything into.
|
||||
const followed = stableRemoteIds || input.remoteUnchanged ? input.items : followMoves(input.items);
|
||||
const items = followed.map(reopen);
|
||||
|
||||
const itemsByRemote = new Map<string, SyncItemState>();
|
||||
const itemsByFile = new Map<number, SyncItemState>();
|
||||
for (const it of items) {
|
||||
if (it.remoteId) itemsByRemote.set(it.remoteId, it);
|
||||
if (it.fileId !== null) itemsByFile.set(it.fileId, it);
|
||||
}
|
||||
|
||||
// Upstream is unchanged: only what happened in TREK can need doing, and the
|
||||
// remote half of the plan would be reading a list the adapter did not fetch.
|
||||
if (input.remoteUnchanged) {
|
||||
for (const it of items) {
|
||||
// A deletion already acted on stays acted on, as in the pair loop. A copy
|
||||
// on record as gone is absent from a full listing too, so a full run
|
||||
// plans nothing for it until it comes back; binning it here would also
|
||||
// write the gap down as TREK's doing, and a restore would then upload
|
||||
// what somebody else deleted.
|
||||
if (!it.remoteId || it.state === 'local_deleted' || it.state === 'remote_missing') continue;
|
||||
const l = it.fileId !== null ? localById.get(it.fileId) : undefined;
|
||||
if (l?.deletedAt || lostItsFile(it)) add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
}
|
||||
if (direction !== 'pull') {
|
||||
for (const l of local) {
|
||||
if (l.deletedAt) continue;
|
||||
const it = itemsByFile.get(l.fileId);
|
||||
if (it?.remoteId) continue;
|
||||
add({ kind: 'push', local: l, itemId: it?.id ?? null, remoteId: null });
|
||||
}
|
||||
}
|
||||
return { actions, massDeleteGuardTripped: false, missingCount: 0 };
|
||||
}
|
||||
|
||||
// A listing that lost most of what it had last time is far more likely to be
|
||||
// a broken mount, a revoked token or a moved folder than a real mass delete.
|
||||
// A row `reopen` resumed is only `synced` here when its copy is listed, so a
|
||||
// restore never reads as part of a listing that shrank.
|
||||
const known = items.filter((i) => i.remoteId && i.state === 'synced');
|
||||
const vanished = known.filter((i) => !remoteById.has(i.remoteId as string));
|
||||
const guardTripped =
|
||||
!remoteTruncated &&
|
||||
known.length >= MASS_DELETE_MIN_ITEMS &&
|
||||
vanished.length / known.length > MASS_DELETE_RATIO;
|
||||
|
||||
// ── Pairs that exist on both sides ────────────────────────────────────────
|
||||
for (const it of items) {
|
||||
if (!it.remoteId) continue;
|
||||
const r = remoteById.get(it.remoteId);
|
||||
if (!r) continue;
|
||||
const l = it.fileId !== null ? localById.get(it.fileId) : undefined;
|
||||
|
||||
const remoteChanged = it.remoteVersion !== null && r.remoteVersion !== it.remoteVersion;
|
||||
// The echo guard: bytes TREK itself pushed come back as a change on the
|
||||
// provider side. They are only TREK's own write if the hash still matches
|
||||
// what was pushed, so this compares content, never timestamps: a time
|
||||
// window would misfire on every clock skew.
|
||||
const isEcho =
|
||||
remoteChanged &&
|
||||
r.contentHash !== null &&
|
||||
it.pushedSha256 !== null &&
|
||||
r.contentHash === it.pushedSha256;
|
||||
|
||||
const localChanged = !!l && l.sha256 !== null && it.contentSha256 !== null && l.sha256 !== it.contentSha256;
|
||||
|
||||
/**
|
||||
* A deletion is acted on once, under the policy in force at that moment.
|
||||
*
|
||||
* This used to be planned again on every run for as long as the file sat
|
||||
* in TREK's trash. Under `unlink` nothing showed; switching the binding to
|
||||
* `trash` later then binned every document ever deleted in TREK, months
|
||||
* back included, and a binding on `trash` from the start asked the
|
||||
* provider to bin the same copy on every run. A row still here is one
|
||||
* `reopen` did not take back: its file is still in the trash or purged.
|
||||
*/
|
||||
if (it.state === 'local_deleted') {
|
||||
if (it.remoteTrashedAt !== null) add({ kind: 'remote_restored', itemId: it.id });
|
||||
continue;
|
||||
}
|
||||
if (l?.deletedAt || lostItsFile(it)) {
|
||||
add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
continue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Paired upstream, absent in TREK: the download never landed.
|
||||
*
|
||||
* A failed pull stores the remote's version alongside the error, so on the
|
||||
* next run `remoteChanged` is false, there is no local file for the other
|
||||
* branches to compare against, and the row fell through to `touch`, which
|
||||
* refreshes it and changes nothing. Nothing anywhere planned a second
|
||||
* attempt: the document never arrived, no retry ever happened, and pressing
|
||||
* "Sync now" did not help either, because the plan was the same.
|
||||
*
|
||||
* A rejected type and an oversized file are left out on purpose. Those are
|
||||
* not transport failures; they are answers, and they stay true until the
|
||||
* document upstream changes, which the `remoteChanged` branch below picks
|
||||
* up on its own.
|
||||
*/
|
||||
if (!l && it.fileId === null && direction !== 'push'
|
||||
&& it.state !== 'rejected_type' && it.state !== 'too_large') {
|
||||
add({ kind: 'pull_update', remote: r, itemId: it.id });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (remoteChanged && !isEcho && localChanged) {
|
||||
settle(it, r, l ?? null);
|
||||
continue;
|
||||
}
|
||||
if (remoteChanged && !isEcho) {
|
||||
if (direction === 'push') { add({ kind: 'touch', itemId: it.id, remote: r }); continue; }
|
||||
add({ kind: 'pull_update', remote: r, itemId: it.id });
|
||||
continue;
|
||||
}
|
||||
if (localChanged && l) {
|
||||
if (direction === 'pull') { add({ kind: 'touch', itemId: it.id, remote: r }); continue; }
|
||||
add({ kind: 'push_update', local: l, itemId: it.id, remoteId: it.remoteId });
|
||||
continue;
|
||||
}
|
||||
|
||||
// Same bytes, different name. Which side renamed is decided by comparing
|
||||
// each against the name they last agreed on, not by direction. Reading
|
||||
// `both` as "TREK always wins" silently renamed the provider's copy back
|
||||
// every time somebody tidied up a folder, which is the opposite of a
|
||||
// two-way sync.
|
||||
//
|
||||
// Compared through the same sanitiser the local name went through, or a
|
||||
// provider name TREK had to clean up (a slash, a control character, a
|
||||
// leading dot) reads as "TREK renamed this" on the very next run, and the
|
||||
// provider's copy gets renamed to the cleaned version, unasked. For the
|
||||
// same reason TREK takes a new name in its cleaned form, the one a download
|
||||
// would have given it: a Paperless title with a slash in it used to land
|
||||
// in the file manager as it was, and the next run read that as a rename
|
||||
// made in TREK and sent it upstream again.
|
||||
const incoming = sanitizeIncomingName(r.name);
|
||||
if (l && incoming !== l.name) {
|
||||
const agreed = it.remoteName === null ? null : sanitizeIncomingName(it.remoteName);
|
||||
const providerRenamed = agreed !== null && incoming !== agreed;
|
||||
const localRenamed = agreed !== null && l.name !== agreed;
|
||||
|
||||
if (providerRenamed && !localRenamed && direction !== 'push') {
|
||||
add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
|
||||
continue;
|
||||
}
|
||||
if (localRenamed && !providerRenamed && direction !== 'pull') {
|
||||
add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
|
||||
continue;
|
||||
}
|
||||
if (providerRenamed && localRenamed) {
|
||||
// Both sides renamed. Nothing here can pick the right one on its own,
|
||||
// and picking wrong loses a name somebody chose, so it follows the
|
||||
// binding's answer, and goes to a person when there is none.
|
||||
settleName(it, r, l, incoming);
|
||||
continue;
|
||||
}
|
||||
// No stored name to arbitrate with (a row from before this column, or a
|
||||
// freshly paired document): follow the binding's direction.
|
||||
if (direction === 'pull') add({ kind: 'rename_local', itemId: it.id, fileId: l.fileId, name: incoming });
|
||||
else if (direction === 'push') add({ kind: 'rename_remote', itemId: it.id, remoteId: it.remoteId, name: l.name });
|
||||
else add({ kind: 'touch', itemId: it.id, remote: r });
|
||||
continue;
|
||||
}
|
||||
|
||||
add({ kind: 'touch', itemId: it.id, remote: r });
|
||||
}
|
||||
|
||||
// ── Upstream only ─────────────────────────────────────────────────────────
|
||||
if (direction !== 'push') {
|
||||
for (const r of remote) {
|
||||
if (r.isDeleted) continue;
|
||||
// A copy that was only renamed or moved is paired already (followMoves).
|
||||
if (itemsByRemote.has(r.remoteId)) continue;
|
||||
add({ kind: 'pull', remote: r, itemId: null });
|
||||
}
|
||||
}
|
||||
|
||||
// ── TREK only ─────────────────────────────────────────────────────────────
|
||||
if (direction !== 'pull') {
|
||||
for (const l of local) {
|
||||
if (l.deletedAt) continue;
|
||||
const it = itemsByFile.get(l.fileId);
|
||||
if (it?.remoteId && remoteById.has(it.remoteId)) continue;
|
||||
// A row that has a remoteId which is no longer listed is handled below as
|
||||
// "missing", not re-uploaded. Otherwise a broken listing duplicates the
|
||||
// entire trip upstream.
|
||||
if (it?.remoteId) continue;
|
||||
add({ kind: 'push', local: l, itemId: it?.id ?? null, remoteId: null });
|
||||
}
|
||||
}
|
||||
|
||||
// ── Vanished upstream ─────────────────────────────────────────────────────
|
||||
if (!guardTripped && !remoteTruncated) {
|
||||
for (const it of vanished) {
|
||||
if (it.remoteMissingAt) continue;
|
||||
add({ kind: 'mark_remote_missing', itemId: it.id });
|
||||
}
|
||||
}
|
||||
|
||||
return { actions, massDeleteGuardTripped: guardTripped, missingCount: vanished.length + foundGone };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this run has to list the provider in full, whatever its cursor says.
|
||||
*
|
||||
* A file back out of TREK's trash whose copy TREK did not bin can only be
|
||||
* settled by a listing that shows whether that copy is still there (see
|
||||
* `reopen`). An unchanged cursor gives the planner nothing to read, and on a
|
||||
* quiet binding the cursor stays unchanged until somebody touches the folder,
|
||||
* so the restore would wait that long.
|
||||
*/
|
||||
export function needsFullListing(items: readonly SyncItemState[], local: readonly LocalDocument[]): boolean {
|
||||
const localById = new Map(local.map((l) => [l.fileId, l]));
|
||||
return items.some((it) => it.remoteTrashedAt === null && isBackFromTrash(it, localById));
|
||||
}
|
||||
|
||||
/** A deletion TREK acted on whose file has since come back out of TREK's trash. */
|
||||
function isBackFromTrash(it: SyncItemState, localById: ReadonlyMap<number, LocalDocument>): boolean {
|
||||
if (it.state !== 'local_deleted' || !it.remoteId || it.fileId === null) return false;
|
||||
const l = localById.get(it.fileId);
|
||||
return !!l && !l.deletedAt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a listing entry is recognisably the copy a pairing had, judged by
|
||||
* content rather than by id.
|
||||
*
|
||||
* A hash on both sides settles it either way. Without one (Synology offers
|
||||
* none), size and modification time together: a rename or a move leaves both
|
||||
* alone, and an edit does not.
|
||||
*/
|
||||
function sameCopy(it: SyncItemState, r: RemoteDocument): boolean {
|
||||
if (r.contentHash !== null && it.contentSha256 !== null) return r.contentHash === it.contentSha256;
|
||||
return r.size !== null && r.remoteModifiedAt !== null
|
||||
&& r.size === it.remoteSize && r.remoteModifiedAt === it.remoteModifiedAt;
|
||||
}
|
||||
|
||||
/** The copy is listed under the name both sides last agreed on: moved, not renamed. */
|
||||
function keptName(it: SyncItemState, r: RemoteDocument): boolean {
|
||||
return it.remoteName !== null && sanitizeIncomingName(it.remoteName) === sanitizeIncomingName(r.name);
|
||||
}
|
||||
|
||||
/** The only candidate, or else the only one that kept its name; otherwise none. */
|
||||
function soleMatch<T>(candidates: T[], keptItsName: (candidate: T) => boolean): T | null {
|
||||
if (candidates.length === 1) return candidates[0];
|
||||
const named = candidates.filter(keptItsName);
|
||||
return named.length === 1 ? named[0] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A file purged from TREK's trash before any run saw it deleted.
|
||||
*
|
||||
* `file_id` is ON DELETE SET NULL, so the row outlives the file with nothing
|
||||
* but a NULL to show for it. Read as a download that never landed, which is
|
||||
* what happened until now, the document came straight back and the delete
|
||||
* policy never ran. A first download that failed has no file either, and is
|
||||
* retried, so `error` and `pending` count only with an agreed hash: that is
|
||||
* written by a pull or a push that landed and by nothing else, and a failed
|
||||
* update keeps the file it had. A row shelved in `error` that the person then
|
||||
* deleted and purged is a deletion, not a document to fetch again.
|
||||
*/
|
||||
function lostItsFile(it: SyncItemState): boolean {
|
||||
if (it.fileId !== null) return false;
|
||||
if (it.state === 'synced' || it.state === 'conflict' || it.state === 'remote_missing') return true;
|
||||
return (it.state === 'error' || it.state === 'pending') && it.contentSha256 !== null;
|
||||
}
|
||||
|
||||
/** Backoff lookup that saturates instead of running off the end of the curve. */
|
||||
export function backoffSeconds(curve: readonly number[], failures: number): number {
|
||||
if (failures <= 0) return curve[0];
|
||||
return curve[Math.min(failures - 1, curve.length - 1)];
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
import { logError, logInfo } from '../audit/audit-log.logger';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { DatabaseService } from '../database/database.service';
|
||||
import { CronRegistrarService } from '../scheduling/cron-registrar.service';
|
||||
import { DocSyncConfigService } from './doc-sync-config.service';
|
||||
import { DocSyncService } from './doc-sync.service';
|
||||
import {
|
||||
DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
MAX_POLL_INTERVAL_SECONDS,
|
||||
MIN_POLL_INTERVAL_SECONDS,
|
||||
SETTING_POLL_INTERVAL,
|
||||
SETTING_SYNC_ENABLED,
|
||||
} from './doc-sync.constants';
|
||||
|
||||
/**
|
||||
* The poll that carries document sync.
|
||||
*
|
||||
* It is the mechanism, not the fallback. Only Paperless lets TREK subscribe to
|
||||
* changes on its own; Papra's webhook API is closed to API keys, Nextcloud's
|
||||
* needs admin rights, OpenCloud has no registrable hook, and Synology has
|
||||
* nothing at all. On top of that no provider here emits an event for every
|
||||
* change that matters: Paperless has no deletion trigger, and a tag change in
|
||||
* Papra moves no timestamp. So the run enumerates, and a webhook only ever
|
||||
* makes it happen sooner.
|
||||
*
|
||||
* The addon gate and the interval are re-read per tick rather than at bootstrap
|
||||
* so an admin toggling either takes effect without a restart. That differs from
|
||||
* the Dawarich job on purpose: this one is driven by a per-trip binding an
|
||||
* ordinary user creates, and telling them "restart your server" is not an
|
||||
* answer.
|
||||
*
|
||||
* Which is why the cron itself runs every minute and the tick decides whether it
|
||||
* is due: baking the interval into the cron expression at bootstrap (as this
|
||||
* did until the claim above was checked against the code) meant a changed
|
||||
* interval did nothing until a restart, quietly, while the admin screen said
|
||||
* otherwise.
|
||||
*/
|
||||
@Injectable()
|
||||
export class DocSyncJob implements OnApplicationBootstrap {
|
||||
constructor(
|
||||
private readonly db: DatabaseService,
|
||||
private readonly sync: DocSyncService,
|
||||
private readonly config: DocSyncConfigService,
|
||||
private readonly addons: AddonsService,
|
||||
private readonly registrar: CronRegistrarService,
|
||||
) {}
|
||||
|
||||
/** When the last pass started; null until the first one, which is never skipped. */
|
||||
private lastRunAt: number | null = null;
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
logInfo(`Document sync: polling every ${this.intervalSeconds()}s`);
|
||||
this.registrar.register('docsync', '* * * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether enough time has passed for another pass.
|
||||
*
|
||||
* The minute the cron wakes up on is not the unit the setting is in, so a
|
||||
* 90-second interval must not become one minute or two depending on where the
|
||||
* boundaries fall. Comparing against the last run keeps the setting's own
|
||||
* resolution; the cost of the extra wake-ups is one read of app_settings.
|
||||
*/
|
||||
private isDue(now: number): boolean {
|
||||
// Null rather than 0: comparing against the epoch means "due" only once the
|
||||
// clock has passed the interval since 1970, which is true in production and
|
||||
// false for any test that picks a small timestamp, a difference that would
|
||||
// have hidden here rather than in the behaviour it is supposed to describe.
|
||||
if (this.lastRunAt === null) return true;
|
||||
return now - this.lastRunAt >= this.intervalSeconds() * 1000;
|
||||
}
|
||||
|
||||
private intervalSeconds(): number {
|
||||
const raw = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', SETTING_POLL_INTERVAL)?.value;
|
||||
const parsed = Number.parseInt(raw || '', 10);
|
||||
if (!Number.isFinite(parsed)) return DEFAULT_POLL_INTERVAL_SECONDS;
|
||||
return Math.min(MAX_POLL_INTERVAL_SECONDS, Math.max(MIN_POLL_INTERVAL_SECONDS, parsed));
|
||||
}
|
||||
|
||||
async tick(): Promise<void> {
|
||||
try {
|
||||
if (!this.addons.isAddonEnabled(ADDON_IDS.DOCUMENTS)) return;
|
||||
const killSwitch = this.db.get<{ value: string }>('SELECT value FROM app_settings WHERE key = ?', SETTING_SYNC_ENABLED)?.value;
|
||||
// Unrecognised values mean ON here because the setting is absent by
|
||||
// default; only an explicit 'false' stops the sync.
|
||||
if (killSwitch === 'false') return;
|
||||
|
||||
const now = Date.now();
|
||||
if (!this.isDue(now)) return;
|
||||
this.lastRunAt = now;
|
||||
|
||||
// Cheap, and it catches a binding whose owner left the trip through a
|
||||
// path that has no hook to attach to: a transfer, a direct DB edit.
|
||||
const orphaned = this.config.markOrphanedLinks();
|
||||
if (orphaned > 0) logInfo(`Document sync: ${orphaned} link(s) orphaned, owner no longer on the trip`);
|
||||
|
||||
const links = this.sync.dueLinks();
|
||||
for (const link of links) {
|
||||
// One failing provider must not stop the others: an unreachable NAS on
|
||||
// one trip is not a reason to skip a Paperless binding on another.
|
||||
try {
|
||||
await this.sync.syncLink(link);
|
||||
} catch (err: unknown) {
|
||||
logError(`Document sync: link ${link.id} failed: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
logError(`Document sync tick failed: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
McpController,
|
||||
Tool,
|
||||
type McpContext,
|
||||
TOOL_ANNOTATIONS_READONLY,
|
||||
TOOL_ANNOTATIONS_WRITE,
|
||||
errorResult,
|
||||
ok,
|
||||
} from '../../nest-mcp';
|
||||
import { ADDON_IDS } from '../../addons';
|
||||
import { noAccess } from '../../mcp/tools/_shared';
|
||||
import { addonGate } from '../addons/addon-gate';
|
||||
import { AddonsService } from '../addons/addons.service';
|
||||
import { FilesService } from '../files/files.service';
|
||||
import { DocSyncConfigService } from './doc-sync-config.service';
|
||||
import { DocSyncService } from './doc-sync.service';
|
||||
import { PROVIDER_DISABLED } from './doc-sync.constants';
|
||||
|
||||
const documentsAddonOn = addonGate(ADDON_IDS.DOCUMENTS);
|
||||
|
||||
/**
|
||||
* The MCP half of document sync, mirroring /api/trips/:tripId/docsync.
|
||||
*
|
||||
* Read and trigger only. There is no tool that creates a connection, and that
|
||||
* is deliberate rather than unfinished: setting one up means handing over a
|
||||
* credential that usually reaches a person's entire document archive, and an
|
||||
* assistant is the wrong place to collect one. The same reason the REST side
|
||||
* restricts those routes to the trip owner applies twice over here.
|
||||
*
|
||||
* `sync_trip_documents` is a write because it moves files in both directions,
|
||||
* even though it creates nothing itself.
|
||||
*/
|
||||
@McpController()
|
||||
export class DocSyncMcp {
|
||||
constructor(
|
||||
private readonly config: DocSyncConfigService,
|
||||
private readonly sync: DocSyncService,
|
||||
private readonly files: FilesService,
|
||||
readonly addons: AddonsService,
|
||||
) {}
|
||||
|
||||
@Tool({
|
||||
name: 'get_trip_document_sync',
|
||||
description:
|
||||
'Show whether this trip\'s documents are synced with an external document store (Paperless-ngx, Papra, Nextcloud, OpenCloud or a Synology NAS), which folder or tag they are bound to, when the last run happened, and how many documents are waiting, in conflict or missing at the provider. Use this before telling someone where their documents live.',
|
||||
inputSchema: {
|
||||
tripId: z.number().int().positive(),
|
||||
},
|
||||
annotations: TOOL_ANNOTATIONS_READONLY,
|
||||
access: { group: 'files', mode: 'read' },
|
||||
when: documentsAddonOn,
|
||||
})
|
||||
getTripDocumentSync({ tripId }: { tripId: number }, ctx: McpContext) {
|
||||
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
|
||||
return ok(this.sync.status(tripId));
|
||||
}
|
||||
|
||||
@Tool({
|
||||
name: 'list_trip_document_sync_issues',
|
||||
description:
|
||||
'List the documents that need a person to look at them: conflicts where both copies changed, documents the provider refused because of their type or size, and documents that disappeared at the provider. Returns an empty list when everything is in step.',
|
||||
inputSchema: {
|
||||
tripId: z.number().int().positive(),
|
||||
},
|
||||
annotations: TOOL_ANNOTATIONS_READONLY,
|
||||
access: { group: 'files', mode: 'read' },
|
||||
when: documentsAddonOn,
|
||||
})
|
||||
listIssues({ tripId }: { tripId: number }, ctx: McpContext) {
|
||||
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
|
||||
const links = this.config.listLinks(tripId);
|
||||
if (links.length === 0) return ok({ configured: false, issues: [] });
|
||||
return ok({ configured: true, issues: this.sync.issues(tripId) });
|
||||
}
|
||||
|
||||
@Tool({
|
||||
name: 'sync_trip_documents',
|
||||
description:
|
||||
'Run the document sync for this trip now instead of waiting for the next scheduled check. Reports how many documents were pulled in from the provider, pushed out to it, and how many are in conflict. Safe to call repeatedly: a run that is already in progress is not started twice.',
|
||||
inputSchema: {
|
||||
tripId: z.number().int().positive(),
|
||||
full: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.default(false)
|
||||
.describe('Ignore the change marker from last time and compare both sides in full'),
|
||||
},
|
||||
annotations: TOOL_ANNOTATIONS_WRITE,
|
||||
access: { group: 'files', mode: 'write' },
|
||||
when: documentsAddonOn,
|
||||
})
|
||||
async syncNow({ tripId, full }: { tripId: number; full?: boolean }, ctx: McpContext) {
|
||||
if (!this.files.verifyTripAccess(tripId, ctx.userId)) return noAccess();
|
||||
const links = this.config.listLinks(tripId);
|
||||
if (links.length === 0) {
|
||||
return errorResult('This trip is not connected to a document store. Connect one in the trip\'s file manager first.');
|
||||
}
|
||||
const results = [];
|
||||
for (const link of links) {
|
||||
// Refused as the REST route refuses it, before the shelved rows below are
|
||||
// touched: a binding an admin switched off stays exactly as it was, so it
|
||||
// resumes where it stopped once the provider is back on.
|
||||
if (this.sync.isSwitchedOff(link)) {
|
||||
results.push({ linkId: link.id, provider: link.provider_id, state: 'disabled', errorCode: PROVIDER_DISABLED });
|
||||
continue;
|
||||
}
|
||||
// Asking for a run by hand means "try again", including the documents
|
||||
// that were shelved after too many failures. The REST route does the
|
||||
// same thing before its run; a tool that skipped it would answer "in
|
||||
// sync" while leaving them shelved.
|
||||
this.sync.retryShelvedItems(link.id);
|
||||
results.push({ linkId: link.id, provider: link.provider_id, ...(await this.sync.syncLink(link, { full: full === true })) });
|
||||
}
|
||||
if (results.every((r) => r.errorCode === PROVIDER_DISABLED)) {
|
||||
return errorResult(
|
||||
`${PROVIDER_DISABLED}: an administrator has switched off the document store this trip is bound to, so nothing was synced. The binding and its documents are kept, and syncing resumes once the store is switched back on.`,
|
||||
);
|
||||
}
|
||||
return ok({ runs: results });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AddonsModule } from '../addons/addons.module';
|
||||
import { AuthModule } from '../auth/auth.module';
|
||||
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
|
||||
import { DatabaseModule } from '../database/database.module';
|
||||
import { FilesModule } from '../files/files.module';
|
||||
import { AllowedFileTypesModule } from '../files/allowed-file-types.module';
|
||||
import { PermissionsModule } from '../permissions/permissions.module';
|
||||
import { RealtimeModule } from '../realtime/realtime.module';
|
||||
import { SchedulingModule } from '../scheduling/scheduling.module';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { DocSyncConfigService } from './doc-sync-config.service';
|
||||
import { DocSyncController } from './doc-sync.controller';
|
||||
import { DocSyncWebhookController } from './doc-sync-webhook.controller';
|
||||
import { DocSyncJob } from './doc-sync.job';
|
||||
import { DocSyncMcp } from './doc-sync.mcp';
|
||||
import { DocSyncService } from './doc-sync.service';
|
||||
import { DOCUMENT_PROVIDERS, type DocumentProvider } from './document-provider';
|
||||
import { DocumentProviderRegistry } from './document-provider.registry';
|
||||
import { PaperlessDocumentProvider } from './providers/paperless.provider';
|
||||
import { PapraDocumentProvider } from './providers/papra.provider';
|
||||
import { SynologyDriveDocumentProvider } from './providers/synology-drive.provider';
|
||||
import { NextcloudDocumentProvider, OpencloudDocumentProvider } from './providers/webdav.provider';
|
||||
import { PaperlessClient } from './providers/paperless.client';
|
||||
import { PapraClient } from './providers/papra.client';
|
||||
import { SynologyDriveClient } from './providers/synology-drive.client';
|
||||
import { WebdavClient } from './providers/webdav.client';
|
||||
|
||||
/**
|
||||
* Document sync: one provider connection per trip, both directions.
|
||||
*
|
||||
* Its own domain rather than an extension of `files/`, because that domain's
|
||||
* coverage ratchet sits at 97/95 and would now be measuring four HTTP clients
|
||||
* and a cron job; and not under `integrations/`, whose ratchet is 78/68 and
|
||||
* would be dragged down by new code landing there. The seam is clean anyway:
|
||||
* this module consumes FilesService and StorageService and nothing consumes it.
|
||||
*
|
||||
* Nextcloud and OpenCloud are two registrations of one WebDAV adapter. They are
|
||||
* separate ids because the admin toggles, the credential fields and the scope
|
||||
* concept differ (a folder with an `oc:fileid` versus a space with a
|
||||
* `driveId`), but the protocol underneath is the same and so is the code.
|
||||
*/
|
||||
@Module({
|
||||
imports: [
|
||||
DatabaseModule,
|
||||
StorageModule,
|
||||
FilesModule,
|
||||
AllowedFileTypesModule,
|
||||
PermissionsModule,
|
||||
RealtimeModule,
|
||||
SchedulingModule,
|
||||
AddonsModule,
|
||||
AuthModule,
|
||||
McpSharedModule,
|
||||
],
|
||||
controllers: [DocSyncController, DocSyncWebhookController],
|
||||
providers: [
|
||||
DocSyncConfigService,
|
||||
DocSyncService,
|
||||
DocSyncJob,
|
||||
DocSyncMcp,
|
||||
DocumentProviderRegistry,
|
||||
// The HTTP clients are providers of their own: each is the ONLY file that
|
||||
// talks to its instance, which makes it the one thing a test has to replace
|
||||
// to be sure a suite never opens a socket.
|
||||
PaperlessClient,
|
||||
PapraClient,
|
||||
SynologyDriveClient,
|
||||
WebdavClient,
|
||||
PaperlessDocumentProvider,
|
||||
PapraDocumentProvider,
|
||||
NextcloudDocumentProvider,
|
||||
OpencloudDocumentProvider,
|
||||
SynologyDriveDocumentProvider,
|
||||
{
|
||||
provide: DOCUMENT_PROVIDERS,
|
||||
useFactory: (
|
||||
paperless: PaperlessDocumentProvider,
|
||||
papra: PapraDocumentProvider,
|
||||
nextcloud: NextcloudDocumentProvider,
|
||||
opencloud: OpencloudDocumentProvider,
|
||||
synology: SynologyDriveDocumentProvider,
|
||||
): readonly DocumentProvider[] => [paperless, papra, nextcloud, opencloud, synology],
|
||||
inject: [
|
||||
PaperlessDocumentProvider,
|
||||
PapraDocumentProvider,
|
||||
NextcloudDocumentProvider,
|
||||
OpencloudDocumentProvider,
|
||||
SynologyDriveDocumentProvider,
|
||||
],
|
||||
},
|
||||
],
|
||||
exports: [DocSyncConfigService, DocSyncService],
|
||||
})
|
||||
export class DocSyncModule {}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { DOCUMENT_PROVIDERS, type DocumentProvider } from './document-provider';
|
||||
|
||||
/**
|
||||
* The one dispatch site for document providers.
|
||||
*
|
||||
* Same shape as PhotoProviderRegistry, and for the same reason: adding a
|
||||
* backend is registering an adapter in the module, not editing a switch that
|
||||
* exists in four places. Whether a registered provider is switched ON is a
|
||||
* different question, answered by the `document_providers` table plus the
|
||||
* `documents` addon. This only knows what the server can talk to at all.
|
||||
*/
|
||||
@Injectable()
|
||||
export class DocumentProviderRegistry {
|
||||
private readonly byId: Map<string, DocumentProvider>;
|
||||
|
||||
constructor(@Inject(DOCUMENT_PROVIDERS) providers: readonly DocumentProvider[]) {
|
||||
this.byId = new Map(providers.map((p) => [p.id, p]));
|
||||
}
|
||||
|
||||
get(id: string | null | undefined): DocumentProvider | undefined {
|
||||
return id ? this.byId.get(id) : undefined;
|
||||
}
|
||||
|
||||
ids(): string[] {
|
||||
return [...this.byId.keys()];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
import type { Readable } from 'node:stream';
|
||||
import type { DocsyncErrorCode } from '@trek/shared';
|
||||
|
||||
/**
|
||||
* The adapter seam for document providers.
|
||||
*
|
||||
* Shaped after `PhotoProvider` (memories/photo-provider.ts): interface, symbol
|
||||
* token, registry. One difference is deliberate: that interface only covers
|
||||
* handing out bytes, and everything else (settings, test, search, album list)
|
||||
* is dispatched through hand-written `if (provider === 'immich')` branches in
|
||||
* the service, the MCP tools and two separate controllers. With two providers
|
||||
* that is untidy; with five it would be five parallel controllers and five
|
||||
* branches per tool. So every operation the sync core needs lives here, and the
|
||||
* core never learns which provider it is talking to.
|
||||
*
|
||||
* Capability differences are answered by data (`DocumentProviderCapabilities`),
|
||||
* never by type-switching: the core asks what a provider can do and plans
|
||||
* accordingly. Synology FileStation has no change feed and no stable file id,
|
||||
* so the core must be correct with nothing but a full enumeration, and because
|
||||
* the weakest provider sets the model, every provider is driven the same way.
|
||||
* A webhook is only ever "look now", never a source of truth.
|
||||
*/
|
||||
|
||||
/** Uniform result shape, mirroring `ServiceResult` in memories.helpers.ts. */
|
||||
export type DocResult<T> =
|
||||
| { success: true; data: T }
|
||||
| { success: false; error: { code: DocsyncErrorCode; detail?: string; status?: number } };
|
||||
|
||||
export type DocFailure = { success: false; error: { code: DocsyncErrorCode; detail?: string; status?: number } };
|
||||
|
||||
export const docOk = <T>(data: T): DocResult<T> => ({ success: true, data });
|
||||
export const docFail = <T>(
|
||||
code: DocsyncErrorCode,
|
||||
detail?: string,
|
||||
status?: number,
|
||||
): DocResult<T> => ({ success: false, error: { code, detail, status } });
|
||||
|
||||
/**
|
||||
* Narrow a result to its failure half.
|
||||
*
|
||||
* `if (!res.success)` does NOT narrow here: the server workspace compiles with
|
||||
* `strict: false`, and without `strictNullChecks` a boolean discriminant stops
|
||||
* discriminating. Testing for the property is what the memories domain already
|
||||
* does (`if ('error' in result)`), and wrapping it in a predicate keeps every
|
||||
* call site reading as an intention rather than as a workaround.
|
||||
*/
|
||||
export function docFailed<T>(result: DocResult<T>): result is DocFailure {
|
||||
return 'error' in result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whose credentials, against which instance.
|
||||
*
|
||||
* A ref object rather than positional arguments, for the reason written into
|
||||
* `PhotoAssetRef`: two providers took the same three values in a different
|
||||
* order, and swapping them compiled cleanly and served another user's data.
|
||||
*/
|
||||
export interface DocumentConnectionRef {
|
||||
connectionId: number;
|
||||
/**
|
||||
* When the connection's row was created, or '' for form values that were
|
||||
* never saved. The id alone does not name one row for good: a backup
|
||||
* restored in place rolls the id sequence back, and the id comes round
|
||||
* again on the next connection saved, on any trip. The two together do.
|
||||
*/
|
||||
createdAt: string;
|
||||
/**
|
||||
* The user whose credentials these are: the trip admin who set the binding
|
||||
* up, not whoever is asking. The core passes both around separately and never
|
||||
* derives one from the other.
|
||||
*/
|
||||
ownerId: number;
|
||||
baseUrl: string;
|
||||
/**
|
||||
* Decrypted `secret = 1` fields, plus whatever the provider earned and kept
|
||||
* through `saveSecret`. Never logged, never returned to a client.
|
||||
*/
|
||||
secrets: Readonly<Record<string, string>>;
|
||||
/** Non-secret fields: username, organization_id, base_path, root_tag, … */
|
||||
settings: Readonly<Record<string, string>>;
|
||||
allowInsecureTls: boolean;
|
||||
/**
|
||||
* Keep a secret the provider earned itself (DSM's device token) with this
|
||||
* connection, or drop it with null. `secrets` is the snapshot this ref was
|
||||
* made from and does not change when this writes.
|
||||
*
|
||||
* Only a saved connection has one. A probe of form values may use what is
|
||||
* stored but never changes it: the values on screen may be for another
|
||||
* account, or never be saved at all.
|
||||
*/
|
||||
saveSecret?: (key: string, value: string | null) => void;
|
||||
}
|
||||
|
||||
/** The container at the provider that one trip is bound to. */
|
||||
export interface DocumentScopeRef {
|
||||
linkId: number;
|
||||
tripId: number;
|
||||
/** Opaque, normalised, never empty. See `docsyncScopeKeySchema`. */
|
||||
scopeKey: string;
|
||||
remoteRootId: string | null;
|
||||
remoteRootPath: string | null;
|
||||
/**
|
||||
* Whatever the adapter stored last run. A WebDAV adapter puts the root ETag
|
||||
* here and can skip the whole walk when it is unchanged; Paperless stores the
|
||||
* highest `modified` it saw. The core treats it as a black box and hands it
|
||||
* back untouched.
|
||||
*/
|
||||
cursor: string | null;
|
||||
}
|
||||
|
||||
export interface RemoteDocument {
|
||||
/** Stable where the provider has one; a normalised path where it does not. */
|
||||
remoteId: string;
|
||||
name: string;
|
||||
size: number | null;
|
||||
mimeType: string | null;
|
||||
/**
|
||||
* Opaque version marker. Different means changed. That is all the core may
|
||||
* conclude from it. It is not a content hash, not comparable across
|
||||
* instances, and not ordered.
|
||||
*/
|
||||
remoteVersion: string;
|
||||
/** sha256 hex when the listing carries one, otherwise null. */
|
||||
contentHash: string | null;
|
||||
remoteModifiedAt: string | null;
|
||||
/** Provider-side tombstone, where one exists (Synology Drive `removed`). */
|
||||
isDeleted: boolean;
|
||||
}
|
||||
|
||||
export interface DocumentScopeOption {
|
||||
scopeKey: string;
|
||||
label: string;
|
||||
remoteRootId: string | null;
|
||||
remoteRootPath: string | null;
|
||||
}
|
||||
|
||||
export interface DocumentProviderCapabilities {
|
||||
/**
|
||||
* `webhook-self-registered`: TREK can create the subscription itself
|
||||
* (Paperless workflows). `webhook-manual`: the API exists but is closed to
|
||||
* API keys, so a human has to paste a URL (Papra, and Nextcloud unless the
|
||||
* account is an admin). `none`: polling only (Synology).
|
||||
*/
|
||||
push: 'webhook-self-registered' | 'webhook-manual' | 'none';
|
||||
/** False means identity is the path, so rename looks like delete + create. */
|
||||
stableId: boolean;
|
||||
/** A delete lands in a recycle bin the user can recover from. */
|
||||
remoteTrash: boolean;
|
||||
/** A new revision can replace the bytes under the same remoteId. */
|
||||
replaceInPlace: boolean;
|
||||
/** The listing already carries a content hash, so no extra round trip. */
|
||||
contentHashInListing: boolean;
|
||||
maxUploadBytes: number | null;
|
||||
/** null means anything; otherwise the provider's own MIME allowlist. */
|
||||
acceptedMimeTypes: readonly string[] | null;
|
||||
/** The adapter can create a new scope container, not just pick one. */
|
||||
canCreateScope: boolean;
|
||||
}
|
||||
|
||||
export interface PushRequest {
|
||||
/** Bytes, streamed straight out of `StorageService.getStream('files', …)`. */
|
||||
body: Readable;
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
/** sha256 of those bytes; adapters that can pin it upstream should. */
|
||||
sha256: string;
|
||||
/**
|
||||
* Unix seconds. WebDAV sends `X-OC-MTime`, Synology multiplies by 1000.
|
||||
* Without it every file TREK uploads comes back looking freshly changed,
|
||||
* which is the classic sync loop.
|
||||
*/
|
||||
mtimeSeconds: number;
|
||||
/** Present means replace that document rather than create a new one. */
|
||||
remoteId?: string;
|
||||
/** Optimistic concurrency (WebDAV `If-Match`); ignored where unsupported. */
|
||||
expectedRemoteVersion?: string;
|
||||
/**
|
||||
* The anchors that survive a human renaming or re-tagging things upstream.
|
||||
* Written into a custom field, a property or a tag where the provider has
|
||||
* somewhere to put them.
|
||||
*/
|
||||
trekDocUid: string;
|
||||
trekTripUid: string;
|
||||
}
|
||||
|
||||
export interface PushResult {
|
||||
remoteId: string;
|
||||
remoteVersion: string;
|
||||
remoteModifiedAt: string | null;
|
||||
/**
|
||||
* True when the provider matched existing bytes instead of storing new ones
|
||||
* (Papra answers 409 on a duplicate hash, and restores a trashed twin under
|
||||
* its ORIGINAL id). The core must not book that as a fresh document.
|
||||
*/
|
||||
deduplicated: boolean;
|
||||
}
|
||||
|
||||
export interface FetchResult {
|
||||
body: Readable;
|
||||
size: number | null;
|
||||
mimeType: string | null;
|
||||
remoteVersion: string;
|
||||
}
|
||||
|
||||
export interface DocumentProvider {
|
||||
readonly id: string;
|
||||
|
||||
capabilities(conn: DocumentConnectionRef): DocumentProviderCapabilities;
|
||||
|
||||
/**
|
||||
* Verify form values before anything is stored, and report what the account
|
||||
* can actually do. Always resolves: a failed probe is data, not an
|
||||
* exception, because the settings UI shows it inline (the photo providers
|
||||
* pin that with a test marked CRITICAL in their e2e suite).
|
||||
*/
|
||||
probe(conn: DocumentConnectionRef): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>>;
|
||||
|
||||
/** Offer the containers a trip could be bound to: folders, tags, spaces. */
|
||||
listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>>;
|
||||
|
||||
/** Create one, where the provider has the concept. */
|
||||
createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>>;
|
||||
|
||||
/** Confirm the bound scope still exists, so `scope_lost` is detectable. */
|
||||
resolveScope(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<DocumentScopeOption>>;
|
||||
|
||||
/**
|
||||
* Enumerate everything currently in the scope, plus the cursor to hand back
|
||||
* next time.
|
||||
*
|
||||
* Full enumeration rather than a delta, deliberately: a changed-since query
|
||||
* cannot see a deletion, several of these providers bump no timestamp when a
|
||||
* tag changes, and Paperless's bulk edit changes documents without touching
|
||||
* `modified` at all. `cursorUnchanged` lets an adapter short-circuit the walk
|
||||
* (the WebDAV root ETag) while keeping the core's model identical.
|
||||
*/
|
||||
list(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<{
|
||||
documents: RemoteDocument[];
|
||||
cursor: string | null;
|
||||
cursorUnchanged: boolean;
|
||||
/** True when the listing hit a page cap and is therefore not the whole truth. */
|
||||
truncated: boolean;
|
||||
}>>;
|
||||
|
||||
fetch(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): Promise<DocResult<FetchResult>>;
|
||||
|
||||
push(conn: DocumentConnectionRef, scope: DocumentScopeRef, req: PushRequest): Promise<DocResult<PushResult>>;
|
||||
|
||||
rename(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string, name: string): Promise<DocResult<{ remoteVersion: string }>>;
|
||||
|
||||
/**
|
||||
* Move to the provider's recycle bin. Never a hard delete: Papra and
|
||||
* Paperless only expose permanent deletion to a session, not to a token, and
|
||||
* on the others an accidental propagation would be unrecoverable.
|
||||
*/
|
||||
trash(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): Promise<DocResult<void>>;
|
||||
|
||||
/** Subscribe to change notifications, where TREK is allowed to do it itself. */
|
||||
registerWebhook?(conn: DocumentConnectionRef, scope: DocumentScopeRef, callbackUrl: string, secret: string): Promise<DocResult<{ subscriptionId: string }>>;
|
||||
unregisterWebhook?(conn: DocumentConnectionRef, subscriptionId: string): Promise<DocResult<void>>;
|
||||
}
|
||||
|
||||
export const DOCUMENT_PROVIDERS = Symbol('DOCUMENT_PROVIDERS');
|
||||
@@ -0,0 +1,894 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { DocsyncErrorCode } from '@trek/shared';
|
||||
import { readCappedJson, readCappedText, discardBody } from '../../../utils/cappedFetch';
|
||||
import {
|
||||
providerFetch,
|
||||
statusErrorCode,
|
||||
type TransportFailure,
|
||||
type TransportFailureCode,
|
||||
} from './provider-http';
|
||||
import {
|
||||
PROVIDER_JSON_MAX_BYTES,
|
||||
PROVIDER_MAX_PAGES,
|
||||
PROVIDER_TIMEOUT_MS,
|
||||
PROVIDER_TRANSFER_TIMEOUT_MS,
|
||||
} from '../doc-sync.constants';
|
||||
|
||||
/**
|
||||
* Thin HTTP client for the Paperless-ngx REST API.
|
||||
* This is the ONLY place that talks to a trip's Paperless instance.
|
||||
*
|
||||
* Measured against Paperless-ngx 3.1.3 / API version 10, not against its
|
||||
* documentation. The two disagree in the places that matter here:
|
||||
*
|
||||
* - **A document carries no top-level `checksum` at 3.x.** The hash lives in
|
||||
* `versions[]`, one entry per uploaded revision, and it is a **sha256** of
|
||||
* the original file (verified: an uploaded file's sha256 came back verbatim).
|
||||
* `versions` arrives newest first with the ORIGINAL marked `is_root: true`,
|
||||
* so the entry describing the current bytes is the newest `added`, not the
|
||||
* root one.
|
||||
* - `GET /api/documents/{id}/download/` hands out the **archived**, OCR'd PDF,
|
||||
* whose bytes are not the ones that were uploaded (398 B in, 8483 B out).
|
||||
* Only `?original=true` round-trips, and only it matches the checksum.
|
||||
* - **An upload answers with a task UUID, not a document.** The consume job is
|
||||
* asynchronous (~3 s for a one-page PDF with OCR), and `GET /api/tasks/`
|
||||
* answers a PAGINATED object at 3.x where older builds answered a bare
|
||||
* array. `status` is lower case (`success`, `failure`) where older builds
|
||||
* shouted it, so both spellings are compared case-insensitively.
|
||||
* - **`POST /api/documents/{id}/update_version/` keeps the document id**, and
|
||||
* with it the tags and custom fields, but the task it returns reports a
|
||||
* DIFFERENT, internal document id (the version row), which answers 404 on
|
||||
* `/api/documents/{id}/`. Callers must keep the id they updated.
|
||||
* - **Duplicate bytes are not refused.** Uploading the same file twice yields
|
||||
* two documents with the same checksum and an empty `duplicate_documents`,
|
||||
* so there is no upstream deduplication to report. `?checksum__iexact=` does
|
||||
* filter, which is what makes a lost upload recoverable instead of doubled.
|
||||
* - **Unknown query parameters are ignored, not rejected.** A filter that does
|
||||
* not exist silently widens the result, so every filter used here was proven
|
||||
* with a matching and a non-matching value.
|
||||
* - The MIME allow-list is enforced by the server: an unsupported type comes
|
||||
* back as `400 {"document": ["File type … not supported"]}` before any task
|
||||
* is created. Office formats pass only on an instance with Tika configured.
|
||||
*
|
||||
* The API version is pinned in the Accept header. Paperless moves its default
|
||||
* with the server version, so an unpinned client changes shape under a routine
|
||||
* `docker pull`. That is exactly how the `checksum` field moved out of the
|
||||
* document serializer.
|
||||
*/
|
||||
|
||||
/** The version this client's response handling was written against. */
|
||||
const API_ACCEPT = 'application/json; version=10';
|
||||
|
||||
/** Tags, documents and tasks are small rows; a page of them is never megabytes. */
|
||||
const MAX_JSON_BYTES = PROVIDER_JSON_MAX_BYTES;
|
||||
|
||||
/** Documents per page. Paperless accepts more, but the response grows with it. */
|
||||
export const DOCUMENT_PAGE_SIZE = 100;
|
||||
export const TAG_PAGE_SIZE = 100;
|
||||
|
||||
/** How often the consume task is polled, and for how long before giving up. */
|
||||
export const TASK_POLL_INTERVAL_MS = 1500;
|
||||
export const TASK_POLL_TIMEOUT_MS = PROVIDER_TRANSFER_TIMEOUT_MS;
|
||||
|
||||
export interface PaperlessCreds {
|
||||
/** Instance origin. The client appends `/api` itself. */
|
||||
baseUrl: string;
|
||||
/** The API token from the user's Paperless profile. Never logged. */
|
||||
token: string;
|
||||
allowInsecureTls: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed Paperless call, carrying the reason as one of TREK's codes.
|
||||
*
|
||||
* The upstream text never reaches a user: it is English, it sometimes quotes
|
||||
* the URL back, and behind an auth proxy it is an HTML login page. `detail`
|
||||
* keeps it for the self-hoster reading their own log.
|
||||
*/
|
||||
export class PaperlessError extends Error {
|
||||
readonly code: DocsyncErrorCode;
|
||||
readonly status?: number;
|
||||
readonly detail?: string;
|
||||
|
||||
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string) {
|
||||
super(message);
|
||||
this.name = 'PaperlessError';
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PaperlessTag {
|
||||
id: number;
|
||||
name: string;
|
||||
/** Absent on the POST response, present in listings. */
|
||||
documentCount: number | null;
|
||||
}
|
||||
|
||||
export interface PaperlessCustomField {
|
||||
id: number;
|
||||
name: string;
|
||||
dataType: string | null;
|
||||
}
|
||||
|
||||
/** One entry of a document's `versions`, i.e. one uploaded revision. */
|
||||
export interface PaperlessVersion {
|
||||
id: number;
|
||||
/** sha256 hex of that revision's original file. */
|
||||
checksum: string | null;
|
||||
added: string | null;
|
||||
isRoot: boolean;
|
||||
}
|
||||
|
||||
export interface PaperlessDocument {
|
||||
id: number;
|
||||
title: string;
|
||||
/** The name the file was uploaded under; null on documents created elsewhere. */
|
||||
originalFileName: string | null;
|
||||
mimeType: string | null;
|
||||
/** The opaque version marker. Bumped by edits, uploads and bulk edits alike. */
|
||||
modified: string | null;
|
||||
added: string | null;
|
||||
/** Non-null only in `/api/trash/`; a listed document is never trashed. */
|
||||
deletedAt: string | null;
|
||||
tagIds: number[];
|
||||
versions: PaperlessVersion[];
|
||||
}
|
||||
|
||||
export interface PaperlessTask {
|
||||
taskId: string;
|
||||
/** Lower case at 3.x, upper case on older builds. Compared case-insensitively. */
|
||||
status: string;
|
||||
documentIds: number[];
|
||||
errorMessage: string | null;
|
||||
}
|
||||
|
||||
export interface PaperlessProfile {
|
||||
username: string;
|
||||
version: string | null;
|
||||
/** Django permission codenames the token's user holds, e.g. `add_workflow`. */
|
||||
permissions: string[];
|
||||
}
|
||||
|
||||
export interface PaperlessUploadFile {
|
||||
fileName: string;
|
||||
mimeType: string;
|
||||
bytes: Buffer;
|
||||
}
|
||||
|
||||
export interface PaperlessWorkflowSpec {
|
||||
name: string;
|
||||
/** Only documents carrying this tag fire the webhook. */
|
||||
tagId: number;
|
||||
callbackUrl: string;
|
||||
/** Sent as a request header, so it never lands in an access log. */
|
||||
secretHeader: { name: string; value: string };
|
||||
/** Flat string map; Paperless sends it as the JSON body. */
|
||||
params: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Paperless's own ceiling for a webhook URL (`max_length` on the field). A
|
||||
* longer callback is rejected by the serializer with a 400 that reads like a
|
||||
* generic validation failure, so it is worth catching before the round trip.
|
||||
*/
|
||||
export const WEBHOOK_URL_MAX_LENGTH = 256;
|
||||
|
||||
/**
|
||||
* Workflow trigger types, as `OPTIONS /api/workflows/` enumerates them. There
|
||||
* is deliberately no deletion trigger upstream: a document moved to the trash
|
||||
* fires nothing, which is why a webhook can only ever mean "look now".
|
||||
*/
|
||||
export const TRIGGER_DOCUMENT_ADDED = 2;
|
||||
export const TRIGGER_DOCUMENT_UPDATED = 3;
|
||||
/** Action type 4 is "Webhook". */
|
||||
export const ACTION_WEBHOOK = 4;
|
||||
/**
|
||||
* Consume folder, API upload, mail fetch, web UI: all four, or a document
|
||||
* dropped into Paperless's own interface would never reach TREK.
|
||||
*/
|
||||
const TRIGGER_SOURCES = [1, 2, 3, 4];
|
||||
|
||||
/**
|
||||
* `https://host/paperless/` → `https://host/paperless/api`, tolerating what
|
||||
* people paste: a trailing slash, or the `/api` they copied out of the browser.
|
||||
*/
|
||||
export function apiBase(baseUrl: string): string {
|
||||
const origin = baseUrl
|
||||
.trim()
|
||||
.replace(/(?<!\/)\/+$/, '')
|
||||
.replace(/\/api$/i, '');
|
||||
return `${origin}/api`;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | null {
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
|
||||
function asNumber(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows of a paginated answer, or of a bare array.
|
||||
*
|
||||
* Both shapes are live: `/api/tasks/` answers a bare array on older builds and
|
||||
* a paginated object at 3.x, and reading only one of them would make the upload
|
||||
* path fail on whichever half of the installed base it guessed wrong.
|
||||
*/
|
||||
function pageRows(data: unknown): { rows: unknown[]; hasNext: boolean; count: number | null } {
|
||||
if (Array.isArray(data)) return { rows: data, hasNext: false, count: data.length };
|
||||
if (isRecord(data) && Array.isArray(data.results)) {
|
||||
return {
|
||||
rows: data.results,
|
||||
hasNext: typeof data.next === 'string' && data.next.length > 0,
|
||||
count: asNumber(data.count),
|
||||
};
|
||||
}
|
||||
throw new PaperlessError('provider_error', 'Paperless answered a listing in an unknown shape');
|
||||
}
|
||||
|
||||
function parseTag(row: unknown): PaperlessTag | null {
|
||||
if (!isRecord(row)) return null;
|
||||
const id = asNumber(row.id);
|
||||
const name = asString(row.name);
|
||||
if (id === null || name === null) return null;
|
||||
return { id, name, documentCount: asNumber(row.document_count) };
|
||||
}
|
||||
|
||||
function parseCustomField(row: unknown): PaperlessCustomField | null {
|
||||
if (!isRecord(row)) return null;
|
||||
const id = asNumber(row.id);
|
||||
const name = asString(row.name);
|
||||
if (id === null || name === null) return null;
|
||||
return { id, name, dataType: asString(row.data_type) };
|
||||
}
|
||||
|
||||
function parseVersions(value: unknown): PaperlessVersion[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
const versions: PaperlessVersion[] = [];
|
||||
for (const row of value) {
|
||||
if (!isRecord(row)) continue;
|
||||
const id = asNumber(row.id);
|
||||
if (id === null) continue;
|
||||
versions.push({
|
||||
id,
|
||||
checksum: asString(row.checksum),
|
||||
added: asString(row.added),
|
||||
isRoot: row.is_root === true,
|
||||
});
|
||||
}
|
||||
return versions;
|
||||
}
|
||||
|
||||
function parseDocument(row: unknown): PaperlessDocument | null {
|
||||
if (!isRecord(row)) return null;
|
||||
const id = asNumber(row.id);
|
||||
if (id === null) return null;
|
||||
const tagIds = Array.isArray(row.tags)
|
||||
? row.tags.filter((tag): tag is number => typeof tag === 'number')
|
||||
: [];
|
||||
return {
|
||||
id,
|
||||
title: asString(row.title) ?? '',
|
||||
originalFileName: asString(row.original_file_name),
|
||||
mimeType: asString(row.mime_type),
|
||||
modified: asString(row.modified),
|
||||
added: asString(row.added),
|
||||
deletedAt: asString(row.deleted_at),
|
||||
tagIds,
|
||||
versions: parseVersions(row.versions),
|
||||
};
|
||||
}
|
||||
|
||||
function parseTask(row: unknown): PaperlessTask | null {
|
||||
if (!isRecord(row)) return null;
|
||||
const taskId = asString(row.task_id);
|
||||
if (taskId === null) return null;
|
||||
const related = Array.isArray(row.related_document_ids)
|
||||
? row.related_document_ids.filter((value): value is number => typeof value === 'number')
|
||||
: [];
|
||||
const fromResult = isRecord(row.result_data) ? asNumber(row.result_data.document_id) : null;
|
||||
const documentIds = related.length > 0 ? related : fromResult === null ? [] : [fromResult];
|
||||
return {
|
||||
taskId,
|
||||
status: asString(row.status) ?? '',
|
||||
documentIds,
|
||||
errorMessage: isRecord(row.result_data) ? asString(row.result_data.error_message) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The revision whose bytes `?original=true` currently serves.
|
||||
*
|
||||
* `versions` arrives newest first at 3.1.3, but that is an ordering nobody
|
||||
* promised, so the newest `added` decides and the array order is only the
|
||||
* tie-breaker. The root entry is explicitly NOT it: after an in-place replace
|
||||
* the root is the file that was replaced.
|
||||
*/
|
||||
export function currentVersion(doc: PaperlessDocument): PaperlessVersion | null {
|
||||
let best: PaperlessVersion | null = null;
|
||||
let bestAt = Number.NEGATIVE_INFINITY;
|
||||
for (const version of doc.versions) {
|
||||
const parsed = version.added === null ? Number.NaN : Date.parse(version.added);
|
||||
const rank = Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
||||
if (best === null || rank > bestAt) {
|
||||
best = version;
|
||||
bestAt = rank;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/** sha256 hex, or null. A hash of another shape must not be sold as one. */
|
||||
export function sha256OrNull(value: string | null): string | null {
|
||||
return value !== null && /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
|
||||
}
|
||||
|
||||
const TRANSPORT_MESSAGES: Record<TransportFailureCode, string> = {
|
||||
ssrf_blocked: 'The address is blocked',
|
||||
timeout: 'Paperless did not answer in time',
|
||||
tls_untrusted: 'The certificate of this Paperless instance is not trusted',
|
||||
unreachable: 'Could not reach Paperless',
|
||||
};
|
||||
|
||||
function transportError(failure: TransportFailure): PaperlessError {
|
||||
return new PaperlessError(failure.code, TRANSPORT_MESSAGES[failure.code], undefined, failure.detail);
|
||||
}
|
||||
|
||||
/**
|
||||
* The code for an HTTP status, refined by the body where the status alone is
|
||||
* ambiguous. Paperless answers 400 both for "I do not take this file type"
|
||||
* (a per-document verdict the user can act on) and for a malformed request,
|
||||
* and the difference is only in the text.
|
||||
*/
|
||||
function classify(status: number, body: string): DocsyncErrorCode {
|
||||
if (status === 400) {
|
||||
if (/not supported/i.test(body)) return 'unsupported_type';
|
||||
if (/unique constraint|already exists/i.test(body)) return 'conflict';
|
||||
return 'provider_error';
|
||||
}
|
||||
return statusErrorCode(status);
|
||||
}
|
||||
|
||||
/** Trim an upstream message to something a log line can hold. */
|
||||
function shorten(text: string): string {
|
||||
const flat = text.replace(/\s+/g, ' ').trim();
|
||||
return flat.length > 400 ? `${flat.slice(0, 400)}…` : flat;
|
||||
}
|
||||
|
||||
interface RequestOptions {
|
||||
method?: string;
|
||||
query?: Record<string, string | undefined>;
|
||||
json?: unknown;
|
||||
body?: Buffer;
|
||||
contentType?: string;
|
||||
timeoutMs?: number;
|
||||
/** Streaming responses are handed back unread; everything else is capped. */
|
||||
raw?: boolean;
|
||||
}
|
||||
|
||||
interface MultipartField {
|
||||
name: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Everything a header value may not contain, replaced rather than rejected. */
|
||||
function headerSafe(value: string): string {
|
||||
return value.replace(/[\r\n"\\]/g, '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* A multipart/form-data body, built by hand.
|
||||
*
|
||||
* `FormData` plus `Blob` would be shorter but hands undici a body without a
|
||||
* length; a Buffer keeps Content-Length honest, which is what lets a reverse
|
||||
* proxy in front of Paperless enforce its own upload limit instead of buffering
|
||||
* an unbounded stream. Field and file names are stripped of quotes and line
|
||||
* breaks first: a document called `a".pdf` would otherwise end the
|
||||
* Content-Disposition header early and rewrite the rest of the request.
|
||||
*/
|
||||
export function buildMultipart(
|
||||
fields: MultipartField[],
|
||||
file: PaperlessUploadFile,
|
||||
): { body: Buffer; contentType: string } {
|
||||
const boundary = `----TrekDocSync${randomUUID().replace(/-/g, '')}`;
|
||||
const parts: Buffer[] = [];
|
||||
for (const field of fields) {
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="${headerSafe(field.name)}"\r\n\r\n${field.value}\r\n`,
|
||||
'utf8',
|
||||
),
|
||||
);
|
||||
}
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="document"; filename="${headerSafe(file.fileName)}"\r\n` +
|
||||
`Content-Type: ${headerSafe(file.mimeType)}\r\n\r\n`,
|
||||
'utf8',
|
||||
),
|
||||
);
|
||||
parts.push(file.bytes);
|
||||
parts.push(Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8'));
|
||||
return {
|
||||
body: Buffer.concat(parts),
|
||||
contentType: `multipart/form-data; boundary=${boundary}`,
|
||||
};
|
||||
}
|
||||
|
||||
export class PaperlessClient {
|
||||
/**
|
||||
* One authenticated request. Every call in this file goes through it, so the
|
||||
* token header, the pinned API version, the timeout and the TLS decision are
|
||||
* made in exactly one place.
|
||||
*/
|
||||
private async request(
|
||||
creds: PaperlessCreds,
|
||||
path: string,
|
||||
options: RequestOptions = {},
|
||||
): Promise<Response> {
|
||||
const url = new URL(apiBase(creds.baseUrl) + path);
|
||||
for (const [key, value] of Object.entries(options.query ?? {})) {
|
||||
if (value !== undefined) url.searchParams.set(key, value);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Authorization: `Token ${creds.token}`,
|
||||
Accept: options.raw === true ? '*/*' : API_ACCEPT,
|
||||
};
|
||||
let body: Buffer | string | undefined;
|
||||
if (options.json !== undefined) {
|
||||
body = JSON.stringify(options.json);
|
||||
headers['Content-Type'] = 'application/json';
|
||||
} else if (options.body !== undefined) {
|
||||
body = options.body;
|
||||
headers['Content-Type'] = options.contentType ?? 'application/octet-stream';
|
||||
// Content-Length is deliberately NOT set: undici derives it from a buffer
|
||||
// body and rejects the request outright (`UND_ERR_INVALID_ARG: invalid
|
||||
// content-length header`) when the header is also present, which surfaces
|
||||
// as a bare `TypeError: fetch failed` two layers up.
|
||||
}
|
||||
|
||||
const response = await providerFetch(
|
||||
url.toString(),
|
||||
{ method: options.method ?? 'GET', headers, body },
|
||||
{
|
||||
timeoutMs: options.timeoutMs ?? PROVIDER_TIMEOUT_MS,
|
||||
allowInsecureTls: creds.allowInsecureTls,
|
||||
onTransportFailure: transportError,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
// The body is what separates "wrong file type" from "wrong request", so it
|
||||
// is read, through the cap, because a proxy's error page is HTML of
|
||||
// unbounded length.
|
||||
const { text } = await readCappedText(response, MAX_JSON_BYTES);
|
||||
throw new PaperlessError(
|
||||
classify(response.status, text),
|
||||
`Paperless answered HTTP ${response.status}`,
|
||||
response.status,
|
||||
shorten(text),
|
||||
);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
private async requestJson<T>(
|
||||
creds: PaperlessCreds,
|
||||
path: string,
|
||||
options: RequestOptions = {},
|
||||
): Promise<T> {
|
||||
const response = await this.request(creds, path, options);
|
||||
const data = await readCappedJson<T>(response, MAX_JSON_BYTES);
|
||||
if (data === undefined) {
|
||||
throw new PaperlessError(
|
||||
'provider_error',
|
||||
'Paperless returned something that is not JSON',
|
||||
response.status,
|
||||
'Check that the URL points at Paperless itself and is reachable without a separate login.',
|
||||
);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Who the token belongs to, which build answers, and what that user may do.
|
||||
*
|
||||
* `/api/ui_settings/` rather than `/api/profile/`: profile echoes the token
|
||||
* back in its own body, and nothing here should carry a credential it does not
|
||||
* need. ui_settings also carries the permission list, which is the only way to
|
||||
* know in advance whether this token may create the webhook workflow.
|
||||
*/
|
||||
async probe(creds: PaperlessCreds): Promise<PaperlessProfile> {
|
||||
const data = await this.requestJson<unknown>(creds, '/ui_settings/');
|
||||
if (!isRecord(data)) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown profile shape');
|
||||
}
|
||||
const user = isRecord(data.user) ? data.user : {};
|
||||
const settings = isRecord(data.settings) ? data.settings : {};
|
||||
return {
|
||||
username: asString(user.username) ?? '',
|
||||
version: asString(settings.version),
|
||||
permissions: Array.isArray(data.permissions)
|
||||
? data.permissions.filter((permission): permission is string => typeof permission === 'string')
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Every tag, or those whose name contains `query`. */
|
||||
async listTags(creds: PaperlessCreds, query?: string): Promise<PaperlessTag[]> {
|
||||
const tags: PaperlessTag[] = [];
|
||||
for (let page = 1; page <= PROVIDER_MAX_PAGES; page++) {
|
||||
const data = await this.requestJson<unknown>(creds, '/tags/', {
|
||||
query: {
|
||||
page: String(page),
|
||||
page_size: String(TAG_PAGE_SIZE),
|
||||
ordering: 'name',
|
||||
name__icontains: query !== undefined && query.trim().length > 0 ? query.trim() : undefined,
|
||||
},
|
||||
});
|
||||
const { rows, hasNext } = pageRows(data);
|
||||
for (const row of rows) {
|
||||
const tag = parseTag(row);
|
||||
if (tag !== null) tags.push(tag);
|
||||
}
|
||||
if (!hasNext) break;
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
async getTag(creds: PaperlessCreds, tagId: number): Promise<PaperlessTag> {
|
||||
const tag = parseTag(await this.requestJson<unknown>(creds, `/tags/${tagId}/`));
|
||||
if (tag === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown tag shape');
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
|
||||
async createTag(creds: PaperlessCreds, name: string): Promise<PaperlessTag> {
|
||||
const tag = parseTag(
|
||||
await this.requestJson<unknown>(creds, '/tags/', { method: 'POST', json: { name } }),
|
||||
);
|
||||
if (tag === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown tag shape');
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every document carrying the tag, oldest id first.
|
||||
*
|
||||
* `tags__id__all` rather than `tags__id__in`: with a single tag the two agree,
|
||||
* and `__all` keeps meaning "carries this tag" if a caller ever passes more.
|
||||
* Ordering by id keeps the pages stable while documents are being added:
|
||||
* ordering by `modified`, the field that changes under us, would let a
|
||||
* document move between pages and never be seen at all.
|
||||
*
|
||||
* Pages are walked by number instead of by following `next`: Paperless builds
|
||||
* that URL from the request's own Host header, so behind a reverse proxy it
|
||||
* can name an origin this connection was never allowed to talk to.
|
||||
*/
|
||||
async listDocuments(
|
||||
creds: PaperlessCreds,
|
||||
tagId: number,
|
||||
): Promise<{ documents: PaperlessDocument[]; truncated: boolean; count: number | null }> {
|
||||
const documents: PaperlessDocument[] = [];
|
||||
let truncated = false;
|
||||
let count: number | null = null;
|
||||
|
||||
for (let page = 1; page <= PROVIDER_MAX_PAGES; page++) {
|
||||
const data = await this.requestJson<unknown>(creds, '/documents/', {
|
||||
query: {
|
||||
tags__id__all: String(tagId),
|
||||
ordering: 'id',
|
||||
page: String(page),
|
||||
page_size: String(DOCUMENT_PAGE_SIZE),
|
||||
},
|
||||
});
|
||||
const { rows, hasNext, count: total } = pageRows(data);
|
||||
count ??= total;
|
||||
for (const row of rows) {
|
||||
const doc = parseDocument(row);
|
||||
if (doc !== null) documents.push(doc);
|
||||
}
|
||||
if (!hasNext) break;
|
||||
if (page === PROVIDER_MAX_PAGES) truncated = true;
|
||||
}
|
||||
|
||||
return { documents, truncated, count };
|
||||
}
|
||||
|
||||
async getDocument(creds: PaperlessCreds, documentId: number): Promise<PaperlessDocument> {
|
||||
const doc = parseDocument(await this.requestJson<unknown>(creds, `/documents/${documentId}/`));
|
||||
if (doc === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown document shape');
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Documents whose original file has exactly these bytes.
|
||||
*
|
||||
* The filter is server-side and exact. It exists here for one case: an upload
|
||||
* whose consume task was never observed, where the document may or may not
|
||||
* have landed and a blind retry would leave two: Paperless does not refuse
|
||||
* duplicate bytes.
|
||||
*/
|
||||
async findByChecksum(creds: PaperlessCreds, sha256: string): Promise<PaperlessDocument[]> {
|
||||
const data = await this.requestJson<unknown>(creds, '/documents/', {
|
||||
query: { checksum__iexact: sha256, ordering: '-id', page_size: String(DOCUMENT_PAGE_SIZE) },
|
||||
});
|
||||
const { rows } = pageRows(data);
|
||||
const documents: PaperlessDocument[] = [];
|
||||
for (const row of rows) {
|
||||
const doc = parseDocument(row);
|
||||
if (doc !== null) documents.push(doc);
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
async patchDocument(
|
||||
creds: PaperlessCreds,
|
||||
documentId: number,
|
||||
patch: { title?: string },
|
||||
): Promise<PaperlessDocument> {
|
||||
const doc = parseDocument(
|
||||
await this.requestJson<unknown>(creds, `/documents/${documentId}/`, {
|
||||
method: 'PATCH',
|
||||
json: patch,
|
||||
}),
|
||||
);
|
||||
if (doc === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown document shape');
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Moves the document to Paperless's trash. There is no hard delete here. */
|
||||
async deleteDocument(creds: PaperlessCreds, documentId: number): Promise<void> {
|
||||
discardBody(await this.request(creds, `/documents/${documentId}/`, { method: 'DELETE' }));
|
||||
}
|
||||
|
||||
/**
|
||||
* The ORIGINAL bytes, streaming. The caller owns the body from here.
|
||||
*
|
||||
* Without `original=true` Paperless serves the archived PDF it produced
|
||||
* itself, which is a different file with a different length and a different
|
||||
* hash. TREK would store something the user never uploaded, and every
|
||||
* comparison against the checksum would fail.
|
||||
*/
|
||||
async downloadOriginal(creds: PaperlessCreds, documentId: number): Promise<Response> {
|
||||
return this.request(creds, `/documents/${documentId}/download/`, {
|
||||
query: { original: 'true' },
|
||||
timeoutMs: PROVIDER_TRANSFER_TIMEOUT_MS,
|
||||
raw: true,
|
||||
});
|
||||
}
|
||||
|
||||
/** Upload a new document. Answers the consume task's id, not a document id. */
|
||||
async postDocument(
|
||||
creds: PaperlessCreds,
|
||||
file: PaperlessUploadFile,
|
||||
fields: { title?: string; tagId: number; customFields?: Record<string, string> },
|
||||
): Promise<string> {
|
||||
const parts: MultipartField[] = [{ name: 'tags', value: String(fields.tagId) }];
|
||||
if (fields.title !== undefined && fields.title.length > 0) {
|
||||
parts.push({ name: 'title', value: fields.title });
|
||||
}
|
||||
if (fields.customFields !== undefined && Object.keys(fields.customFields).length > 0) {
|
||||
// Paperless expects the custom fields as ONE JSON object in a single form
|
||||
// field, keyed by field id. Repeating `custom_fields=<id>` instead
|
||||
// attaches the fields with no value, which loses the anchor silently.
|
||||
parts.push({ name: 'custom_fields', value: JSON.stringify(fields.customFields) });
|
||||
}
|
||||
return this.postMultipart(creds, '/documents/post_document/', parts, file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the bytes of an existing document, keeping its id, tags and custom
|
||||
* fields. Answers the consume task's id.
|
||||
*/
|
||||
async updateVersion(
|
||||
creds: PaperlessCreds,
|
||||
documentId: number,
|
||||
file: PaperlessUploadFile,
|
||||
): Promise<string> {
|
||||
return this.postMultipart(creds, `/documents/${documentId}/update_version/`, [], file);
|
||||
}
|
||||
|
||||
private async postMultipart(
|
||||
creds: PaperlessCreds,
|
||||
path: string,
|
||||
fields: MultipartField[],
|
||||
file: PaperlessUploadFile,
|
||||
): Promise<string> {
|
||||
const { body, contentType } = buildMultipart(fields, file);
|
||||
const data = await this.requestJson<unknown>(creds, path, {
|
||||
method: 'POST',
|
||||
body,
|
||||
contentType,
|
||||
timeoutMs: PROVIDER_TRANSFER_TIMEOUT_MS,
|
||||
});
|
||||
// The whole answer is the task UUID as a bare JSON string.
|
||||
const taskId = asString(data);
|
||||
if (taskId === null || taskId.length === 0) {
|
||||
throw new PaperlessError('provider_error', 'Paperless accepted the upload without naming a task');
|
||||
}
|
||||
return taskId;
|
||||
}
|
||||
|
||||
async getTask(creds: PaperlessCreds, taskId: string): Promise<PaperlessTask | null> {
|
||||
const data = await this.requestJson<unknown>(creds, '/tasks/', { query: { task_id: taskId } });
|
||||
const { rows } = pageRows(data);
|
||||
for (const row of rows) {
|
||||
const task = parseTask(row);
|
||||
// `task_id` is compared rather than trusted: the filter is a query
|
||||
// parameter, and Paperless ignores parameters it does not know instead of
|
||||
// rejecting them, so an older build would answer with every task there is.
|
||||
if (task !== null && task.taskId === taskId) return task;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a consume task to finish and answer the document it produced.
|
||||
*
|
||||
* Bounded twice: each poll carries its own request timeout, and the loop as a
|
||||
* whole gives up at `timeoutMs`. A large scan with OCR can outlast any budget
|
||||
* worth holding a worker for, and the caller has a cheaper way to find out
|
||||
* whether the upload landed (the checksum filter), so this reports `timeout`
|
||||
* rather than waiting longer.
|
||||
*/
|
||||
async awaitConsume(
|
||||
creds: PaperlessCreds,
|
||||
taskId: string,
|
||||
options: { intervalMs?: number; timeoutMs?: number } = {},
|
||||
): Promise<{ documentId: number }> {
|
||||
const interval = options.intervalMs ?? TASK_POLL_INTERVAL_MS;
|
||||
const deadline = Date.now() + (options.timeoutMs ?? TASK_POLL_TIMEOUT_MS);
|
||||
|
||||
for (;;) {
|
||||
const task = await this.getTask(creds, taskId);
|
||||
const status = task?.status.toLowerCase() ?? '';
|
||||
if (status === 'failure' || status === 'revoked') {
|
||||
// A consume that fails on the file itself (a broken PDF, an encrypted
|
||||
// one) is a per-document verdict, not a transport fault.
|
||||
const detail = task?.errorMessage ?? null;
|
||||
throw new PaperlessError(
|
||||
detail !== null && /duplicate/i.test(detail) ? 'conflict' : 'provider_error',
|
||||
'Paperless could not process the uploaded file',
|
||||
undefined,
|
||||
detail === null ? undefined : shorten(detail),
|
||||
);
|
||||
}
|
||||
if (status === 'success' && task !== null) {
|
||||
const documentId = task.documentIds[0];
|
||||
if (documentId === undefined) {
|
||||
throw new PaperlessError(
|
||||
'provider_error',
|
||||
'Paperless finished the upload without naming a document',
|
||||
);
|
||||
}
|
||||
return { documentId };
|
||||
}
|
||||
if (Date.now() + interval >= deadline) {
|
||||
throw new PaperlessError(
|
||||
'timeout',
|
||||
'Paperless is still processing the uploaded file',
|
||||
undefined,
|
||||
`task ${taskId} was still ${status === '' ? 'unknown' : status}`,
|
||||
);
|
||||
}
|
||||
await sleep(interval);
|
||||
}
|
||||
}
|
||||
|
||||
/** The custom field with this exact name, or null. */
|
||||
async findCustomField(creds: PaperlessCreds, name: string): Promise<PaperlessCustomField | null> {
|
||||
const data = await this.requestJson<unknown>(creds, '/custom_fields/', {
|
||||
query: { name__iexact: name, page_size: String(TAG_PAGE_SIZE) },
|
||||
});
|
||||
const { rows } = pageRows(data);
|
||||
for (const row of rows) {
|
||||
const field = parseCustomField(row);
|
||||
// The filter is server-side, but an unknown query parameter is IGNORED by
|
||||
// Paperless rather than rejected, so a build without it would hand back
|
||||
// every field there is. Comparing the name here makes that harmless.
|
||||
if (field !== null && field.name.toLowerCase() === name.toLowerCase()) return field;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async createCustomField(creds: PaperlessCreds, name: string): Promise<PaperlessCustomField> {
|
||||
const field = parseCustomField(
|
||||
await this.requestJson<unknown>(creds, '/custom_fields/', {
|
||||
method: 'POST',
|
||||
json: { name, data_type: 'string' },
|
||||
}),
|
||||
);
|
||||
if (field === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless answered an unknown custom field shape');
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe Paperless to its own changes for one tag.
|
||||
*
|
||||
* Two triggers rather than one: "document added" misses every later edit, and
|
||||
* "document updated" misses the arrival. Both are filtered on the tag, so the
|
||||
* rest of a household's papers never cause a callback.
|
||||
*
|
||||
* The payload is deliberately thin. Placeholders such as `{doc_id}` are NOT
|
||||
* substituted in a webhook body at 3.1.3 (measured: they arrive literally),
|
||||
* and `as_json` wraps a body STRING into a JSON string rather than sending an
|
||||
* object, so the parameters, which do arrive as a real JSON object, carry the
|
||||
* link, and the callback means nothing more than "look now".
|
||||
*/
|
||||
async createWorkflow(creds: PaperlessCreds, spec: PaperlessWorkflowSpec): Promise<number> {
|
||||
if (spec.callbackUrl.length > WEBHOOK_URL_MAX_LENGTH) {
|
||||
throw new PaperlessError(
|
||||
'provider_error',
|
||||
'The callback URL is longer than Paperless accepts',
|
||||
undefined,
|
||||
`${spec.callbackUrl.length} characters, limit ${WEBHOOK_URL_MAX_LENGTH}`,
|
||||
);
|
||||
}
|
||||
const trigger = (type: number) => ({
|
||||
type,
|
||||
sources: TRIGGER_SOURCES,
|
||||
filter_has_tags: [spec.tagId],
|
||||
matching_algorithm: 0,
|
||||
match: '',
|
||||
is_insensitive: true,
|
||||
});
|
||||
const data = await this.requestJson<unknown>(creds, '/workflows/', {
|
||||
method: 'POST',
|
||||
json: {
|
||||
name: spec.name,
|
||||
order: 1,
|
||||
enabled: true,
|
||||
triggers: [trigger(TRIGGER_DOCUMENT_ADDED), trigger(TRIGGER_DOCUMENT_UPDATED)],
|
||||
actions: [
|
||||
{
|
||||
type: ACTION_WEBHOOK,
|
||||
webhook: {
|
||||
url: spec.callbackUrl,
|
||||
use_params: true,
|
||||
as_json: true,
|
||||
params: spec.params,
|
||||
headers: { [spec.secretHeader.name]: spec.secretHeader.value },
|
||||
include_document: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const id = isRecord(data) ? asNumber(data.id) : null;
|
||||
if (id === null) {
|
||||
throw new PaperlessError('provider_error', 'Paperless created the workflow without naming it');
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
/** Removing a subscription that is already gone is success, not a failure. */
|
||||
async deleteWorkflow(creds: PaperlessCreds, workflowId: number): Promise<void> {
|
||||
try {
|
||||
discardBody(await this.request(creds, `/workflows/${workflowId}/`, { method: 'DELETE' }));
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof PaperlessError && err.code === 'not_found') return;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,723 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Readable } from 'node:stream';
|
||||
import {
|
||||
docFail,
|
||||
docOk,
|
||||
type DocResult,
|
||||
type DocumentConnectionRef,
|
||||
type DocumentProvider,
|
||||
type DocumentProviderCapabilities,
|
||||
type DocumentScopeOption,
|
||||
type DocumentScopeRef,
|
||||
type FetchResult,
|
||||
type PushRequest,
|
||||
type PushResult,
|
||||
type RemoteDocument,
|
||||
} from '../document-provider';
|
||||
import {
|
||||
PaperlessClient,
|
||||
PaperlessError,
|
||||
currentVersion,
|
||||
sha256OrNull,
|
||||
type PaperlessCreds,
|
||||
type PaperlessDocument,
|
||||
type PaperlessTag,
|
||||
} from './paperless.client';
|
||||
import { DOWNLOAD_MAX_BYTES, guardDownload } from './provider-http';
|
||||
|
||||
/**
|
||||
* Paperless-ngx as a document scope.
|
||||
*
|
||||
* The scope is a TAG, because that is the only container Paperless has: there
|
||||
* are no folders, and a document belongs to as many tags as someone gives it.
|
||||
* A tag is also the one thing the instance's owner can see and reason about in
|
||||
* their own interface, which matters: this is their archive, and TREK is a
|
||||
* guest in it.
|
||||
*
|
||||
* The tag is paired with a custom field (`trek_trip_id`) carrying the trip's
|
||||
* uid. A tag can be removed by a human in two clicks and the binding would be
|
||||
* silently empty; the custom field survives that and says which trip a document
|
||||
* came from even after someone re-tags it. Nothing here treats the field as
|
||||
* authority: it is a breadcrumb for the person cleaning up, and for a support
|
||||
* question that starts with "where did this come from".
|
||||
*
|
||||
* Deletion propagates only as far as Paperless's own trash. `DELETE` on a
|
||||
* document is a soft delete there (`deleted_at` is set, the document leaves
|
||||
* every listing and `/api/documents/{id}/` answers 404), and the retention is
|
||||
* the instance's `trash_delay`. There is no hard delete on the API at all.
|
||||
*/
|
||||
|
||||
/** `tag:<id>`, the only scope key shape this adapter reads or writes. */
|
||||
const SCOPE_PREFIX = 'tag:';
|
||||
|
||||
/** The custom field the trip uid is written into, created on demand. */
|
||||
export const TRIP_UID_FIELD_NAME = 'trek_trip_id';
|
||||
|
||||
/** The header the self-registered workflow sends its shared secret in. */
|
||||
export const PAPERLESS_WEBHOOK_SECRET_HEADER = 'X-Trek-Docsync-Secret';
|
||||
|
||||
/**
|
||||
* What one push may buffer when the core did not say how large the file is.
|
||||
*
|
||||
* The body has to become a Buffer (Paperless wants a sized multipart request),
|
||||
* so an unbounded read would be a memory bomb driven by a remote file. The
|
||||
* declared size wins where there is one; this is the fallback ceiling.
|
||||
*/
|
||||
const PUSH_FALLBACK_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* What Paperless accepts without Tika. Measured against 3.1.3: an unsupported
|
||||
* type is refused with `400 … not supported` before a task is created.
|
||||
*
|
||||
* Note that Paperless decides on the SNIFFED type of the bytes, not on the
|
||||
* Content-Type of the multipart part: a .csv arrives as `text/plain` and is
|
||||
* stored as such. The list is therefore a pre-filter that saves a round trip,
|
||||
* never the last word; the 400 is still mapped to the same code.
|
||||
*/
|
||||
const BASE_MIME_TYPES = [
|
||||
'application/pdf',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/tiff',
|
||||
'image/gif',
|
||||
'image/bmp',
|
||||
'image/webp',
|
||||
'image/heic',
|
||||
'image/heif',
|
||||
'text/plain',
|
||||
'text/csv',
|
||||
'message/rfc822',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Accepted only on an instance that runs Tika and Gotenberg. They stay in the
|
||||
* list: an instance WITH Tika is a supported setup, and refusing a .docx here
|
||||
* would drop a file the provider would have taken. Without Tika the upload
|
||||
* comes back as `unsupported_type` from Paperless itself, which is the same
|
||||
* verdict one round trip later.
|
||||
*/
|
||||
const TIKA_MIME_TYPES = [
|
||||
'application/msword',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/vnd.ms-excel',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'application/vnd.ms-powerpoint',
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||
'application/vnd.oasis.opendocument.text',
|
||||
'application/vnd.oasis.opendocument.spreadsheet',
|
||||
'application/vnd.oasis.opendocument.presentation',
|
||||
'application/rtf',
|
||||
] as const;
|
||||
|
||||
const ACCEPTED_MIME_TYPES: readonly string[] = [...BASE_MIME_TYPES, ...TIKA_MIME_TYPES];
|
||||
|
||||
/**
|
||||
* The extension a Paperless document should carry in TREK.
|
||||
*
|
||||
* Paperless keeps a `title` without an extension and the uploaded file name
|
||||
* separately, while TREK's file list is a list of file names. The extension is
|
||||
* taken from the original name where there is one and derived from the MIME
|
||||
* type otherwise, so a document pulled twice keeps the same name.
|
||||
*/
|
||||
const EXTENSION_BY_MIME: Readonly<Record<string, string>> = {
|
||||
'application/pdf': '.pdf',
|
||||
'image/jpeg': '.jpg',
|
||||
'image/png': '.png',
|
||||
'image/tiff': '.tif',
|
||||
'image/gif': '.gif',
|
||||
'image/bmp': '.bmp',
|
||||
'image/webp': '.webp',
|
||||
'image/heic': '.heic',
|
||||
'image/heif': '.heif',
|
||||
'text/plain': '.txt',
|
||||
'text/csv': '.csv',
|
||||
'message/rfc822': '.eml',
|
||||
'application/msword': '.doc',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
|
||||
'application/vnd.ms-excel': '.xls',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
|
||||
'application/vnd.ms-powerpoint': '.ppt',
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
|
||||
'application/vnd.oasis.opendocument.text': '.odt',
|
||||
'application/vnd.oasis.opendocument.spreadsheet': '.ods',
|
||||
'application/vnd.oasis.opendocument.presentation': '.odp',
|
||||
'application/rtf': '.rtf',
|
||||
};
|
||||
|
||||
const KNOWN_EXTENSIONS = new Set(Object.values(EXTENSION_BY_MIME));
|
||||
|
||||
/**
|
||||
* The bare type, without the parameters a stored Content-Type may carry.
|
||||
* `text/plain; charset=utf-8` is text/plain, and comparing it verbatim against
|
||||
* the allow-list would refuse a file the provider takes happily.
|
||||
*/
|
||||
export function normalizeMime(mimeType: string): string {
|
||||
const [type = ''] = mimeType.split(';');
|
||||
return type.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function extensionOf(name: string): string | null {
|
||||
const match = /\.[A-Za-z0-9]{1,5}$/.exec(name);
|
||||
return match === null ? null : match[0].toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* The file name a document gets in TREK: the title, plus the extension it lost
|
||||
* on the way into Paperless.
|
||||
*/
|
||||
export function documentFileName(doc: PaperlessDocument): string {
|
||||
const base = doc.title.trim().length > 0
|
||||
? doc.title.trim()
|
||||
: (doc.originalFileName ?? `document-${doc.id}`);
|
||||
const fromOriginal = doc.originalFileName === null ? null : extensionOf(doc.originalFileName);
|
||||
const fromMime = doc.mimeType === null ? null : (EXTENSION_BY_MIME[doc.mimeType] ?? null);
|
||||
const extension = fromOriginal ?? fromMime;
|
||||
if (extension === null) return base;
|
||||
return base.toLowerCase().endsWith(extension) ? base : `${base}${extension}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The title for a file name. Only an extension this adapter would have ADDED is
|
||||
* removed, so a document genuinely called `Vertrag 2026.2` keeps its name.
|
||||
*/
|
||||
export function titleFromFileName(name: string): string {
|
||||
const trimmed = name.trim();
|
||||
const extension = extensionOf(trimmed);
|
||||
if (extension === null || !KNOWN_EXTENSIONS.has(extension)) return trimmed;
|
||||
const base = trimmed.slice(0, trimmed.length - extension.length).trim();
|
||||
return base.length > 0 ? base : trimmed;
|
||||
}
|
||||
|
||||
export function formatScopeKey(tagId: number): string {
|
||||
return `${SCOPE_PREFIX}${tagId}`;
|
||||
}
|
||||
|
||||
/** The tag id in a scope key, or null when it is not one of ours. */
|
||||
export function parseScopeKey(scopeKey: string): number | null {
|
||||
if (!scopeKey.startsWith(SCOPE_PREFIX)) return null;
|
||||
const raw = scopeKey.slice(SCOPE_PREFIX.length);
|
||||
if (!/^\d+$/.test(raw)) return null;
|
||||
const id = Number.parseInt(raw, 10);
|
||||
return Number.isSafeInteger(id) && id > 0 ? id : null;
|
||||
}
|
||||
|
||||
function scopeOption(tag: PaperlessTag): DocumentScopeOption {
|
||||
return {
|
||||
scopeKey: formatScopeKey(tag.id),
|
||||
label: tag.name,
|
||||
remoteRootId: String(tag.id),
|
||||
// The filter URL of the instance's own document list: paste it after the
|
||||
// base URL and you are looking at exactly what TREK syncs.
|
||||
remoteRootPath: `/documents?tags__id__all=${tag.id}`,
|
||||
};
|
||||
}
|
||||
|
||||
function toRemoteDocument(doc: PaperlessDocument): RemoteDocument {
|
||||
const version = currentVersion(doc);
|
||||
return {
|
||||
remoteId: String(doc.id),
|
||||
name: documentFileName(doc),
|
||||
// Paperless publishes no byte count anywhere in the document serializer;
|
||||
// the only way to learn it is to download, which is not worth a round trip
|
||||
// per document per run.
|
||||
size: null,
|
||||
mimeType: doc.mimeType,
|
||||
// `modified` moves on every edit, including the bulk edits that older
|
||||
// Paperless builds left untouched (3.1.3 does bump it, measured).
|
||||
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
|
||||
contentHash: sha256OrNull(version?.checksum ?? null),
|
||||
remoteModifiedAt: doc.modified,
|
||||
isDeleted: doc.deletedAt !== null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cursor shape. The count travels with the timestamp because Paperless has no
|
||||
* change feed: a document leaving the tag moves no timestamp at all, and only
|
||||
* the count notices it.
|
||||
*/
|
||||
function buildCursor(count: number | null, documents: PaperlessDocument[]): string {
|
||||
let newest = '';
|
||||
for (const doc of documents) {
|
||||
const modified = doc.modified ?? '';
|
||||
if (modified > newest) newest = modified;
|
||||
}
|
||||
return `v1:${count ?? documents.length}:${newest}`;
|
||||
}
|
||||
|
||||
/** Read a stream into memory, refusing to grow past the cap. */
|
||||
async function readStreamCapped(
|
||||
body: Readable,
|
||||
maxBytes: number,
|
||||
): Promise<{ bytes: Buffer; truncated: boolean }> {
|
||||
const chunks: Buffer[] = [];
|
||||
let received = 0;
|
||||
for await (const chunk of body as AsyncIterable<Buffer | Uint8Array | string>) {
|
||||
const buffer = typeof chunk === 'string' ? Buffer.from(chunk, 'utf8') : Buffer.from(chunk);
|
||||
received += buffer.length;
|
||||
if (received > maxBytes) {
|
||||
body.destroy();
|
||||
return { bytes: Buffer.concat(chunks), truncated: true };
|
||||
}
|
||||
chunks.push(buffer);
|
||||
}
|
||||
return { bytes: Buffer.concat(chunks), truncated: false };
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PaperlessDocumentProvider implements DocumentProvider {
|
||||
readonly id = 'paperless';
|
||||
|
||||
constructor(private readonly client: PaperlessClient) {}
|
||||
|
||||
capabilities(_conn: DocumentConnectionRef): DocumentProviderCapabilities {
|
||||
return {
|
||||
// Paperless can subscribe itself: a workflow with a webhook action is
|
||||
// creatable through the same token, provided its user holds
|
||||
// `add_workflow`. probe() reports the downgrade when it does not.
|
||||
push: 'webhook-self-registered',
|
||||
stableId: true,
|
||||
remoteTrash: true,
|
||||
// `update_version` replaces the bytes under the same document id, which
|
||||
// is what keeps the mapping and the document's history intact.
|
||||
replaceInPlace: true,
|
||||
// Every listed document carries the sha256 of its current revision in
|
||||
// `versions[]`, so no extra round trip is needed to compare content.
|
||||
contentHashInListing: true,
|
||||
// Paperless imposes no limit of its own; what a large upload runs into is
|
||||
// the reverse proxy in front of it, which the adapter cannot know.
|
||||
maxUploadBytes: null,
|
||||
acceptedMimeTypes: ACCEPTED_MIME_TYPES,
|
||||
canCreateScope: true,
|
||||
};
|
||||
}
|
||||
|
||||
async probe(
|
||||
conn: DocumentConnectionRef,
|
||||
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
try {
|
||||
const profile = await this.client.probe(creds);
|
||||
const capabilities = this.capabilities(conn);
|
||||
// A token whose user may not create workflows can still sync; it just
|
||||
// cannot subscribe itself, and the operator has to be told that here
|
||||
// rather than after the first silent hour of polling.
|
||||
const canSubscribe = profile.permissions.length === 0
|
||||
|| profile.permissions.includes('add_workflow');
|
||||
return docOk({
|
||||
account: profile.username.length > 0 ? profile.username : 'paperless',
|
||||
capabilities: canSubscribe ? capabilities : { ...capabilities, push: 'webhook-manual' },
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async listScopes(
|
||||
conn: DocumentConnectionRef,
|
||||
query?: string,
|
||||
): Promise<DocResult<DocumentScopeOption[]>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
try {
|
||||
const tags = await this.client.listTags(creds, query);
|
||||
return docOk(tags.map(scopeOption));
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async createScope(
|
||||
conn: DocumentConnectionRef,
|
||||
name: string,
|
||||
): Promise<DocResult<DocumentScopeOption>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
try {
|
||||
return docOk(scopeOption(await this.client.createTag(creds, name.trim())));
|
||||
} catch (err: unknown) {
|
||||
// A name that is already taken comes back as a 400 about the owner/name
|
||||
// unique constraint. It is not adopted silently: an existing tag may be
|
||||
// someone else's, and binding a trip to it would publish their documents
|
||||
// to everyone on that trip.
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async resolveScope(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<DocumentScopeOption>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
const tagId = parseScopeKey(scope.scopeKey);
|
||||
if (tagId === null) return badScope(scope.scopeKey);
|
||||
try {
|
||||
return docOk(scopeOption(await this.client.getTag(creds, tagId)));
|
||||
} catch (err: unknown) {
|
||||
// A tag that is gone is `scope_lost` for the link, not a generic 404:
|
||||
// someone deleted or renamed it upstream and a human has to choose again.
|
||||
if (err instanceof PaperlessError && err.code === 'not_found') {
|
||||
return docFail('scope_missing', `tag ${tagId} no longer exists`, err.status);
|
||||
}
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<{
|
||||
documents: RemoteDocument[];
|
||||
cursor: string | null;
|
||||
cursorUnchanged: boolean;
|
||||
truncated: boolean;
|
||||
}>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
const tagId = parseScopeKey(scope.scopeKey);
|
||||
if (tagId === null) return badScope(scope.scopeKey);
|
||||
try {
|
||||
const { documents, truncated, count } = await this.client.listDocuments(creds, tagId);
|
||||
const cursor = buildCursor(count, documents);
|
||||
return docOk({
|
||||
documents: documents.map(toRemoteDocument),
|
||||
cursor,
|
||||
// A truncated walk never claims the scope is unchanged: the pages it
|
||||
// did not read could hold anything.
|
||||
cursorUnchanged: !truncated && scope.cursor !== null && scope.cursor === cursor,
|
||||
truncated,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
// An empty tag and a deleted tag look alike in a document listing, so the
|
||||
// tag is only ever confirmed by resolveScope. A 404 here is the documents
|
||||
// endpoint itself, which is a different problem.
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async fetch(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<FetchResult>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
|
||||
const documentId = parseRemoteId(remoteId);
|
||||
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
|
||||
|
||||
try {
|
||||
// The metadata first: it carries the version marker the core stores
|
||||
// alongside the bytes, and it fails cheaply when the document is gone.
|
||||
const doc = await this.client.getDocument(creds, documentId);
|
||||
const response = await this.client.downloadOriginal(creds, documentId);
|
||||
const download = guardDownload(response, {
|
||||
maxBytes: DOWNLOAD_MAX_BYTES,
|
||||
tooLarge: (declared) =>
|
||||
new PaperlessError(
|
||||
'too_large',
|
||||
'The document is larger than TREK will transfer',
|
||||
undefined,
|
||||
`content_length=${declared}`,
|
||||
),
|
||||
noBody: () => new PaperlessError('provider_error', 'Paperless sent no body for the document'),
|
||||
});
|
||||
return docOk({
|
||||
body: download.body,
|
||||
size: download.size,
|
||||
mimeType: response.headers.get('content-type') ?? doc.mimeType,
|
||||
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async push(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
req: PushRequest,
|
||||
): Promise<DocResult<PushResult>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
const tagId = parseScopeKey(scope.scopeKey);
|
||||
if (tagId === null) return badScope(scope.scopeKey);
|
||||
|
||||
const mimeType = normalizeMime(req.mimeType);
|
||||
if (!ACCEPTED_MIME_TYPES.includes(mimeType)) {
|
||||
return docFail('unsupported_type', `Paperless does not accept ${req.mimeType}`);
|
||||
}
|
||||
|
||||
const cap = req.size > 0 ? req.size : PUSH_FALLBACK_MAX_BYTES;
|
||||
let bytes: Buffer;
|
||||
try {
|
||||
const read = await readStreamCapped(req.body, cap);
|
||||
if (read.truncated) {
|
||||
return docFail('too_large', `the file is larger than the declared ${cap} bytes`);
|
||||
}
|
||||
bytes = read.bytes;
|
||||
} catch (err: unknown) {
|
||||
return docFail(
|
||||
'provider_error',
|
||||
err instanceof Error ? err.message : 'could not read the local file',
|
||||
);
|
||||
}
|
||||
|
||||
const file = { fileName: req.fileName, mimeType, bytes };
|
||||
const replacing = req.remoteId !== undefined;
|
||||
try {
|
||||
const documentId = replacing
|
||||
? await this.replace(creds, file, req)
|
||||
: await this.create(creds, tagId, file, req);
|
||||
const doc = await this.client.getDocument(creds, documentId);
|
||||
|
||||
// Paperless stores the sha256 of what it received. Comparing it here
|
||||
// turns a truncated upload into a reported mismatch instead of a
|
||||
// document that silently differs from the one in the trip.
|
||||
const stored = sha256OrNull(currentVersion(doc)?.checksum ?? null);
|
||||
if (stored !== null && stored !== req.sha256.toLowerCase()) {
|
||||
// A document this call created and nobody has seen is litter, so it
|
||||
// goes to the trash where the owner can still get at it. A REPLACE is
|
||||
// left alone: that document existed before and its earlier revisions
|
||||
// are still attached to it.
|
||||
if (!replacing) await this.discard(creds, doc.id);
|
||||
return docFail(
|
||||
'checksum_mismatch',
|
||||
`Paperless stored ${stored} for document ${doc.id}`,
|
||||
);
|
||||
}
|
||||
|
||||
return docOk({
|
||||
remoteId: String(doc.id),
|
||||
remoteVersion: doc.modified ?? doc.added ?? String(doc.id),
|
||||
remoteModifiedAt: doc.modified,
|
||||
// Paperless does not deduplicate: the same bytes uploaded twice become
|
||||
// two documents with the same checksum and an empty
|
||||
// `duplicate_documents`. Nothing here may claim otherwise.
|
||||
deduplicated: false,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trash a document this push created but is not going to report.
|
||||
*
|
||||
* Best effort on purpose: the caller is already returning a failure, and a
|
||||
* second one on top of it would replace a precise diagnosis with the fallout
|
||||
* of the cleanup.
|
||||
*/
|
||||
private async discard(creds: PaperlessCreds, documentId: number): Promise<void> {
|
||||
try {
|
||||
await this.client.deleteDocument(creds, documentId);
|
||||
} catch {
|
||||
// Nothing to do: the document stays in the archive and the mismatch is
|
||||
// still what the caller hears about.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A new document, with both anchors set in the same request: the tag, which
|
||||
* is what the listing filters on, and the trip uid in the custom field, which
|
||||
* outlives someone re-tagging the document by hand.
|
||||
*
|
||||
* The field id is resolved per upload rather than cached. It is one small GET
|
||||
* against a local instance, and a cached id goes stale exactly when it hurts
|
||||
* most: after a human deleted the field, when every later upload would fail
|
||||
* with a validation error nobody can read.
|
||||
*/
|
||||
private async create(
|
||||
creds: PaperlessCreds,
|
||||
tagId: number,
|
||||
file: { fileName: string; mimeType: string; bytes: Buffer },
|
||||
req: PushRequest,
|
||||
): Promise<number> {
|
||||
const field = (await this.client.findCustomField(creds, TRIP_UID_FIELD_NAME))
|
||||
?? (await this.client.createCustomField(creds, TRIP_UID_FIELD_NAME));
|
||||
|
||||
const taskId = await this.client.postDocument(creds, file, {
|
||||
title: titleFromFileName(req.fileName),
|
||||
tagId,
|
||||
customFields: { [String(field.id)]: req.trekTripUid },
|
||||
});
|
||||
try {
|
||||
return (await this.client.awaitConsume(creds, taskId)).documentId;
|
||||
} catch (err: unknown) {
|
||||
// The consume outlived the poll budget. The document may well be there
|
||||
// (OCR on a long scan takes minutes), and retrying blind would leave two,
|
||||
// because Paperless accepts duplicate bytes. So the checksum decides.
|
||||
if (err instanceof PaperlessError && err.code === 'timeout') {
|
||||
const existing = await this.client.findByChecksum(creds, req.sha256);
|
||||
// Only a document carrying this scope's tag can be the one this push
|
||||
// made: the tag goes on at upload. Anything else with the same bytes
|
||||
// belongs to someone else's filing, and adopting it would put a
|
||||
// stranger's document in the trip, and delete it on the next
|
||||
// delete-through.
|
||||
const mine = existing.find((doc) => doc.tagIds.includes(tagId));
|
||||
if (mine !== undefined) return mine.id;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the bytes of a document TREK already knows.
|
||||
*
|
||||
* The document id is the one that was passed in, never the one the consume
|
||||
* task reports: `update_version` files the new revision under a fresh
|
||||
* internal id that answers 404 on its own, and adopting it would break the
|
||||
* mapping on the first replace.
|
||||
*/
|
||||
private async replace(
|
||||
creds: PaperlessCreds,
|
||||
file: { fileName: string; mimeType: string; bytes: Buffer },
|
||||
req: PushRequest,
|
||||
): Promise<number> {
|
||||
const documentId = parseRemoteId(req.remoteId ?? '');
|
||||
if (documentId === null) {
|
||||
throw new PaperlessError('not_found', `unreadable document id ${req.remoteId ?? ''}`);
|
||||
}
|
||||
const taskId = await this.client.updateVersion(creds, documentId, file);
|
||||
try {
|
||||
await this.client.awaitConsume(creds, taskId);
|
||||
} catch (err: unknown) {
|
||||
if (!(err instanceof PaperlessError) || err.code !== 'timeout') throw err;
|
||||
// Same reasoning as create(), one step shorter: the document keeps its
|
||||
// id, so whether the new revision landed is visible on the document.
|
||||
const doc = await this.client.getDocument(creds, documentId);
|
||||
if (sha256OrNull(currentVersion(doc)?.checksum ?? null) !== req.sha256.toLowerCase()) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return documentId;
|
||||
}
|
||||
|
||||
async rename(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
name: string,
|
||||
): Promise<DocResult<{ remoteVersion: string }>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
|
||||
const documentId = parseRemoteId(remoteId);
|
||||
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
|
||||
|
||||
try {
|
||||
// Only the title moves. The uploaded file name is immutable in Paperless,
|
||||
// and the archived file it serves is named after the title anyway.
|
||||
const doc = await this.client.patchDocument(creds, documentId, {
|
||||
title: titleFromFileName(name),
|
||||
});
|
||||
return docOk({ remoteVersion: doc.modified ?? doc.added ?? String(doc.id) });
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async trash(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<void>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
if (parseScopeKey(scope.scopeKey) === null) return badScope(scope.scopeKey);
|
||||
const documentId = parseRemoteId(remoteId);
|
||||
if (documentId === null) return docFail('not_found', `unreadable document id ${remoteId}`);
|
||||
|
||||
try {
|
||||
await this.client.deleteDocument(creds, documentId);
|
||||
return docOk(undefined);
|
||||
} catch (err: unknown) {
|
||||
// Already in the trash is the state that was asked for.
|
||||
if (err instanceof PaperlessError && err.code === 'not_found') return docOk(undefined);
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async registerWebhook(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
callbackUrl: string,
|
||||
secret: string,
|
||||
): Promise<DocResult<{ subscriptionId: string }>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
const tagId = parseScopeKey(scope.scopeKey);
|
||||
if (tagId === null) return badScope(scope.scopeKey);
|
||||
|
||||
try {
|
||||
const workflowId = await this.client.createWorkflow(creds, {
|
||||
name: `TREK document sync (link ${scope.linkId})`,
|
||||
tagId,
|
||||
callbackUrl,
|
||||
secretHeader: { name: PAPERLESS_WEBHOOK_SECRET_HEADER, value: secret },
|
||||
// Whatever Paperless sends is untrusted and only means "look now", so
|
||||
// the payload carries the link and nothing about the document. It could
|
||||
// not carry more: placeholders are not substituted in a webhook payload.
|
||||
params: { linkId: String(scope.linkId), tripId: String(scope.tripId) },
|
||||
});
|
||||
return docOk({ subscriptionId: String(workflowId) });
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
|
||||
async unregisterWebhook(
|
||||
conn: DocumentConnectionRef,
|
||||
subscriptionId: string,
|
||||
): Promise<DocResult<void>> {
|
||||
const creds = credsOf(conn);
|
||||
if (creds === null) return missingToken();
|
||||
const workflowId = parseRemoteId(subscriptionId);
|
||||
if (workflowId === null) {
|
||||
return docFail('not_found', `unreadable subscription id ${subscriptionId}`);
|
||||
}
|
||||
try {
|
||||
await this.client.deleteWorkflow(creds, workflowId);
|
||||
return docOk(undefined);
|
||||
} catch (err: unknown) {
|
||||
return failure(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function credsOf(conn: DocumentConnectionRef): PaperlessCreds | null {
|
||||
const token = conn.secrets.api_token ?? '';
|
||||
if (token.trim().length === 0) return null;
|
||||
return {
|
||||
baseUrl: conn.baseUrl,
|
||||
token: token.trim(),
|
||||
allowInsecureTls: conn.allowInsecureTls,
|
||||
};
|
||||
}
|
||||
|
||||
function parseRemoteId(remoteId: string): number | null {
|
||||
if (!/^\d+$/.test(remoteId.trim())) return null;
|
||||
const id = Number.parseInt(remoteId.trim(), 10);
|
||||
return Number.isSafeInteger(id) && id > 0 ? id : null;
|
||||
}
|
||||
|
||||
function missingToken<T>(): DocResult<T> {
|
||||
return docFail<T>('unauthorized', 'this connection has no Paperless API token');
|
||||
}
|
||||
|
||||
function badScope<T>(scopeKey: string): DocResult<T> {
|
||||
return docFail<T>('scope_missing', `not a Paperless scope key: ${scopeKey}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every failure leaves through here, so a stray upstream exception cannot
|
||||
* escape as an `unknown` with an English sentence in it.
|
||||
*/
|
||||
function failure<T>(err: unknown): DocResult<T> {
|
||||
if (err instanceof PaperlessError) {
|
||||
return docFail<T>(err.code, err.detail ?? err.message, err.status);
|
||||
}
|
||||
return docFail<T>('unknown', err instanceof Error ? err.message : undefined);
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
import { readCappedJson } from '../../../utils/cappedFetch';
|
||||
import type { DocsyncErrorCode } from '@trek/shared';
|
||||
import {
|
||||
guardDownload,
|
||||
providerFetch,
|
||||
statusErrorCode,
|
||||
type TransportFailure,
|
||||
type TransportFailureCode,
|
||||
} from './provider-http';
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
/**
|
||||
* Thin HTTP client for the Papra REST API (github.com/papra-hq/papra).
|
||||
* This is the ONLY place that talks to a Papra instance.
|
||||
*
|
||||
* Verified against a running 26.6.2 instance on 2026-09-18, because Papra's
|
||||
* prose documentation is silent on most of what a two-way sync depends on:
|
||||
*
|
||||
* - **Auth** is `Authorization: Bearer ppapi_…`. A key carries per-verb scopes
|
||||
* (`documents:read`, `tags:create`, …) and a missing scope answers **401
|
||||
* `auth.unauthorized`**, exactly like a wrong key: there is no 403 to tell
|
||||
* the two apart, so a failed call names the route it failed on instead.
|
||||
* - **No timestamp is usable for change detection.** `updatedAt` did not move
|
||||
* when a tag was added, when the document was renamed, or when it was
|
||||
* trashed; only `createdAt` and `deletedAt` are ever written. Hence
|
||||
* `snapshotVersion()` (a hash over a fixed field order) rather than an
|
||||
* ETag or a modified time.
|
||||
* - **Uploads deduplicate on the sha256 of the bytes**, not on the name. An
|
||||
* identical file answers 409 `document.already_exists`, and if the twin is
|
||||
* in the trash it is **restored** and returned with its ORIGINAL id under a
|
||||
* plain 200. Both are `deduplicated` to the sync core; see `uploadDocument`.
|
||||
* - **The search grammar needs quoting**: `tag:"Trek Reise 2026"` matches,
|
||||
* `tag:Trek Reise 2026` matches nothing. It is an exact, case-insensitive
|
||||
* match on the tag name (`tag:trek-trip-4` does not find `trek-trip-42`),
|
||||
* and it cannot search a sha256 at all.
|
||||
* - **`custom-properties`, `webhooks` and the restore and hard-delete routes
|
||||
* are session-only.** With a valid API key they answer 401; with a browser
|
||||
* session the same request succeeds. So there is no anchor property to write
|
||||
* and no subscription TREK can register for itself.
|
||||
* - `pageSize` caps at 100 (101 is a 400), `pageIndex` is 0-based, and
|
||||
* `documentsCount` is the total rather than the length of the page. The
|
||||
* trash listing takes those two parameters and refuses every other one.
|
||||
*/
|
||||
|
||||
const TIMEOUT_MS = 20000;
|
||||
|
||||
/** Uploads and downloads move real files, so they get their own budget. */
|
||||
const TRANSFER_TIMEOUT_MS = 180000;
|
||||
|
||||
/** Papra's JSON answers are documents metadata; a page of 100 is tens of kilobytes. */
|
||||
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* A runaway guard on a download, not the sync core's file-size policy. That
|
||||
* one lives in the core, which knows what it is willing to store. This only has
|
||||
* to stop an instance that answers a document request with an endless body.
|
||||
*/
|
||||
const MAX_DOWNLOAD_BYTES = 512 * 1024 * 1024;
|
||||
|
||||
/** Papra refuses anything above this outright. */
|
||||
export const PAPRA_PAGE_SIZE = 100;
|
||||
|
||||
/**
|
||||
* How many pages one enumeration walks before reporting itself incomplete.
|
||||
*
|
||||
* Papra has no change feed, so a sync is a full walk of the tag every time. A
|
||||
* cap rather than a loop: a caller told the answer is partial can keep the
|
||||
* previous state instead of reading absence as deletion.
|
||||
*/
|
||||
export const PAPRA_MAX_PAGES = 50;
|
||||
|
||||
/** Papra requires a colour on every tag, so scope creation has to pick one. */
|
||||
export const PAPRA_DEFAULT_TAG_COLOR = '#4F46E5';
|
||||
|
||||
/** `org_` + 24 lowercase alphanumerics. Papra 400s on anything else. */
|
||||
const ORG_ID_PATTERN = /^org_[a-z0-9]{24}$/;
|
||||
|
||||
export interface PapraCreds {
|
||||
/** Instance origin. The client appends `/api` itself. */
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
organizationId: string;
|
||||
allowInsecureTls: boolean;
|
||||
}
|
||||
|
||||
export interface PapraTag {
|
||||
id: string;
|
||||
name: string;
|
||||
color: string | null;
|
||||
description: string | null;
|
||||
documentsCount: number | null;
|
||||
}
|
||||
|
||||
export interface PapraDocument {
|
||||
id: string;
|
||||
name: string;
|
||||
originalName: string | null;
|
||||
originalSize: number | null;
|
||||
/** Real sha256 of the stored bytes, and the key Papra deduplicates on. */
|
||||
originalSha256Hash: string | null;
|
||||
mimeType: string | null;
|
||||
createdAt: string | null;
|
||||
updatedAt: string | null;
|
||||
deletedAt: string | null;
|
||||
isDeleted: boolean;
|
||||
/**
|
||||
* Absent from the upload and rename answers, present in every listing. Null
|
||||
* therefore means "not stated here", never "no tags".
|
||||
*/
|
||||
tags: PapraTag[] | null;
|
||||
}
|
||||
|
||||
export interface PapraPage {
|
||||
documents: PapraDocument[];
|
||||
/** Total across all pages, not the length of this one. */
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface PapraUploadResult {
|
||||
document: PapraDocument;
|
||||
/**
|
||||
* True when Papra answered with a document it already held: a trashed twin it
|
||||
* restored under its original id, or, on the 409 path, the live twin this
|
||||
* client then looked up.
|
||||
*/
|
||||
deduplicated: boolean;
|
||||
}
|
||||
|
||||
export interface PapraDownload {
|
||||
body: Readable;
|
||||
/** From `content-length`, which Papra does send on the file route. */
|
||||
size: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed Papra call, carrying the reason as a sync-core error code.
|
||||
*
|
||||
* The upstream message is English and occasionally an HTML error page from a
|
||||
* reverse proxy, so it travels in `detail` for the self-hoster's log while the
|
||||
* code is what the user sees through i18n. `papraCode` is kept separately
|
||||
* because several branches turn on it: `document.already_exists` is a dedup,
|
||||
* not a failure, and `tags.not_found` is a lost scope rather than a bad id.
|
||||
*/
|
||||
export class PapraError extends Error {
|
||||
readonly code: DocsyncErrorCode;
|
||||
readonly status?: number;
|
||||
readonly detail?: string;
|
||||
readonly papraCode?: string;
|
||||
|
||||
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string, papraCode?: string) {
|
||||
super(message);
|
||||
this.name = 'PapraError';
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
this.papraCode = papraCode;
|
||||
}
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function str(value: unknown): string | null {
|
||||
return typeof value === 'string' ? value : null;
|
||||
}
|
||||
|
||||
function num(value: unknown): number | null {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function parseTag(value: unknown): PapraTag | null {
|
||||
if (!isRecord(value)) return null;
|
||||
const id = str(value.id);
|
||||
const name = str(value.name);
|
||||
if (!id || name === null) return null;
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
color: str(value.color),
|
||||
description: str(value.description),
|
||||
documentsCount: num(value.documentsCount),
|
||||
};
|
||||
}
|
||||
|
||||
function parseDocument(value: unknown): PapraDocument | null {
|
||||
if (!isRecord(value)) return null;
|
||||
const id = str(value.id);
|
||||
if (!id) return null;
|
||||
const rawTags = value.tags;
|
||||
let tags: PapraTag[] | null = null;
|
||||
if (Array.isArray(rawTags)) {
|
||||
tags = [];
|
||||
for (const entry of rawTags) {
|
||||
const tag = parseTag(entry);
|
||||
if (tag) tags.push(tag);
|
||||
}
|
||||
}
|
||||
return {
|
||||
id,
|
||||
// Papra has always sent a name; falling back to the original one keeps a
|
||||
// stripped-down answer usable instead of dropping the whole document.
|
||||
name: str(value.name) ?? str(value.originalName) ?? id,
|
||||
originalName: str(value.originalName),
|
||||
originalSize: num(value.originalSize),
|
||||
originalSha256Hash: str(value.originalSha256Hash),
|
||||
mimeType: str(value.mimeType),
|
||||
createdAt: str(value.createdAt),
|
||||
updatedAt: str(value.updatedAt),
|
||||
deletedAt: str(value.deletedAt),
|
||||
isDeleted: value.isDeleted === true,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
|
||||
/** The `{ error: { message, code } }` envelope Papra answers every 4xx with. */
|
||||
function parseErrorBody(body: unknown): { message: string | null; code: string | null } {
|
||||
if (!isRecord(body) || !isRecord(body.error)) return { message: null, code: null };
|
||||
return { message: str(body.error.message), code: str(body.error.code) };
|
||||
}
|
||||
|
||||
/**
|
||||
* `https://host/papra/api/` → `https://host/papra/api`, tolerating what people
|
||||
* paste. Without it a pasted `…/api` becomes `…/api/api` and every call 404s.
|
||||
*/
|
||||
export function papraApiBase(baseUrl: string): string {
|
||||
const origin = baseUrl
|
||||
.trim()
|
||||
.replace(/(?<!\/)\/+$/, '')
|
||||
.replace(/\/api$/i, '');
|
||||
return `${origin}/api`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable snapshot hash over a FIXED field order, standing in for the version
|
||||
* marker Papra does not have. `updatedAt` is in it although it never moves: it
|
||||
* costs nothing and it is the field that would start carrying the signal if a
|
||||
* later Papra ever began writing it.
|
||||
*/
|
||||
export function snapshotVersion(doc: PapraDocument): string {
|
||||
const snapshot = [doc.id, doc.name, doc.originalSha256Hash ?? '', doc.updatedAt ?? ''];
|
||||
return crypto.createHash('sha256').update(JSON.stringify(snapshot)).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* A tag name as the search grammar wants it: quoted, because an unquoted name
|
||||
* with a space matches nothing at all.
|
||||
*
|
||||
* Returns null for a name the grammar cannot express (Papra accepts `"` in a
|
||||
* tag name and offers no escape for it), so the caller can fall back to
|
||||
* enumerating the organisation and filtering by tag id.
|
||||
*/
|
||||
export function tagSearchQuery(tagName: string): string | null {
|
||||
if (tagName.includes('"') || tagName.includes('\\')) return null;
|
||||
return `tag:"${tagName}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Papra's status codes mapped onto the sync core's vocabulary.
|
||||
*
|
||||
* 400 on a path parameter is a `not_found` rather than a provider fault: the
|
||||
* ids are format-checked upstream, so a remote id Papra refuses to parse is one
|
||||
* that cannot exist. The organisation id is validated here before the call, so
|
||||
* it can never be the parameter Papra is complaining about.
|
||||
*/
|
||||
function classify(status: number, papraCode: string | null): DocsyncErrorCode {
|
||||
if (papraCode === 'tags.not_found') return 'scope_missing';
|
||||
if (status === 400) return papraCode === 'server.invalid_request.params' ? 'not_found' : 'provider_error';
|
||||
return statusErrorCode(status);
|
||||
}
|
||||
|
||||
const TRANSPORT_MESSAGES: Record<TransportFailureCode, string> = {
|
||||
ssrf_blocked: 'The Papra URL is not allowed',
|
||||
timeout: 'Papra did not answer in time',
|
||||
tls_untrusted: 'Papra presented a certificate that is not trusted',
|
||||
unreachable: 'Could not reach Papra',
|
||||
};
|
||||
|
||||
/** Why a request never produced a response at all. */
|
||||
function transportError(failure: TransportFailure): PapraError {
|
||||
return new PapraError(failure.code, TRANSPORT_MESSAGES[failure.code], undefined, failure.detail);
|
||||
}
|
||||
|
||||
/** RFC 7578 as undici's own FormData writes it: UTF-8 bytes, three characters escaped. */
|
||||
function escapeFormValue(value: string): string {
|
||||
return value.replace(/\r/g, '%0D').replace(/\n/g, '%0A').replace(/"/g, '%22');
|
||||
}
|
||||
|
||||
/**
|
||||
* A media type that is safe to put in a header.
|
||||
*
|
||||
* The value comes from `trip_files.mime_type`, which an upload can influence,
|
||||
* and it went into the part header untouched while the filename beside it was
|
||||
* escaped. A CR or LF in it ends the header early and lets whatever follows be
|
||||
* read as headers of its own. Anything that is not a plain media type is
|
||||
* replaced rather than repaired: a mangled type is not worth guessing at, and
|
||||
* the generic one is what an unknown file gets anyway.
|
||||
*/
|
||||
function safeMimeType(raw: string): string {
|
||||
const TOKEN = /^[A-Za-z0-9!#$&^_.+-]+\/[A-Za-z0-9!#$&^_.+-]+$/;
|
||||
return TOKEN.test(raw) ? raw : 'application/octet-stream';
|
||||
}
|
||||
|
||||
/** The bytes before the file in a one-field multipart body. Exported so the tests can read them. */
|
||||
export function multipartHeader(boundary: string, fileName: string, mimeType: string): Buffer {
|
||||
return Buffer.from(
|
||||
`--${boundary}\r\n` +
|
||||
`Content-Disposition: form-data; name="file"; filename="${escapeFormValue(fileName)}"\r\n` +
|
||||
`Content-Type: ${safeMimeType(mimeType)}\r\n\r\n`,
|
||||
'utf8',
|
||||
);
|
||||
}
|
||||
|
||||
export function multipartFooter(boundary: string): Buffer {
|
||||
return Buffer.from(`\r\n--${boundary}--\r\n`, 'utf8');
|
||||
}
|
||||
|
||||
export class PapraClient {
|
||||
/**
|
||||
* One authenticated JSON call.
|
||||
*
|
||||
* `expectedEmpty` covers the 204s (tagging, some deletes): those have no body
|
||||
* to parse and asking for one would turn a success into a parse failure.
|
||||
*/
|
||||
private async call(
|
||||
creds: PapraCreds,
|
||||
method: string,
|
||||
path: string,
|
||||
options: { query?: Record<string, string>; json?: unknown; expectEmpty?: boolean } = {},
|
||||
): Promise<unknown> {
|
||||
const url = new URL(papraApiBase(creds.baseUrl) + path);
|
||||
for (const [key, value] of Object.entries(options.query ?? {})) url.searchParams.set(key, value);
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
Authorization: `Bearer ${creds.apiKey}`,
|
||||
Accept: 'application/json',
|
||||
};
|
||||
if (options.json !== undefined) headers['Content-Type'] = 'application/json';
|
||||
|
||||
const response = await providerFetch(
|
||||
url.toString(),
|
||||
{ method, headers, body: options.json === undefined ? undefined : JSON.stringify(options.json) },
|
||||
{ timeoutMs: TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
|
||||
);
|
||||
|
||||
return this.readBody(response, `${method} ${path}`, options.expectEmpty === true);
|
||||
}
|
||||
|
||||
/** Shared tail of every JSON call: cap the body, parse it, turn a non-2xx into a PapraError. */
|
||||
private async readBody(response: Response, route: string, expectEmpty: boolean): Promise<unknown> {
|
||||
const body = response.status === 204 ? null : await readCappedJson<unknown>(response, MAX_JSON_BYTES);
|
||||
|
||||
if (!response.ok) {
|
||||
const { message, code } = parseErrorBody(body);
|
||||
throw new PapraError(
|
||||
classify(response.status, code),
|
||||
`Papra answered HTTP ${response.status} on ${route}`,
|
||||
response.status,
|
||||
message ?? undefined,
|
||||
code ?? undefined,
|
||||
);
|
||||
}
|
||||
if (expectEmpty) return null;
|
||||
if (body === undefined) {
|
||||
throw new PapraError(
|
||||
'provider_error',
|
||||
'Papra returned something that is not JSON',
|
||||
response.status,
|
||||
`on ${route}; check the URL points at Papra itself and not at a login page in front of it`,
|
||||
);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/**
|
||||
* The organisation the connection names, checked before it is put in a URL.
|
||||
*
|
||||
* Papra answers a malformed organisation id with the same 400 shape it uses
|
||||
* for a malformed document id, and those two mean very different things to a
|
||||
* caller. Settling it here keeps the wire-level mapping unambiguous.
|
||||
*/
|
||||
private orgPath(creds: PapraCreds): string {
|
||||
if (!ORG_ID_PATTERN.test(creds.organizationId)) {
|
||||
throw new PapraError(
|
||||
'provider_error',
|
||||
'The organisation id is not in Papra’s format',
|
||||
undefined,
|
||||
`expected org_ followed by 24 lowercase alphanumerics, got "${creds.organizationId}"`,
|
||||
);
|
||||
}
|
||||
return `/organizations/${creds.organizationId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The organisations the key can reach, for the connection's display name.
|
||||
*
|
||||
* Separate from the reachability test on purpose: this needs the
|
||||
* `organizations:read` scope, which a key limited to documents and tags does
|
||||
* not carry, and a connection that syncs perfectly well should not fail its
|
||||
* probe over a missing label.
|
||||
*/
|
||||
async listOrganizations(creds: PapraCreds): Promise<Array<{ id: string; name: string }>> {
|
||||
const body = await this.call(creds, 'GET', '/organizations');
|
||||
const raw = isRecord(body) && Array.isArray(body.organizations) ? body.organizations : [];
|
||||
const out: Array<{ id: string; name: string }> = [];
|
||||
for (const entry of raw) {
|
||||
if (!isRecord(entry)) continue;
|
||||
const id = str(entry.id);
|
||||
if (!id) continue;
|
||||
out.push({ id, name: str(entry.name) ?? id });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async listTags(creds: PapraCreds): Promise<PapraTag[]> {
|
||||
const body = await this.call(creds, 'GET', `${this.orgPath(creds)}/tags`);
|
||||
const raw = isRecord(body) && Array.isArray(body.tags) ? body.tags : [];
|
||||
const tags: PapraTag[] = [];
|
||||
for (const entry of raw) {
|
||||
const tag = parseTag(entry);
|
||||
if (tag) tags.push(tag);
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
async createTag(creds: PapraCreds, name: string, color: string): Promise<PapraTag> {
|
||||
const body = await this.call(creds, 'POST', `${this.orgPath(creds)}/tags`, { json: { name, color } });
|
||||
const tag = isRecord(body) ? parseTag(body.tag) : null;
|
||||
if (!tag) throw new PapraError('provider_error', 'Papra created a tag but did not describe it');
|
||||
return tag;
|
||||
}
|
||||
|
||||
/**
|
||||
* One page of documents.
|
||||
*
|
||||
* Sorted by `createdAt` ascending rather than descending: a document uploaded
|
||||
* while the walk is in flight then lands at the END of the last page, where
|
||||
* missing it costs one sync cycle. Descending puts it at the front and pushes
|
||||
* every later page down by one, which silently skips an existing document.
|
||||
*
|
||||
* The trash route takes `pageIndex` and `pageSize` and NOTHING else (it
|
||||
* rejects a sort or a search with 400 `Invalid key: Expected never` rather
|
||||
* than ignoring it), so the sort and the query are dropped there.
|
||||
*/
|
||||
async listDocuments(
|
||||
creds: PapraCreds,
|
||||
options: { searchQuery?: string; pageIndex: number; deleted?: boolean },
|
||||
): Promise<PapraPage> {
|
||||
const trash = options.deleted === true;
|
||||
const query: Record<string, string> = {
|
||||
pageIndex: String(options.pageIndex),
|
||||
pageSize: String(PAPRA_PAGE_SIZE),
|
||||
};
|
||||
if (!trash) {
|
||||
query.sortField = 'createdAt';
|
||||
query.sortOrder = 'asc';
|
||||
if (options.searchQuery) query.searchQuery = options.searchQuery;
|
||||
}
|
||||
|
||||
const path = `${this.orgPath(creds)}/documents${trash ? '/deleted' : ''}`;
|
||||
const body = await this.call(creds, 'GET', path, { query });
|
||||
const raw = isRecord(body) && Array.isArray(body.documents) ? body.documents : [];
|
||||
const documents: PapraDocument[] = [];
|
||||
for (const entry of raw) {
|
||||
const doc = parseDocument(entry);
|
||||
if (doc) documents.push(doc);
|
||||
}
|
||||
const total = isRecord(body) ? num(body.documentsCount) : null;
|
||||
return { documents, total: total ?? documents.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Every page, up to the cap.
|
||||
*
|
||||
* `truncated` is the honest half of the answer: the sync core reads absence
|
||||
* from a listing as a possible deletion, so a walk that stopped early has to
|
||||
* say so or a capped enumeration would look like a mass delete.
|
||||
*/
|
||||
async listAllDocuments(
|
||||
creds: PapraCreds,
|
||||
options: { searchQuery?: string; deleted?: boolean } = {},
|
||||
): Promise<{ documents: PapraDocument[]; truncated: boolean }> {
|
||||
const documents: PapraDocument[] = [];
|
||||
let truncated = false;
|
||||
|
||||
for (let pageIndex = 0; pageIndex < PAPRA_MAX_PAGES; pageIndex++) {
|
||||
const page = await this.listDocuments(creds, { ...options, pageIndex });
|
||||
documents.push(...page.documents);
|
||||
if (page.documents.length < PAPRA_PAGE_SIZE) break;
|
||||
if (documents.length >= page.total) break;
|
||||
if (pageIndex === PAPRA_MAX_PAGES - 1) truncated = true;
|
||||
}
|
||||
|
||||
return { documents, truncated };
|
||||
}
|
||||
|
||||
async getDocument(creds: PapraCreds, documentId: string): Promise<PapraDocument> {
|
||||
const body = await this.call(creds, 'GET', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`);
|
||||
const doc = isRecord(body) ? parseDocument(body.document) : null;
|
||||
if (!doc) throw new PapraError('provider_error', 'Papra returned a document in an unexpected shape');
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* The stored bytes.
|
||||
*
|
||||
* The response's own `content-type` is always `application/octet-stream`, so
|
||||
* the caller takes the MIME type from the document record instead of from the
|
||||
* transfer.
|
||||
*/
|
||||
async downloadDocument(creds: PapraCreds, documentId: string): Promise<PapraDownload> {
|
||||
const url = `${papraApiBase(creds.baseUrl) + this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}/file`;
|
||||
|
||||
const response = await providerFetch(
|
||||
url,
|
||||
{ method: 'GET', headers: { Authorization: `Bearer ${creds.apiKey}` } },
|
||||
{ timeoutMs: TRANSFER_TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await readCappedJson<unknown>(response, MAX_JSON_BYTES);
|
||||
const { message, code } = parseErrorBody(body);
|
||||
throw new PapraError(
|
||||
classify(response.status, code),
|
||||
`Papra answered HTTP ${response.status} on the document file`,
|
||||
response.status,
|
||||
message ?? undefined,
|
||||
code ?? undefined,
|
||||
);
|
||||
}
|
||||
return guardDownload(response, {
|
||||
maxBytes: MAX_DOWNLOAD_BYTES,
|
||||
tooLarge: () =>
|
||||
new PapraError('too_large', 'The Papra document is larger than TREK will transfer', response.status),
|
||||
noBody: () => new PapraError('provider_error', 'Papra answered the document file with no body', response.status),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Store bytes as a new document.
|
||||
*
|
||||
* Streamed rather than buffered: the sync core hands over whatever is in trip
|
||||
* storage, and holding a document in memory to build a multipart body is the
|
||||
* one thing a file sync must not do. The body is assembled by hand because
|
||||
* `FormData` would want the bytes first.
|
||||
*
|
||||
* Two answers are not what they look like. A 200 carrying a `createdAt` from
|
||||
* before this request is a trashed twin Papra restored under its original id,
|
||||
* and a 409 `document.already_exists` is a live twin. Neither is a new
|
||||
* document, and the caller resolves the second one through `findByHash`.
|
||||
*/
|
||||
async uploadDocument(
|
||||
creds: PapraCreds,
|
||||
file: { body: Readable; fileName: string; mimeType: string },
|
||||
): Promise<PapraUploadResult> {
|
||||
const boundary = `----trek${crypto.randomBytes(16).toString('hex')}`;
|
||||
const header = multipartHeader(boundary, file.fileName, file.mimeType);
|
||||
const footer = multipartFooter(boundary);
|
||||
|
||||
async function* frame(): AsyncGenerator<Buffer> {
|
||||
yield header;
|
||||
for await (const chunk of file.body) {
|
||||
yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
}
|
||||
yield footer;
|
||||
}
|
||||
|
||||
const sentAt = Date.now();
|
||||
const response = await providerFetch(
|
||||
`${papraApiBase(creds.baseUrl) + this.orgPath(creds)}/documents`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${creds.apiKey}`,
|
||||
Accept: 'application/json',
|
||||
'Content-Type': `multipart/form-data; boundary=${boundary}`,
|
||||
},
|
||||
body: Readable.toWeb(Readable.from(frame())),
|
||||
// Required by undici for a streamed request body; the platform types
|
||||
// carry the field, the DOM lib this project does not load does not.
|
||||
duplex: 'half',
|
||||
},
|
||||
{ timeoutMs: TRANSFER_TIMEOUT_MS, allowInsecureTls: creds.allowInsecureTls, onTransportFailure: transportError },
|
||||
);
|
||||
const elapsedMs = Date.now() - sentAt;
|
||||
|
||||
const body = await this.readBody(response, 'POST /documents', false);
|
||||
const document = isRecord(body) ? parseDocument(body.document) : null;
|
||||
if (!document)
|
||||
throw new PapraError('provider_error', 'Papra accepted the upload but did not describe the document');
|
||||
|
||||
return {
|
||||
document,
|
||||
deduplicated: isRestoredTwin(document, response.headers.get('date'), elapsedMs),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The document holding these bytes, for the 409 path.
|
||||
*
|
||||
* Three widening passes because Papra cannot search a hash: the name the
|
||||
* upload used, then every live document, then the trash. The name pass is
|
||||
* almost always enough and costs one request; the others exist because a twin
|
||||
* can have been stored under a different name entirely, which is exactly the
|
||||
* case where guessing would pair the wrong document.
|
||||
*/
|
||||
async findByHash(creds: PapraCreds, sha256: string, nameHint?: string): Promise<PapraDocument | null> {
|
||||
const wanted = sha256.toLowerCase();
|
||||
const match = (docs: PapraDocument[]): PapraDocument | null =>
|
||||
docs.find((doc) => doc.originalSha256Hash?.toLowerCase() === wanted) ?? null;
|
||||
|
||||
if (nameHint) {
|
||||
const byName = await this.listAllDocuments(creds, { searchQuery: nameHint });
|
||||
const hit = match(byName.documents);
|
||||
if (hit) return hit;
|
||||
}
|
||||
const live = await this.listAllDocuments(creds);
|
||||
const hit = match(live.documents);
|
||||
if (hit) return hit;
|
||||
|
||||
const trashed = await this.listAllDocuments(creds, { deleted: true });
|
||||
return match(trashed.documents);
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a tag on a document.
|
||||
*
|
||||
* An already-present tag answers 409 `documents.already_has_tag`, which is the
|
||||
* state the caller wanted, so it is swallowed here rather than left for every
|
||||
* call site to recognise.
|
||||
*/
|
||||
async addTag(creds: PapraCreds, documentId: string, tagId: string): Promise<void> {
|
||||
try {
|
||||
await this.call(creds, 'POST', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}/tags`, {
|
||||
json: { tagId },
|
||||
expectEmpty: true,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof PapraError && err.papraCode === 'documents.already_has_tag') return;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async renameDocument(creds: PapraCreds, documentId: string, name: string): Promise<PapraDocument> {
|
||||
const body = await this.call(creds, 'PATCH', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`, {
|
||||
json: { name },
|
||||
});
|
||||
const doc = isRecord(body) ? parseDocument(body.document) : null;
|
||||
if (!doc) throw new PapraError('provider_error', 'Papra renamed the document but did not describe it');
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a document to Papra's trash.
|
||||
*
|
||||
* This is the only destructive call the API offers a key: permanent deletion
|
||||
* and restoring are session-only, and `?permanent=true` is accepted and then
|
||||
* ignored, which is worse than being refused. Papra keeps trashed documents
|
||||
* for its configured retention window, so a wrong propagation stays
|
||||
* recoverable through its web interface.
|
||||
*/
|
||||
async trashDocument(creds: PapraCreds, documentId: string): Promise<void> {
|
||||
await this.call(creds, 'DELETE', `${this.orgPath(creds)}/documents/${encodeURIComponent(documentId)}`, {
|
||||
expectEmpty: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a 200 from the upload route describes a document Papra already held:
|
||||
* a trashed twin it silently restored under its original id.
|
||||
*
|
||||
* Papra says nothing about which of the two happened, so this has to be read
|
||||
* off `createdAt`: a restored document was created before this request started.
|
||||
* That start is not a time this process knows in Papra's clock, so it is
|
||||
* reconstructed from the answer as `Date` minus how long the call took.
|
||||
* Comparing a duration instead of two clocks is what makes the test survive a
|
||||
* self-hosted instance whose clock is minutes out.
|
||||
*
|
||||
* `Date` carries whole seconds, so the reconstruction always lands at or BEFORE
|
||||
* the true start. That asymmetry is chosen rather than tolerated: a document
|
||||
* this request really did create can never be mistaken for a restore, and the
|
||||
* price is blindness to a twin trashed and re-pushed within roughly a second of
|
||||
* being created. Nothing else in the answer closes that second: the name is
|
||||
* not a tiebreaker, because a restore overwrites the stored name with the
|
||||
* filename of the upload that triggered it.
|
||||
*
|
||||
* Without a `Date` header at all there is nothing to anchor to and the local
|
||||
* clock has to stand in, which is why that branch needs a tolerance.
|
||||
*/
|
||||
function isRestoredTwin(doc: PapraDocument, dateHeader: string | null, elapsedMs: number): boolean {
|
||||
if (!doc.createdAt) return false;
|
||||
const created = Date.parse(doc.createdAt);
|
||||
if (!Number.isFinite(created)) return false;
|
||||
|
||||
const answeredAt = dateHeader ? Date.parse(dateHeader) : Number.NaN;
|
||||
if (Number.isFinite(answeredAt)) return created < answeredAt - elapsedMs;
|
||||
return created < Date.now() - elapsedMs - UNANCHORED_CLOCK_TOLERANCE_MS;
|
||||
}
|
||||
|
||||
/** How far a headerless instance's clock may sit from this one before dedup detection misfires. */
|
||||
const UNANCHORED_CLOCK_TOLERANCE_MS = 30000;
|
||||
@@ -0,0 +1,395 @@
|
||||
import {
|
||||
docFail,
|
||||
docOk,
|
||||
type DocResult,
|
||||
type DocumentConnectionRef,
|
||||
type DocumentProvider,
|
||||
type DocumentProviderCapabilities,
|
||||
type DocumentScopeOption,
|
||||
type DocumentScopeRef,
|
||||
type FetchResult,
|
||||
type PushRequest,
|
||||
type PushResult,
|
||||
type RemoteDocument,
|
||||
} from '../document-provider';
|
||||
import {
|
||||
PapraClient,
|
||||
PapraError,
|
||||
PAPRA_DEFAULT_TAG_COLOR,
|
||||
snapshotVersion,
|
||||
tagSearchQuery,
|
||||
type PapraCreds,
|
||||
type PapraDocument,
|
||||
type PapraTag,
|
||||
} from './papra.client';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
/**
|
||||
* Papra as a document scope: one organisation, one tag per trip.
|
||||
*
|
||||
* Papra has no folders, so the container a trip binds to is a tag, which is
|
||||
* why the scope key carries both halves (`org:…/tag:…`). The organisation is in
|
||||
* there although it is also on the connection: an API key reaches every
|
||||
* organisation its owner belongs to, so a binding that only named the tag would
|
||||
* silently follow the connection if its organisation were ever re-pointed.
|
||||
*
|
||||
* Two capabilities are `false`/`manual` because of what Papra withholds from an
|
||||
* API key rather than what it cannot do. Webhook CRUD and the restore and
|
||||
* hard-delete routes answer 401 to a key and 200 to a browser session, so TREK
|
||||
* cannot subscribe itself, and there is no endpoint that replaces the bytes of
|
||||
* an existing document at all.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PapraDocumentProvider implements DocumentProvider {
|
||||
readonly id = 'papra';
|
||||
|
||||
private readonly client = new PapraClient();
|
||||
|
||||
capabilities(_conn: DocumentConnectionRef): DocumentProviderCapabilities {
|
||||
return {
|
||||
// The API exists, but only a browser session may call it.
|
||||
push: 'webhook-manual',
|
||||
stableId: true,
|
||||
remoteTrash: true,
|
||||
// No endpoint replaces a document's bytes. A new revision is a new
|
||||
// document, and an identical one is deduplicated into the old.
|
||||
replaceInPlace: false,
|
||||
// `originalSha256Hash` is a real sha256 of the stored bytes.
|
||||
contentHashInListing: true,
|
||||
// The ceiling is DOCUMENT_STORAGE_MAX_UPLOAD_SIZE on the instance and is
|
||||
// not exposed anywhere in the API, so claiming Papra's 10 MB default here
|
||||
// would refuse uploads a tuned instance accepts. An oversized one comes
|
||||
// back as 413 `too_large` instead.
|
||||
maxUploadBytes: null,
|
||||
acceptedMimeTypes: null,
|
||||
canCreateScope: true,
|
||||
};
|
||||
}
|
||||
|
||||
async probe(
|
||||
conn: DocumentConnectionRef,
|
||||
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
// Tags rather than `/organizations`: it is the narrowest call that proves
|
||||
// all three things at once: the instance answers, the key is valid, and
|
||||
// the key's owner is in this organisation.
|
||||
await this.client.listTags(creds);
|
||||
return { account: await this.accountLabel(creds), capabilities: this.capabilities(conn) };
|
||||
});
|
||||
}
|
||||
|
||||
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
const tags = await this.client.listTags(creds);
|
||||
const needle = query?.trim().toLowerCase();
|
||||
return tags
|
||||
.filter((tag) => !needle || tag.name.toLowerCase().includes(needle))
|
||||
.map((tag) => toScopeOption(creds.organizationId, tag));
|
||||
});
|
||||
}
|
||||
|
||||
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
// A name collision is reported rather than resolved to the existing tag:
|
||||
// that tag may be someone else's filing, and the picker already lists it
|
||||
// for an admin who meant to bind to it.
|
||||
const tag = await this.client.createTag(creds, name, PAPRA_DEFAULT_TAG_COLOR);
|
||||
return toScopeOption(creds.organizationId, tag);
|
||||
});
|
||||
}
|
||||
|
||||
async resolveScope(conn: DocumentConnectionRef, scope: DocumentScopeRef): Promise<DocResult<DocumentScopeOption>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
const tag = await this.requireTag(creds, scope);
|
||||
return toScopeOption(creds.organizationId, tag);
|
||||
});
|
||||
}
|
||||
|
||||
async list(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<
|
||||
DocResult<{ documents: RemoteDocument[]; cursor: string | null; cursorUnchanged: boolean; truncated: boolean }>
|
||||
> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
const tag = await this.requireTag(creds, scope);
|
||||
const searchQuery = tagSearchQuery(tag.name);
|
||||
const walked = await this.client.listAllDocuments(creds, searchQuery === null ? {} : { searchQuery });
|
||||
|
||||
const documents = walked.documents.filter((doc) => inScope(doc, tag.id, searchQuery !== null)).map(toRemote);
|
||||
|
||||
// A capped walk saw part of the tag, and the core reads absence as a
|
||||
// possible deletion. Reporting no cursor keeps the incomplete picture out
|
||||
// of the agreed state instead of pinning it as the new truth.
|
||||
if (walked.truncated) {
|
||||
return { documents, cursor: null, cursorUnchanged: false, truncated: true };
|
||||
}
|
||||
|
||||
const cursor = enumerationDigest(documents);
|
||||
return { documents, cursor, cursorUnchanged: scope.cursor === cursor, truncated: false };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The scope is not re-checked here. Every route is organisation-scoped
|
||||
* upstream, so a document from elsewhere cannot be reached, and a document
|
||||
* that has lost its tag is exactly the one the core is fetching to resolve
|
||||
* the drift.
|
||||
*/
|
||||
async fetch(
|
||||
conn: DocumentConnectionRef,
|
||||
_scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<FetchResult>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
// The document first: the file route answers
|
||||
// `content-type: application/octet-stream` for everything, so the real
|
||||
// MIME type and the version marker only exist on the record.
|
||||
const doc = await this.client.getDocument(creds, remoteId);
|
||||
const download = await this.client.downloadDocument(creds, remoteId);
|
||||
return {
|
||||
body: download.body,
|
||||
size: download.size ?? doc.originalSize,
|
||||
mimeType: doc.mimeType,
|
||||
remoteVersion: snapshotVersion(doc),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a trip file as a Papra document and tag it into the trip's scope.
|
||||
*
|
||||
* `trekDocUid`/`trekTripUid` are not written anywhere: Papra's custom
|
||||
* properties are session-only, so a key has nowhere to put an anchor. The tag
|
||||
* is the whole binding, and a human who removes it upstream detaches the
|
||||
* document, which is `scope_drift` in the core, not a silent loss.
|
||||
*
|
||||
* The tag goes on in a second call because the upload route ignores every
|
||||
* multipart field except the file. That leaves a window in which the document
|
||||
* exists untagged; it closes on the next call, and a concurrent listing that
|
||||
* misses it picks it up on the following run.
|
||||
*/
|
||||
async push(conn: DocumentConnectionRef, scope: DocumentScopeRef, req: PushRequest): Promise<DocResult<PushResult>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
if (req.remoteId !== undefined) {
|
||||
throw new PapraError(
|
||||
'provider_error',
|
||||
'Papra has no endpoint that replaces the bytes of a document; push a new one instead (replaceInPlace is false)',
|
||||
);
|
||||
}
|
||||
|
||||
const tag = await this.requireTag(creds, scope);
|
||||
|
||||
let document: PapraDocument;
|
||||
let deduplicated: boolean;
|
||||
try {
|
||||
const uploaded = await this.client.uploadDocument(creds, {
|
||||
body: req.body,
|
||||
fileName: req.fileName,
|
||||
mimeType: req.mimeType,
|
||||
});
|
||||
document = uploaded.document;
|
||||
deduplicated = uploaded.deduplicated;
|
||||
} catch (err: unknown) {
|
||||
if (!(err instanceof PapraError) || err.papraCode !== 'document.already_exists') throw err;
|
||||
|
||||
// Papra refused the bytes because it already holds them. The twin is
|
||||
// the document this push is about, so it has to be found before the tag
|
||||
// can go on, and it is found by hash, because a twin stored under
|
||||
// another name would otherwise pair the wrong document.
|
||||
const twin = await this.client.findByHash(creds, req.sha256, req.fileName);
|
||||
if (!twin) {
|
||||
throw new PapraError(
|
||||
'conflict',
|
||||
'Papra holds these bytes already but did not return the document that has them',
|
||||
err.status,
|
||||
err.detail,
|
||||
err.papraCode,
|
||||
);
|
||||
}
|
||||
document = twin;
|
||||
deduplicated = true;
|
||||
}
|
||||
|
||||
await this.client.addTag(creds, document.id, tag.id);
|
||||
|
||||
return {
|
||||
remoteId: document.id,
|
||||
remoteVersion: snapshotVersion(document),
|
||||
remoteModifiedAt: document.updatedAt ?? document.createdAt,
|
||||
deduplicated,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async rename(
|
||||
conn: DocumentConnectionRef,
|
||||
_scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
name: string,
|
||||
): Promise<DocResult<{ remoteVersion: string }>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
const doc = await this.client.renameDocument(creds, remoteId, name);
|
||||
return { remoteVersion: snapshotVersion(doc) };
|
||||
});
|
||||
}
|
||||
|
||||
async trash(conn: DocumentConnectionRef, _scope: DocumentScopeRef, remoteId: string): Promise<DocResult<void>> {
|
||||
return this.withCreds(conn, async (creds) => {
|
||||
await this.client.trashDocument(creds, remoteId);
|
||||
});
|
||||
}
|
||||
|
||||
/** The bound tag, or a `scope_missing` a caller can surface as `scope_lost`. */
|
||||
private async requireTag(creds: PapraCreds, scope: DocumentScopeRef): Promise<PapraTag> {
|
||||
const parsed = parseScopeKey(scope.scopeKey);
|
||||
if (!parsed) {
|
||||
throw new PapraError('scope_missing', 'The stored scope key is not a Papra scope', undefined, scope.scopeKey);
|
||||
}
|
||||
if (parsed.organizationId !== creds.organizationId) {
|
||||
throw new PapraError(
|
||||
'scope_missing',
|
||||
'The bound tag belongs to a different Papra organisation than this connection',
|
||||
undefined,
|
||||
`scope ${parsed.organizationId}, connection ${creds.organizationId}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Papra has no single-tag route, so existence is answered by the list, the
|
||||
// same call that supplies the name the document search needs.
|
||||
const tags = await this.client.listTags(creds);
|
||||
const tag = tags.find((candidate) => candidate.id === parsed.tagId);
|
||||
if (!tag) {
|
||||
throw new PapraError('scope_missing', 'The tag this trip is bound to no longer exists', undefined, parsed.tagId);
|
||||
}
|
||||
return tag;
|
||||
}
|
||||
|
||||
/** The organisation's name, if the key is allowed to read it. */
|
||||
private async accountLabel(creds: PapraCreds): Promise<string> {
|
||||
try {
|
||||
const organizations = await this.client.listOrganizations(creds);
|
||||
const mine = organizations.find((org) => org.id === creds.organizationId);
|
||||
if (mine) return mine.name;
|
||||
} catch {
|
||||
// `organizations:read` is a separate scope a working sync key need not
|
||||
// hold, so its absence costs a nicer label and nothing else.
|
||||
}
|
||||
return creds.organizationId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the connection's credentials and run one operation on them.
|
||||
*
|
||||
* Every operation answers with a result rather than a throw, because the
|
||||
* settings screen shows a failed probe inline. Reading the credentials inside
|
||||
* the same guard is what keeps a half-filled connection form on that path
|
||||
* too: it is reported exactly like a provider fault, and nothing reaches the
|
||||
* network to find out.
|
||||
*/
|
||||
private async withCreds<T>(
|
||||
conn: DocumentConnectionRef,
|
||||
run: (creds: PapraCreds) => Promise<T>,
|
||||
): Promise<DocResult<T>> {
|
||||
try {
|
||||
return docOk(await run(readCreds(conn)));
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof PapraError) return docFail<T>(err.code, describe(err), err.status);
|
||||
return docFail<T>('unknown', err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function toScopeOption(organizationId: string, tag: PapraTag): DocumentScopeOption {
|
||||
return {
|
||||
scopeKey: buildScopeKey(organizationId, tag.id),
|
||||
label: tag.name,
|
||||
remoteRootId: tag.id,
|
||||
remoteRootPath: `#${tag.name}`,
|
||||
};
|
||||
}
|
||||
|
||||
function toRemote(doc: PapraDocument): RemoteDocument {
|
||||
return {
|
||||
remoteId: doc.id,
|
||||
name: doc.name,
|
||||
size: doc.originalSize,
|
||||
mimeType: doc.mimeType,
|
||||
remoteVersion: snapshotVersion(doc),
|
||||
contentHash: doc.originalSha256Hash,
|
||||
// In practice the creation time: Papra writes `updatedAt` once and never
|
||||
// touches it again, not on a rename, a tag change or a delete.
|
||||
remoteModifiedAt: doc.updatedAt ?? doc.createdAt,
|
||||
isDeleted: doc.isDeleted,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a listed document really carries the bound tag.
|
||||
*
|
||||
* The search already filters server-side, so this is belt and braces, except
|
||||
* for the one case that matters: a tag name the grammar cannot express makes
|
||||
* the client enumerate the whole organisation, and then this is the only filter
|
||||
* there is. A document that states no tags at all is kept when the server did
|
||||
* the filtering, because reading a missing field as "no tags" would empty the
|
||||
* listing, and an empty listing is how a mass delete looks to the core.
|
||||
*/
|
||||
function inScope(doc: PapraDocument, tagId: string, serverFiltered: boolean): boolean {
|
||||
if (doc.tags === null) return serverFiltered;
|
||||
return doc.tags.some((tag) => tag.id === tagId);
|
||||
}
|
||||
|
||||
/**
|
||||
* A fingerprint of the whole enumeration, used as the cursor.
|
||||
*
|
||||
* Papra offers nothing to short-circuit a walk with, so this cannot save the
|
||||
* enumeration. It only lets the core skip diffing when the tag is provably
|
||||
* unchanged. It covers deletions as well as edits because it is built from the
|
||||
* complete set rather than from the newest entry.
|
||||
*/
|
||||
function enumerationDigest(documents: RemoteDocument[]): string {
|
||||
const pairs = documents
|
||||
.map((doc) => `${doc.remoteId}:${doc.remoteVersion}`)
|
||||
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
|
||||
return crypto.createHash('sha256').update(pairs.join('\n')).digest('hex');
|
||||
}
|
||||
|
||||
export function buildScopeKey(organizationId: string, tagId: string): string {
|
||||
return `org:${organizationId}/tag:${tagId}`;
|
||||
}
|
||||
|
||||
export function parseScopeKey(scopeKey: string): { organizationId: string; tagId: string } | null {
|
||||
const match = /^org:([A-Za-z0-9_-]{1,64})\/tag:([A-Za-z0-9_-]{1,64})$/.exec(scopeKey.trim());
|
||||
if (!match) return null;
|
||||
return { organizationId: match[1], tagId: match[2] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection fields to client credentials.
|
||||
*
|
||||
* A missing field is answered here rather than as a 401 from Papra, because the
|
||||
* two look identical to the admin staring at the form and only one of them is
|
||||
* fixed by issuing a new key.
|
||||
*/
|
||||
function readCreds(conn: DocumentConnectionRef): PapraCreds {
|
||||
const apiKey = conn.secrets.api_key?.trim();
|
||||
if (!apiKey) throw new PapraError('unauthorized', 'The connection has no Papra API key');
|
||||
|
||||
const organizationId = conn.settings.organization_id?.trim();
|
||||
if (!organizationId) throw new PapraError('provider_error', 'The connection names no Papra organisation');
|
||||
|
||||
const baseUrl = conn.baseUrl.trim();
|
||||
if (!baseUrl) throw new PapraError('provider_error', 'The connection has no Papra URL');
|
||||
|
||||
return { baseUrl, apiKey, organizationId, allowInsecureTls: conn.allowInsecureTls };
|
||||
}
|
||||
|
||||
/** Upstream text for the self-hoster's log, never for the user's file list. */
|
||||
function describe(err: PapraError): string {
|
||||
const parts = [err.message];
|
||||
if (err.papraCode) parts.push(`papra=${err.papraCode}`);
|
||||
if (err.detail) parts.push(err.detail);
|
||||
return parts.join(' | ');
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
import { pipeline, Readable, Transform } from 'node:stream';
|
||||
import type { DocsyncErrorCode } from '@trek/shared';
|
||||
import { safeFetch, SsrfBlockedError } from '../../../utils/ssrfGuard';
|
||||
import { discardBody } from '../../../utils/cappedFetch';
|
||||
|
||||
/**
|
||||
* The transport the four document clients share: the SSRF-guarded fetch with
|
||||
* its timeout and certificate switch, the reading of a request that never got
|
||||
* an answer, the base status table, and the guard on a streamed download.
|
||||
*
|
||||
* What stays in the clients is what differs on purpose: how each one
|
||||
* authenticates, the shape of its requests (Papra's JSON, WebDAV's methods and
|
||||
* Depth header, DSM's JSON-encoded form parameters), how it reads an error
|
||||
* body, and which statuses mean something narrower on that product. Each
|
||||
* client also keeps its own error class and wording, so the `instanceof`
|
||||
* checks in the adapters see exactly the errors they always saw.
|
||||
*/
|
||||
|
||||
/** The four ways a request can fail without an HTTP answer. */
|
||||
export type TransportFailureCode = Extract<
|
||||
DocsyncErrorCode,
|
||||
'ssrf_blocked' | 'timeout' | 'tls_untrusted' | 'unreachable'
|
||||
>;
|
||||
|
||||
export interface TransportFailure {
|
||||
code: TransportFailureCode;
|
||||
/** Every message in the cause chain, for the self-hoster's log. */
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface ProviderFetchOptions {
|
||||
timeoutMs: number;
|
||||
/** The connection's self-signed switch. It relaxes the certificate check only; the SSRF guard still applies. */
|
||||
allowInsecureTls: boolean;
|
||||
/** The client's own error for a request that never got an answer. */
|
||||
onTransportFailure: (failure: TransportFailure) => Error;
|
||||
}
|
||||
|
||||
export interface DownloadGuard {
|
||||
maxBytes: number;
|
||||
/** The client's error for a download that announces more than `maxBytes`. */
|
||||
tooLarge: (declared: number) => Error;
|
||||
/** And for one that arrives without a body to stream. */
|
||||
noBody: () => Error;
|
||||
}
|
||||
|
||||
export interface GuardedDownload {
|
||||
body: Readable;
|
||||
/** The announced length, or null when the answer did not state one. */
|
||||
size: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The ceiling for one download where an adapter has no number of its own.
|
||||
*
|
||||
* A runaway guard, not the install's file-size policy: the sync core applies
|
||||
* that to every byte it pulls, and this sits far above it so that the core's
|
||||
* verdict is the one a user sees.
|
||||
*/
|
||||
export const DOWNLOAD_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* The abort `AbortSignal.timeout` fires, and undici's header and body timers:
|
||||
* a host that took the request and then went quiet. undici's connect timer is
|
||||
* left out on purpose. A host that never completed the handshake (switched
|
||||
* off, or behind a firewall that drops the SYN) did not answer at all, so it
|
||||
* reads as unreachable.
|
||||
*/
|
||||
const TIMEOUT_NAME = /^(TimeoutError|AbortError)$/;
|
||||
const TIMEOUT_CODE = /ABORT_ERR|HEADERS_TIMEOUT|BODY_TIMEOUT/i;
|
||||
|
||||
/**
|
||||
* Codes that are a verdict on the certificate rather than on the connection,
|
||||
* which is what lets the settings screen offer the self-signed switch instead
|
||||
* of a dead end. Most carry CERT or TLS in the name; the verify codes listed
|
||||
* here do not, and the switch gets past each of them all the same.
|
||||
*
|
||||
* ERR_SSL_* is deliberately not among them: a handshake that fails on the
|
||||
* protocol (https against a plain http port, say) is not something the switch
|
||||
* can fix, and `tls_untrusted` tells the user it is.
|
||||
*/
|
||||
const CERTIFICATE_CODES = new Set([
|
||||
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
|
||||
'HOSTNAME_MISMATCH',
|
||||
'INVALID_PURPOSE',
|
||||
'INVALID_CA',
|
||||
'PATH_LENGTH_EXCEEDED',
|
||||
]);
|
||||
const CERTIFICATE_CODE = /CERT|SELF_SIGNED|ERR_TLS/i;
|
||||
|
||||
/**
|
||||
* The codes every product answers alike. A client layers its own meaning on
|
||||
* top through `overrides`, and `fallback` answers whatever neither table knows.
|
||||
*/
|
||||
const STATUS_CODES: Readonly<Record<number, DocsyncErrorCode>> = {
|
||||
401: 'unauthorized',
|
||||
403: 'forbidden',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'too_large',
|
||||
415: 'unsupported_type',
|
||||
429: 'rate_limited',
|
||||
507: 'quota_exceeded',
|
||||
};
|
||||
|
||||
/** The error and each `cause` below it. undici hides the real reason one or two levels down. */
|
||||
function causeChain(err: unknown): unknown[] {
|
||||
const chain: unknown[] = [];
|
||||
let current: unknown = err;
|
||||
for (let depth = 0; depth < 5 && current !== undefined && current !== null; depth++) {
|
||||
chain.push(current);
|
||||
current = current instanceof Error ? current.cause : undefined;
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
|
||||
function codeOf(entry: unknown): string {
|
||||
return typeof entry === 'object' && entry !== null && 'code' in entry
|
||||
? String((entry as { code: unknown }).code)
|
||||
: '';
|
||||
}
|
||||
|
||||
/** Why a request never produced a response. */
|
||||
export function classifyTransportFailure(err: unknown): TransportFailure {
|
||||
const chain = causeChain(err);
|
||||
const detail = chain
|
||||
.map((entry) => (entry instanceof Error ? entry.message : String(entry)))
|
||||
.filter((message) => message.length > 0)
|
||||
.join(': ');
|
||||
|
||||
if (err instanceof SsrfBlockedError) return { code: 'ssrf_blocked', detail };
|
||||
|
||||
const codes = chain.map(codeOf);
|
||||
const names = chain.map((entry) => (entry instanceof Error ? entry.name : ''));
|
||||
if (names.some((name) => TIMEOUT_NAME.test(name)) || codes.some((code) => TIMEOUT_CODE.test(code))) {
|
||||
return { code: 'timeout', detail };
|
||||
}
|
||||
if (codes.some((code) => CERTIFICATE_CODES.has(code) || CERTIFICATE_CODE.test(code))) {
|
||||
return { code: 'tls_untrusted', detail };
|
||||
}
|
||||
return { code: 'unreachable', detail };
|
||||
}
|
||||
|
||||
/**
|
||||
* One request to a user's instance: through the SSRF guard, bounded by
|
||||
* `timeoutMs`, with the certificate check relaxed only when the connection
|
||||
* asks for it. The timeout covers the body as well, so a streamed transfer is
|
||||
* bounded end to end rather than only until the headers arrive.
|
||||
*/
|
||||
export async function providerFetch(url: string, init: RequestInit, options: ProviderFetchOptions): Promise<Response> {
|
||||
try {
|
||||
return await safeFetch(
|
||||
url,
|
||||
{ ...init, signal: AbortSignal.timeout(options.timeoutMs) as AbortSignal },
|
||||
{ rejectUnauthorized: !options.allowInsecureTls },
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
throw options.onTransportFailure(classifyTransportFailure(err));
|
||||
}
|
||||
}
|
||||
|
||||
/** The docsync code for an HTTP status that is not a product-specific answer. */
|
||||
export function statusErrorCode(
|
||||
status: number,
|
||||
overrides: Readonly<Record<number, DocsyncErrorCode>> = {},
|
||||
fallback: DocsyncErrorCode = 'provider_error',
|
||||
): DocsyncErrorCode {
|
||||
return overrides[status] ?? STATUS_CODES[status] ?? fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* The byte count a response announced, or null.
|
||||
*
|
||||
* An absent header must not become 0: `Number('')` is 0, and a size of zero
|
||||
* reads as "an empty file" to everything downstream. Compressed and chunked
|
||||
* answers routinely leave it out.
|
||||
*/
|
||||
export function declaredLength(header: string | null): number | null {
|
||||
if (header === null || header.trim().length === 0) return null;
|
||||
const value = Number(header);
|
||||
return Number.isFinite(value) && value >= 0 ? value : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the bytes as they pass. A declared length is only half a cap: a
|
||||
* chunked answer declares nothing, and nothing stops a server from sending
|
||||
* more than it announced.
|
||||
*/
|
||||
function limitStream(source: Readable, maxBytes: number): Readable {
|
||||
let seen = 0;
|
||||
const limiter = new Transform({
|
||||
transform(chunk: Buffer, _encoding, done) {
|
||||
seen += chunk.length;
|
||||
if (seen > maxBytes) {
|
||||
done(new Error(`The download exceeded ${maxBytes} bytes`));
|
||||
return;
|
||||
}
|
||||
done(null, chunk);
|
||||
},
|
||||
});
|
||||
// pipeline rather than pipe: a limiter that gives up, or a consumer that
|
||||
// stops reading, has to take the source down with it. undici keeps the
|
||||
// socket reserved until the body is released.
|
||||
pipeline(source, limiter, () => {});
|
||||
return limiter;
|
||||
}
|
||||
|
||||
/**
|
||||
* A download handed on as a stream, never buffered: the caller pipes it into
|
||||
* storage. An announced length over the limit is refused before a byte is
|
||||
* read, and the bytes are counted on the way through for the answers that
|
||||
* announce nothing.
|
||||
*/
|
||||
export function guardDownload(response: Response, guard: DownloadGuard): GuardedDownload {
|
||||
const size = declaredLength(response.headers.get('content-length'));
|
||||
if (size !== null && size > guard.maxBytes) {
|
||||
discardBody(response);
|
||||
throw guard.tooLarge(size);
|
||||
}
|
||||
if (!response.body) throw guard.noBody();
|
||||
return {
|
||||
body: limitStream(Readable.fromWeb(response.body as Parameters<typeof Readable.fromWeb>[0]), guard.maxBytes),
|
||||
size,
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,631 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { createHash, type Hash } from 'node:crypto';
|
||||
import { Readable } from 'node:stream';
|
||||
import {
|
||||
docFail,
|
||||
docOk,
|
||||
type DocResult,
|
||||
type DocumentConnectionRef,
|
||||
type DocumentProvider,
|
||||
type DocumentProviderCapabilities,
|
||||
type DocumentScopeOption,
|
||||
type DocumentScopeRef,
|
||||
type FetchResult,
|
||||
type PushRequest,
|
||||
type PushResult,
|
||||
type RemoteDocument,
|
||||
} from '../document-provider';
|
||||
import { PROVIDER_MAX_PAGES } from '../doc-sync.constants';
|
||||
import { contentTypeFor } from '../../storage/content-type';
|
||||
import {
|
||||
isValidSynoName,
|
||||
isWithinScope,
|
||||
joinSynoPath,
|
||||
MAX_SCOPE_PATH_LENGTH,
|
||||
normalizeSynoPath,
|
||||
snapshotVersion,
|
||||
SynologyDriveClient,
|
||||
SynologyDriveError,
|
||||
type SynoEntry,
|
||||
type SynologyDriveCreds,
|
||||
} from './synology-drive.client';
|
||||
|
||||
/**
|
||||
* Synology Drive over the FileStation Web API.
|
||||
*
|
||||
* The scope is a folder and identity is its path (FileStation has no file id,
|
||||
* no change feed and no webhook), so this adapter is the one the sync core's
|
||||
* "full enumeration, nothing assumed" model was shaped around. Everything it
|
||||
* cannot honestly promise it declares as absent in `capabilities()` rather than
|
||||
* faking: no push, no stable id, no hash in the listing.
|
||||
*
|
||||
* All HTTP lives in `synology-drive.client.ts`; this file is scope arithmetic,
|
||||
* the mapping into the sync core's vocabulary, and the boundary checks DSM does
|
||||
* not do for us.
|
||||
*/
|
||||
|
||||
/** Matches `DOCUMENT_PROVIDER_IDS` in shared/src/docsync/docsync.schema.ts. */
|
||||
const PROVIDER_ID = 'synologydrive';
|
||||
|
||||
/** What the connection form offers as a placeholder, and the fallback here. */
|
||||
const DEFAULT_BASE_PATH = '/trek';
|
||||
|
||||
/** Scope keys are `path:` plus the normalised path. See `docsyncScopeKeySchema`. */
|
||||
const SCOPE_PREFIX = 'path:';
|
||||
|
||||
/**
|
||||
* DSM exposes no recycle-bin switch on the API's delete, so a propagated delete
|
||||
* moves the file here instead. It is inside the scope so it inherits the scope's
|
||||
* permissions, and it is skipped by every listing so a trashed document does not
|
||||
* come back as a new one on the next run.
|
||||
*/
|
||||
const TRASH_FOLDER = '.trek-trash';
|
||||
|
||||
/**
|
||||
* The adapter's own ceiling for one transfer. Not a policy: the install's limit
|
||||
* is the upload limit (`MAX_FILE_SIZE`), enforced by the core before and during
|
||||
* every transfer. This is just a bound so a wrong path or a truncated header
|
||||
* cannot start an unbounded read.
|
||||
*/
|
||||
const MAX_TRANSFER_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Up to this size an upload is verified against the NAS's own digest.
|
||||
*
|
||||
* SYNO.FileStation.MD5 is a task that reads the whole file, so it is priced per
|
||||
* megabyte on the NAS's disks. Below the threshold that is a fraction of the
|
||||
* upload that just happened; above it, the verification would cost more than the
|
||||
* transfer and the core's own sha256 of the local bytes is the better check.
|
||||
*/
|
||||
const MD5_VERIFY_MAX_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
/** How many scope options one picker call may answer with. */
|
||||
const MAX_SCOPE_OPTIONS = 200;
|
||||
|
||||
/**
|
||||
* Where the connection keeps DSM's device token among its secrets. No form
|
||||
* field has this key, which is what keeps it out of every response and out of
|
||||
* the form's reach; see `saveEarnedSecret` in the config service.
|
||||
*/
|
||||
const DEVICE_TOKEN_SECRET = 'device_token';
|
||||
|
||||
const CAPABILITIES: DocumentProviderCapabilities = {
|
||||
// FileStation has no subscription API at all. DSM's own notifications are
|
||||
// mail and mobile push, neither reachable from here.
|
||||
push: 'none',
|
||||
// The path IS the id: a rename upstream reads as delete + create.
|
||||
stableId: false,
|
||||
// Through the move into TRASH_FOLDER below, not through DSM's own recycle bin:
|
||||
// the API's delete is permanent and offers no flag to route it elsewhere.
|
||||
remoteTrash: true,
|
||||
// An upload with overwrite=true replaces the bytes under the same path.
|
||||
replaceInPlace: true,
|
||||
// FileStation computes MD5 only, as a task per file, and MD5 is not the
|
||||
// sha256 the core compares against. `contentHash` therefore stays null.
|
||||
contentHashInListing: false,
|
||||
maxUploadBytes: MAX_TRANSFER_BYTES,
|
||||
// DSM stores whatever it is given; there is no server-side type allow-list.
|
||||
acceptedMimeTypes: null,
|
||||
canCreateScope: true,
|
||||
};
|
||||
|
||||
/** Hash the bytes on their way past, without buffering them. */
|
||||
async function* hashing(source: Readable, hash: Hash): AsyncGenerator<Buffer> {
|
||||
for await (const chunk of source) {
|
||||
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array);
|
||||
hash.update(buffer);
|
||||
yield buffer;
|
||||
}
|
||||
}
|
||||
|
||||
function baseName(path: string): string {
|
||||
const cut = path.lastIndexOf('/');
|
||||
return cut === -1 ? path : path.slice(cut + 1);
|
||||
}
|
||||
|
||||
function parentPath(path: string): string {
|
||||
const cut = path.lastIndexOf('/');
|
||||
return cut <= 0 ? '/' : path.slice(0, cut);
|
||||
}
|
||||
|
||||
function isoFromSeconds(seconds: number | null): string | null {
|
||||
return seconds === null ? null : new Date(seconds * 1000).toISOString();
|
||||
}
|
||||
|
||||
function toRemoteDocument(entry: SynoEntry): RemoteDocument {
|
||||
return {
|
||||
remoteId: entry.path,
|
||||
name: entry.name,
|
||||
size: entry.size,
|
||||
// `additional.type` is the extension in capitals, not a media type, so the
|
||||
// shared mapping is the honest source for what the bytes are.
|
||||
mimeType: contentTypeFor(entry.name),
|
||||
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
|
||||
contentHash: null,
|
||||
remoteModifiedAt: isoFromSeconds(entry.mtimeSeconds),
|
||||
// FileStation keeps no tombstone: a file that is gone is simply absent.
|
||||
isDeleted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function scopeOption(path: string, label?: string): DocumentScopeOption {
|
||||
return {
|
||||
scopeKey: `${SCOPE_PREFIX}${path}`,
|
||||
label: label ?? baseName(path),
|
||||
// There is no stable folder id on a Synology share: the path is all there
|
||||
// is, and pretending otherwise would put a fiction in the binding.
|
||||
remoteRootId: null,
|
||||
remoteRootPath: path,
|
||||
};
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SynologyDriveDocumentProvider implements DocumentProvider {
|
||||
readonly id = PROVIDER_ID;
|
||||
|
||||
/**
|
||||
* Injected, like the Paperless and WebDAV clients: it holds the SID cache,
|
||||
* the lockouts and the device token slots, so there has to be exactly one of
|
||||
* it per process, and the module's singleton is that one.
|
||||
*/
|
||||
constructor(private readonly client: SynologyDriveClient) {}
|
||||
|
||||
capabilities(): DocumentProviderCapabilities {
|
||||
return CAPABILITIES;
|
||||
}
|
||||
|
||||
async probe(
|
||||
conn: DocumentConnectionRef,
|
||||
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
try {
|
||||
const probe = await this.client.probe(creds);
|
||||
if (!probe.availableApis.includes('SYNO.FileStation.List')) {
|
||||
return docFail(
|
||||
'provider_error',
|
||||
'The account logged in, but this DSM does not expose File Station. Install the File Station package and allow this account to use it.',
|
||||
);
|
||||
}
|
||||
return docOk({ account: creds.username, capabilities: CAPABILITIES });
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The shares this account can see, plus the folders under the configured base
|
||||
* path, the two levels an operator actually picks from. Shares are offered
|
||||
* too because a small NAS often has a share per purpose and no sub-folders.
|
||||
*/
|
||||
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
const basePath = this.basePath(conn);
|
||||
|
||||
try {
|
||||
const options = new Map<string, DocumentScopeOption>();
|
||||
for (const share of await this.client.listShares(creds)) {
|
||||
const path = normalizeSynoPath(share.path, MAX_SCOPE_PATH_LENGTH);
|
||||
if (path && share.isdir) options.set(path, scopeOption(path, share.name));
|
||||
}
|
||||
|
||||
// The base path is a convenience, not a requirement: an install that has
|
||||
// not created it yet should still see its shares rather than an error.
|
||||
try {
|
||||
const listing = await this.client.listFolder(creds, basePath);
|
||||
for (const entry of listing.entries) {
|
||||
if (!entry.isdir || entry.name === TRASH_FOLDER) continue;
|
||||
const path = normalizeSynoPath(entry.path, MAX_SCOPE_PATH_LENGTH);
|
||||
if (path) options.set(path, scopeOption(path, entry.name));
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (!(error instanceof SynologyDriveError) || error.code !== 'not_found') throw error;
|
||||
}
|
||||
|
||||
const needle = query?.trim().toLowerCase();
|
||||
const filtered = [...options.values()]
|
||||
.filter((option) => !needle || option.label.toLowerCase().includes(needle) || option.remoteRootPath!.toLowerCase().includes(needle))
|
||||
.sort((a, b) => a.remoteRootPath!.localeCompare(b.remoteRootPath!))
|
||||
.slice(0, MAX_SCOPE_OPTIONS);
|
||||
return docOk(filtered);
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
const trimmed = name.trim();
|
||||
if (!isValidSynoName(trimmed)) {
|
||||
return docFail('provider_error', 'DSM will not accept that folder name');
|
||||
}
|
||||
const basePath = this.basePath(conn);
|
||||
const path = normalizeSynoPath(joinSynoPath(basePath, trimmed), MAX_SCOPE_PATH_LENGTH);
|
||||
if (!path) return docFail('provider_error', 'The resulting folder path is not one DSM accepts');
|
||||
|
||||
try {
|
||||
const folder = await this.client.createFolder(creds, basePath, trimmed);
|
||||
const created = normalizeSynoPath(folder.path, MAX_SCOPE_PATH_LENGTH) ?? path;
|
||||
return docOk(scopeOption(created, folder.name));
|
||||
} catch (error: unknown) {
|
||||
// With force_parent DSM creates missing folders, but never a missing
|
||||
// share: that is a storage operation, not a file one. Saying "the base
|
||||
// path does not exist" is the actionable version of its 408.
|
||||
if (error instanceof SynologyDriveError && error.code === 'not_found') {
|
||||
return docFail('scope_missing', `The base path ${basePath} does not exist on the NAS`);
|
||||
}
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
async resolveScope(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<DocumentScopeOption>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
const scopePath = this.scopePath(scope);
|
||||
if (!scopePath) return docFail('scope_missing', 'The stored scope key is not a usable path');
|
||||
|
||||
try {
|
||||
const entry = await this.client.getInfo(creds, scopePath);
|
||||
if (!entry.isdir) {
|
||||
return docFail('scope_missing', 'The bound path is a file on the NAS, not a folder');
|
||||
}
|
||||
return docOk(scopeOption(scopePath, entry.name));
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof SynologyDriveError && error.code === 'not_found') {
|
||||
return docFail('scope_missing', `The NAS no longer has ${scopePath}`);
|
||||
}
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file under the scope, folders walked breadth-first.
|
||||
*
|
||||
* Recursive because FileStation's `list` is not, and a sub-folder someone made
|
||||
* upstream would otherwise be an invisible hole in the trip's documents. One
|
||||
* request budget covers the whole walk, so a deep tree runs out of budget and
|
||||
* says `truncated` instead of holding a worker.
|
||||
*
|
||||
* The cursor is a digest of the result, not a change token: DSM offers nothing
|
||||
* to ask "what changed", and a folder's own mtime does not move when a file
|
||||
* inside it is edited in place. It cannot save the walk, so it is only used to
|
||||
* tell the core that the walk found exactly what it found last time.
|
||||
*/
|
||||
async list(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<{ documents: RemoteDocument[]; cursor: string | null; cursorUnchanged: boolean; truncated: boolean }>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
const scopePath = this.scopePath(scope);
|
||||
if (!scopePath) return docFail('scope_missing', 'The stored scope key is not a usable path');
|
||||
|
||||
const documents: RemoteDocument[] = [];
|
||||
const folders = [scopePath];
|
||||
const seen = new Set<string>([scopePath]);
|
||||
let budget = PROVIDER_MAX_PAGES;
|
||||
let truncated = false;
|
||||
|
||||
try {
|
||||
while (folders.length > 0) {
|
||||
if (budget <= 0) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const folder = folders.shift()!;
|
||||
const listing = await this.client.listFolder(creds, folder, { maxPages: budget });
|
||||
budget -= listing.pages;
|
||||
if (listing.truncated) truncated = true;
|
||||
|
||||
for (const entry of listing.entries) {
|
||||
const path = normalizeSynoPath(entry.path);
|
||||
// A path the adapter cannot spell is one it could not fetch, rename or
|
||||
// trash either, so it is left out rather than offered to the core.
|
||||
if (!path || !isWithinScope(scopePath, path)) continue;
|
||||
if (entry.isdir) {
|
||||
if (entry.name === TRASH_FOLDER || seen.has(path)) continue;
|
||||
seen.add(path);
|
||||
folders.push(path);
|
||||
continue;
|
||||
}
|
||||
documents.push(toRemoteDocument({ ...entry, path }));
|
||||
}
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof SynologyDriveError && error.code === 'not_found') {
|
||||
return docFail('scope_missing', `The NAS no longer has ${scopePath}`);
|
||||
}
|
||||
return this.failure(error);
|
||||
}
|
||||
|
||||
const digest = createHash('sha256');
|
||||
for (const document of [...documents].sort((a, b) => a.remoteId.localeCompare(b.remoteId))) {
|
||||
digest.update(`${document.remoteId}\t${document.remoteVersion}\n`);
|
||||
}
|
||||
const cursor = `v1:${digest.digest('hex')}`;
|
||||
return docOk({
|
||||
documents,
|
||||
cursor,
|
||||
// A partial walk saw a partial tree, so its digest says nothing about
|
||||
// whether anything changed.
|
||||
cursorUnchanged: !truncated && scope.cursor === cursor,
|
||||
truncated,
|
||||
});
|
||||
}
|
||||
|
||||
async fetch(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<FetchResult>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
|
||||
try {
|
||||
const path = this.requireInScope(scope, remoteId);
|
||||
// The version is read before the bytes so it describes the file that is
|
||||
// about to be streamed. Reading it after would describe whatever the file
|
||||
// became while it was being read.
|
||||
const entry = await this.client.getInfo(creds, path);
|
||||
if (entry.isdir) return docFail('not_found', 'That path is a folder on the NAS');
|
||||
const download = await this.client.download(creds, path, MAX_TRANSFER_BYTES);
|
||||
return docOk({
|
||||
body: download.body,
|
||||
size: download.size ?? entry.size,
|
||||
mimeType: contentTypeFor(entry.name),
|
||||
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload, then confirm what landed.
|
||||
*
|
||||
* The confirmation is not ceremony: FileStation answers `success: true` as
|
||||
* soon as it has written the part, and the version the core stores has to
|
||||
* describe the file as the NAS now sees it (including the mtime DSM rounded
|
||||
* to whole seconds), or the next listing reads TREK's own upload as a change
|
||||
* made upstream.
|
||||
*/
|
||||
async push(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
req: PushRequest,
|
||||
): Promise<DocResult<PushResult>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
if (req.size > MAX_TRANSFER_BYTES) {
|
||||
return docFail('too_large', `This adapter refuses transfers over ${MAX_TRANSFER_BYTES} bytes`);
|
||||
}
|
||||
|
||||
const md5 = createHash('md5');
|
||||
try {
|
||||
const scopePath = this.requireScopePath(scope);
|
||||
let folderPath = scopePath;
|
||||
let fileName = req.fileName;
|
||||
if (req.remoteId !== undefined) {
|
||||
const replacing = this.requireInScope(scope, req.remoteId);
|
||||
folderPath = parentPath(replacing);
|
||||
// The name follows the path being replaced, not the request: changing
|
||||
// both the bytes and the name in one upload would leave the old path
|
||||
// behind as a second document.
|
||||
fileName = baseName(replacing);
|
||||
}
|
||||
if (!isValidSynoName(fileName)) {
|
||||
return docFail('provider_error', 'DSM will not accept that file name');
|
||||
}
|
||||
const targetPath = joinSynoPath(folderPath, fileName);
|
||||
if (normalizeSynoPath(targetPath) !== targetPath || !isWithinScope(scopePath, targetPath)) {
|
||||
return docFail('provider_error', 'The resulting file path is not one DSM accepts');
|
||||
}
|
||||
|
||||
await this.client.upload(creds, {
|
||||
folderPath,
|
||||
fileName,
|
||||
body: Readable.from(hashing(req.body, md5)),
|
||||
size: req.size,
|
||||
mimeType: req.mimeType,
|
||||
mtimeSeconds: req.mtimeSeconds,
|
||||
// Replacing is only ever what the caller asked for. Without a remoteId a
|
||||
// name collision is a real conflict: the file at that path is somebody
|
||||
// else's, and overwriting it would destroy it silently.
|
||||
overwrite: req.remoteId !== undefined,
|
||||
createParents: true,
|
||||
});
|
||||
|
||||
const entry = await this.client.getInfo(creds, targetPath);
|
||||
if (entry.size !== null && entry.size !== req.size) {
|
||||
return docFail(
|
||||
'checksum_mismatch',
|
||||
`The NAS stored ${entry.size} bytes of an ${req.size}-byte upload`,
|
||||
);
|
||||
}
|
||||
if (req.size <= MD5_VERIFY_MAX_BYTES) {
|
||||
const remote = await this.client.md5(creds, targetPath);
|
||||
if (remote !== md5.digest('hex')) {
|
||||
return docFail('checksum_mismatch', 'The NAS computed a different digest for the uploaded bytes');
|
||||
}
|
||||
}
|
||||
|
||||
return docOk({
|
||||
remoteId: entry.path,
|
||||
remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds),
|
||||
remoteModifiedAt: isoFromSeconds(entry.mtimeSeconds),
|
||||
// FileStation stores what it is given; there is no content-addressed
|
||||
// store behind it that could answer with an existing file.
|
||||
deduplicated: false,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename in place.
|
||||
*
|
||||
* The caller's `remoteId` is stale afterwards, because the path is the id.
|
||||
* That is why `capabilities().stableId` is false: the core is expected to
|
||||
* re-derive the id from the folder and the new name, or to find it on the next
|
||||
* enumeration.
|
||||
*/
|
||||
async rename(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
name: string,
|
||||
): Promise<DocResult<{ remoteVersion: string }>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
const trimmed = name.trim();
|
||||
if (!isValidSynoName(trimmed)) {
|
||||
return docFail('provider_error', 'DSM will not accept that file name');
|
||||
}
|
||||
|
||||
try {
|
||||
const entry = await this.client.rename(creds, this.requireInScope(scope, remoteId), trimmed);
|
||||
return docOk({ remoteVersion: snapshotVersion(entry.path, entry.size, entry.mtimeSeconds) });
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Move into the scope's own bin.
|
||||
*
|
||||
* DSM's API delete is permanent and takes no "to the recycle bin" flag, and a
|
||||
* recycle bin is not even enabled on every share. A document that four people
|
||||
* are travelling on must not be one API call away from gone, so a propagated
|
||||
* delete becomes a move that any of them can undo in File Station.
|
||||
*/
|
||||
async trash(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<void>> {
|
||||
const creds = this.credentials(conn);
|
||||
if (!creds) return docFail('unauthorized', 'username and password are required');
|
||||
|
||||
try {
|
||||
const scopePath = this.requireScopePath(scope);
|
||||
const path = this.requireInScope(scope, remoteId);
|
||||
const trashPath = joinSynoPath(scopePath, TRASH_FOLDER);
|
||||
await this.client.createFolder(creds, scopePath, TRASH_FOLDER);
|
||||
try {
|
||||
await this.client.move(creds, path, trashPath, { overwrite: false });
|
||||
} catch (error: unknown) {
|
||||
if (!(error instanceof SynologyDriveError) || error.code !== 'conflict') throw error;
|
||||
// A file of that name is already in the bin. Overwriting it would throw
|
||||
// away the older copy, which is the one thing a bin exists to prevent,
|
||||
// so the older copy steps aside under a timestamped name.
|
||||
const occupant = joinSynoPath(trashPath, baseName(path));
|
||||
await this.client.rename(creds, occupant, this.archivedName(baseName(path)));
|
||||
await this.client.move(creds, path, trashPath, { overwrite: false });
|
||||
}
|
||||
return docOk(undefined);
|
||||
} catch (error: unknown) {
|
||||
return this.failure(error);
|
||||
}
|
||||
}
|
||||
|
||||
/** `boarding.pdf` → `boarding.1789745537.pdf`, keeping the extension readable. */
|
||||
private archivedName(name: string): string {
|
||||
const stamp = Math.floor(Date.now() / 1000);
|
||||
const dot = name.lastIndexOf('.');
|
||||
return dot <= 0 ? `${name}.${stamp}` : `${name.slice(0, dot)}.${stamp}${name.slice(dot)}`;
|
||||
}
|
||||
|
||||
private credentials(conn: DocumentConnectionRef): SynologyDriveCreds | null {
|
||||
const username = (conn.settings.username ?? '').trim();
|
||||
const password = conn.secrets.password ?? '';
|
||||
if (!username || !password || !conn.baseUrl.trim()) return null;
|
||||
const otpCode = (conn.secrets.otp_code ?? '').trim();
|
||||
const save = conn.saveSecret;
|
||||
return {
|
||||
connectionId: conn.connectionId,
|
||||
connectionCreatedAt: conn.createdAt,
|
||||
baseUrl: conn.baseUrl,
|
||||
username,
|
||||
password,
|
||||
otpCode: otpCode === '' ? undefined : otpCode,
|
||||
storedDeviceToken: conn.secrets[DEVICE_TOKEN_SECRET],
|
||||
saveDeviceToken: save ? (stored) => save(DEVICE_TOKEN_SECRET, stored) : undefined,
|
||||
allowInsecureTls: conn.allowInsecureTls,
|
||||
};
|
||||
}
|
||||
|
||||
private basePath(conn: DocumentConnectionRef): string {
|
||||
const configured = (conn.settings.base_path ?? '').trim();
|
||||
if (!configured) return DEFAULT_BASE_PATH;
|
||||
return normalizeSynoPath(configured, MAX_SCOPE_PATH_LENGTH) ?? DEFAULT_BASE_PATH;
|
||||
}
|
||||
|
||||
/**
|
||||
* The folder this trip is bound to, from the scope key.
|
||||
*
|
||||
* `remoteRootPath` is only a fallback: the key is what a unique index protects
|
||||
* and what a restore keeps, so a row whose two columns disagree is read the
|
||||
* way the constraint reads it.
|
||||
*/
|
||||
private scopePath(scope: DocumentScopeRef): string | null {
|
||||
const raw = scope.scopeKey.startsWith(SCOPE_PREFIX)
|
||||
? scope.scopeKey.slice(SCOPE_PREFIX.length)
|
||||
: scope.remoteRootPath;
|
||||
if (!raw) return null;
|
||||
const path = normalizeSynoPath(raw, MAX_SCOPE_PATH_LENGTH);
|
||||
// The FileStation root is not a scope: it holds the shares, and binding a
|
||||
// trip to it would put every share on the NAS inside one trip.
|
||||
return path === null || path === '/' ? null : path;
|
||||
}
|
||||
|
||||
/**
|
||||
* A remote id turned into a path this call is allowed to touch.
|
||||
*
|
||||
* DSM checks DSM permissions and nothing else: the account TREK logs in with
|
||||
* can reach every share it owns, so `/trips/japan/../../payroll/2026.pdf` is a
|
||||
* perfectly legal request that another trip's binding must never be able to
|
||||
* make. The trip boundary exists only here.
|
||||
*
|
||||
* Throws rather than returning a result, so a caller cannot forget to check
|
||||
* it. Every caller already runs inside the try that turns a
|
||||
* `SynologyDriveError` into a `DocResult`.
|
||||
*/
|
||||
private requireInScope(scope: DocumentScopeRef, remoteId: string): string {
|
||||
const scopePath = this.requireScopePath(scope);
|
||||
const path = normalizeSynoPath(remoteId);
|
||||
if (!path || path === scopePath || !isWithinScope(scopePath, path)) {
|
||||
throw new SynologyDriveError('not_found', 'That path is outside the folder this trip is bound to', {
|
||||
detail: 'That path is outside the folder this trip is bound to',
|
||||
});
|
||||
}
|
||||
if (path.split('/').includes(TRASH_FOLDER)) {
|
||||
throw new SynologyDriveError('not_found', 'That path is inside the sync bin', {
|
||||
detail: 'That path is inside the sync bin',
|
||||
});
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
private requireScopePath(scope: DocumentScopeRef): string {
|
||||
const scopePath = this.scopePath(scope);
|
||||
if (!scopePath) {
|
||||
throw new SynologyDriveError('scope_missing', 'The stored scope key is not a usable path', {
|
||||
detail: 'The stored scope key is not a usable path',
|
||||
});
|
||||
}
|
||||
return scopePath;
|
||||
}
|
||||
|
||||
private failure<T>(error: unknown): DocResult<T> {
|
||||
if (error instanceof SynologyDriveError) {
|
||||
return docFail<T>(error.code, error.detail, error.status);
|
||||
}
|
||||
return docFail<T>('unknown', error instanceof Error ? error.message : undefined);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,782 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Readable } from 'node:stream';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import type { DocsyncErrorCode } from '@trek/shared';
|
||||
import { readCappedJson, readCappedText } from '../../../utils/cappedFetch';
|
||||
import {
|
||||
PROVIDER_JSON_MAX_BYTES,
|
||||
PROVIDER_TIMEOUT_MS,
|
||||
PROVIDER_TRANSFER_TIMEOUT_MS,
|
||||
} from '../doc-sync.constants';
|
||||
import { DOWNLOAD_MAX_BYTES, guardDownload, providerFetch, statusErrorCode } from './provider-http';
|
||||
|
||||
/**
|
||||
* HTTP client for Nextcloud and OpenCloud. This is the ONLY place that talks to
|
||||
* either instance; the adapter above it holds no URLs and no transport logic.
|
||||
*
|
||||
* Measured against Nextcloud 31.0.14 and OpenCloud 8.0.1 on 2026-09-18, not
|
||||
* against their prose documentation, which is silent or wrong on most of this:
|
||||
*
|
||||
* - **`OC-Checksum` is a Nextcloud-only header here.** OpenCloud answers HTTP
|
||||
* 400 to `SHA256:<hex>`: it knows SHA1, MD5 and ADLER32, and treats an
|
||||
* unknown algorithm exactly like a mismatch, so sending the sha256 TREK
|
||||
* already has would fail every upload into a space. Nextcloud accepts SHA256
|
||||
* but does NOT verify it: `SHA256:0000` is stored verbatim and served back
|
||||
* through `oc:checksums`. A checksum from a listing is therefore a claim,
|
||||
* never a proof, and the core still has to hash the bytes it downloads.
|
||||
* - **OpenCloud computes SHA1/MD5/ADLER32 itself** for every file, whether or
|
||||
* not one was sent, and never SHA256. That is why `contentHashInListing` is
|
||||
* false for both products rather than true for one.
|
||||
* - **`X-OC-MTime` is honoured by both**, answered with `X-OC-MTime: accepted`.
|
||||
* Without it every file TREK uploads comes back on the next run looking
|
||||
* freshly changed upstream, which is the classic sync loop.
|
||||
* - **Nextcloud's `OC-FileId` response header is not its `oc:fileid`**: the PUT
|
||||
* that creates `oc:fileid` 62 answers `OC-FileId: 00000062oc9fy1g5e2cj`
|
||||
* (zero-padded id plus instance id). Storing the header form would make every
|
||||
* file TREK uploaded look like a stranger on the next listing, so it is
|
||||
* normalised back to the numeric id. OpenCloud's two spellings agree.
|
||||
* - **PROPFIND has no pagination.** `Paginate: true` plus `Nc-Paginate-*` is
|
||||
* advertised in the `DAV:` header (`nc-paginate`) and ignored on PROPFIND, so
|
||||
* one Depth:1 request is the whole folder. Hence the capped read and the
|
||||
* repair of a cut-off body into the responses that did arrive, with
|
||||
* `truncated` set, rather than failing a big folder outright.
|
||||
* - **Webhook registration is form-encoded only.** A JSON body reaches the
|
||||
* controller with every argument null and the instance answers HTTP 500.
|
||||
*/
|
||||
|
||||
/** Which product a connection points at. Decided by the connection, not by the class. */
|
||||
export type WebdavFlavor = 'nextcloud' | 'opencloud';
|
||||
|
||||
export interface WebdavCreds {
|
||||
/** Instance origin, already normalised: no trailing slash, no DAV suffix. */
|
||||
origin: string;
|
||||
username: string;
|
||||
password: string;
|
||||
allowInsecureTls: boolean;
|
||||
flavor: WebdavFlavor;
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed call, carrying the reason as a code.
|
||||
*
|
||||
* The code is what reaches the user through i18n; `detail` keeps the upstream
|
||||
* text, which is English, sometimes an HTML login page from a reverse proxy, and
|
||||
* in Sabre's case a stack-shaped XML document.
|
||||
*/
|
||||
export class WebdavError extends Error {
|
||||
readonly code: DocsyncErrorCode;
|
||||
readonly status?: number;
|
||||
readonly detail?: string;
|
||||
|
||||
constructor(code: DocsyncErrorCode, message: string, status?: number, detail?: string) {
|
||||
super(message);
|
||||
this.name = 'WebdavError';
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
this.detail = detail;
|
||||
}
|
||||
}
|
||||
|
||||
/** One `<d:response>` of a PROPFIND, reduced to what the sync core reads. */
|
||||
export interface WebdavEntry {
|
||||
/** Path portion of the `d:href`, still percent-encoded, as sent. */
|
||||
href: string;
|
||||
/** The same path decoded, for comparing against a requested path. */
|
||||
path: string;
|
||||
name: string;
|
||||
isCollection: boolean;
|
||||
/** Quoted ETag exactly as the server spelled it, or '' when absent. */
|
||||
etag: string;
|
||||
fileId: string | null;
|
||||
size: number | null;
|
||||
mimeType: string | null;
|
||||
lastModifiedIso: string | null;
|
||||
/** Only when the instance carries a SHA256; see the header note above. */
|
||||
sha256: string | null;
|
||||
}
|
||||
|
||||
export interface WebdavPropfindResult {
|
||||
/** The requested resource itself. */
|
||||
self: WebdavEntry;
|
||||
children: WebdavEntry[];
|
||||
/** The body hit the size cap, so `children` is a prefix of the folder. */
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
/** An OpenCloud space, as `/graph/v1.0/…/drives` describes it. */
|
||||
export interface WebdavDrive {
|
||||
id: string;
|
||||
name: string;
|
||||
driveType: string;
|
||||
/**
|
||||
* Path of `root.webDavUrl`, never its origin: the instance answers with its
|
||||
* own configured `OC_URL` (`https://localhost:8804` for a host reached as
|
||||
* 127.0.0.1), and following a host that an untrusted response chose is how an
|
||||
* SSRF guard gets walked around. Only the path survives, re-based on the
|
||||
* connection's own origin.
|
||||
*/
|
||||
webDavPath: string;
|
||||
}
|
||||
|
||||
export interface WebdavPutOptions {
|
||||
size: number;
|
||||
mimeType: string;
|
||||
/** Unix seconds, sent as `X-OC-MTime`. */
|
||||
mtimeSeconds: number;
|
||||
sha256: string;
|
||||
/** Optimistic concurrency. Both products answer a mismatch with 412. */
|
||||
ifMatch?: string;
|
||||
/** `*` refuses to create over an existing resource (RFC 9110 §13.1.2). */
|
||||
ifNoneMatch?: string;
|
||||
}
|
||||
|
||||
export interface WebdavPutResult {
|
||||
fileId: string | null;
|
||||
etag: string | null;
|
||||
/** 201 rather than 204: a new resource rather than a replaced one. */
|
||||
created: boolean;
|
||||
}
|
||||
|
||||
export interface WebdavDownload {
|
||||
body: Readable;
|
||||
size: number | null;
|
||||
mimeType: string | null;
|
||||
etag: string;
|
||||
}
|
||||
|
||||
const PROPFIND_BODY =
|
||||
'<?xml version="1.0" encoding="UTF-8"?>' +
|
||||
'<d:propfind xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns"><d:prop>' +
|
||||
'<d:getetag/><d:getlastmodified/><d:getcontentlength/><d:getcontenttype/><d:resourcetype/>' +
|
||||
'<oc:fileid/><oc:checksums/>' +
|
||||
'</d:prop></d:propfind>';
|
||||
|
||||
/**
|
||||
* Namespace for the dead properties TREK writes. A URI, not an address: it is
|
||||
* never fetched, it only has to be unmistakably ours so a second tool writing
|
||||
* `doc-uid` on the same file cannot collide with it.
|
||||
*/
|
||||
export const TREK_PROP_NS = 'https://github.com/liketrek/TREK/ns/docsync';
|
||||
|
||||
/** Property names under that namespace, matching the anchors in `PushRequest`. */
|
||||
export const TREK_PROP_DOC_UID = 'doc-uid';
|
||||
export const TREK_PROP_TRIP_UID = 'trip-uid';
|
||||
|
||||
const OCS_WEBHOOKS = '/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks';
|
||||
|
||||
/** OCS refuses any request without the first header and answers XML without the second. */
|
||||
const OCS_HEADERS: Record<string, string> = {
|
||||
'OCS-APIRequest': 'true',
|
||||
Accept: 'application/json',
|
||||
};
|
||||
|
||||
/**
|
||||
* `removeNSPrefix` because the prefix is the server's choice rather than the
|
||||
* protocol's (`d:` on both of these, `D:` on other Sabre builds), and
|
||||
* `parseTagValue: false` because an ETag, a file id and a checksum are opaque
|
||||
* strings that have to survive a server spelling them as digits.
|
||||
*/
|
||||
const propfindParser = new XMLParser({
|
||||
removeNSPrefix: true,
|
||||
parseTagValue: false,
|
||||
ignoreAttributes: true,
|
||||
isArray: (name) => name === 'response' || name === 'propstat' || name === 'checksum',
|
||||
});
|
||||
|
||||
/** Nextcloud's `OC-FileId` header carries the instance id after the number. */
|
||||
export function normalizeFileId(raw: string | null, flavor: WebdavFlavor): string | null {
|
||||
const value = raw?.trim();
|
||||
if (!value) return null;
|
||||
if (flavor !== 'nextcloud') return value;
|
||||
const digits = /^0*(\d+)/.exec(value);
|
||||
return digits ? digits[1] : value;
|
||||
}
|
||||
|
||||
/**
|
||||
* What a user pastes into the URL field, reduced to an origin.
|
||||
*
|
||||
* People paste the address bar of their own file list, so the DAV and Graph
|
||||
* prefixes come off rather than turning into `/remote.php/dav/remote.php/dav`
|
||||
* 404s, which read like a wrong password.
|
||||
*/
|
||||
export function normalizeOrigin(baseUrl: string): string {
|
||||
return baseUrl
|
||||
.trim()
|
||||
.replace(/[?#].*$/, '')
|
||||
.replace(/\/+$/, '')
|
||||
.replace(/\/(?:remote|index)\.php(?:\/.*)?$/i, '')
|
||||
.replace(/\/dav\/spaces(?:\/.*)?$/i, '')
|
||||
.replace(/\/graph(?:\/.*)?$/i, '')
|
||||
.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/** Percent-encode each segment of a decoded path, keeping the separators. */
|
||||
export function encodePath(path: string): string {
|
||||
return path
|
||||
.split('/')
|
||||
.map((segment) => encodeURIComponent(segment))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
/** `/a/b/` and `a//b` both mean `/a/b`. Every comparison here depends on one spelling. */
|
||||
export function normalizePath(path: string): string {
|
||||
const collapsed = `/${path}`.replace(/\/+/g, '/').replace(/\/+$/, '');
|
||||
return collapsed === '' ? '/' : collapsed;
|
||||
}
|
||||
|
||||
/** The path portion of a stored root, which is a URL on OpenCloud and a path on Nextcloud. */
|
||||
export function davPathOf(rootPathOrUrl: string): string | null {
|
||||
const value = rootPathOrUrl.trim();
|
||||
if (!value) return null;
|
||||
if (/^https?:\/\//i.test(value)) {
|
||||
try {
|
||||
return normalizePath(new URL(value).pathname);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return value.startsWith('/') ? normalizePath(value) : null;
|
||||
}
|
||||
|
||||
function hrefPathname(href: string): string {
|
||||
try {
|
||||
return new URL(href).pathname;
|
||||
} catch {
|
||||
return href;
|
||||
}
|
||||
}
|
||||
|
||||
/** The still-encoded path of an href, which is what the next request is built from. */
|
||||
function hrefPath(href: string): string {
|
||||
return /^https?:\/\//i.test(href) ? hrefPathname(href) : href;
|
||||
}
|
||||
|
||||
function decodeHrefPath(href: string): string {
|
||||
const raw = hrefPath(href);
|
||||
try {
|
||||
return normalizePath(decodeURIComponent(raw));
|
||||
} catch {
|
||||
return normalizePath(raw);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 405 is MKCOL on an existing collection, 412 a failed If-Match, 423 a locked
|
||||
* node: with 409, four ways of saying the caller's picture of the remote is
|
||||
* stale.
|
||||
*/
|
||||
const STALE_STATUSES: Readonly<Record<number, DocsyncErrorCode>> = {
|
||||
405: 'conflict',
|
||||
412: 'conflict',
|
||||
423: 'conflict',
|
||||
};
|
||||
|
||||
function classifyStatus(status: number): DocsyncErrorCode {
|
||||
return statusErrorCode(status, STALE_STATUSES, status >= 500 ? 'provider_error' : 'unknown');
|
||||
}
|
||||
|
||||
/**
|
||||
* A body cut off at the cap is still a run of complete `<d:response>` blocks
|
||||
* followed by a fragment. Dropping the fragment and closing the document keeps
|
||||
* the first few thousand files of a huge folder usable, which is what
|
||||
* `truncated` is for. The alternative is telling someone with one big folder
|
||||
* that nothing works at all.
|
||||
*/
|
||||
function repairTruncatedMultistatus(xml: string): string | null {
|
||||
const closing = /<\/([a-z0-9]+:)?response\s*>/gi;
|
||||
let cut = -1;
|
||||
for (let match = closing.exec(xml); match !== null; match = closing.exec(xml)) {
|
||||
cut = match.index + match[0].length;
|
||||
}
|
||||
if (cut < 0) return null;
|
||||
const open = /<([a-z0-9]+:)?multistatus[^>]*>/i.exec(xml);
|
||||
if (!open) return null;
|
||||
return `${xml.slice(0, cut)}</${open[1] ?? ''}multistatus>`;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] {
|
||||
if (Array.isArray(value)) return value;
|
||||
return value === undefined || value === null ? [] : [value];
|
||||
}
|
||||
|
||||
function textOf(value: unknown): string | null {
|
||||
if (typeof value === 'string') return value;
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Props out of the propstat blocks that reported 2xx only. A PROPFIND answers
|
||||
* one block per status, and reading the 404 block too turns "this folder has no
|
||||
* content type" into an empty string that looks like a real value.
|
||||
*/
|
||||
function propsOf(response: Record<string, unknown>): Record<string, unknown> {
|
||||
const props: Record<string, unknown> = {};
|
||||
for (const stat of asArray(response.propstat)) {
|
||||
const record = asRecord(stat);
|
||||
if (!record) continue;
|
||||
const status = textOf(record.status) ?? '';
|
||||
if (!/\s2\d\d(\s|$)/.test(status)) continue;
|
||||
const prop = asRecord(record.prop);
|
||||
if (prop) Object.assign(props, prop);
|
||||
}
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* `SHA256:<hex>` out of either spelling: Nextcloud sends one element carrying
|
||||
* one algorithm, OpenCloud sends one element carrying all three separated by
|
||||
* spaces.
|
||||
*/
|
||||
function sha256Of(checksums: unknown): string | null {
|
||||
const record = asRecord(checksums);
|
||||
if (!record) return null;
|
||||
for (const entry of asArray(record.checksum)) {
|
||||
const text = textOf(entry);
|
||||
if (!text) continue;
|
||||
for (const token of text.split(/\s+/)) {
|
||||
const match = /^SHA256:([0-9a-f]{64})$/i.exec(token);
|
||||
if (match) return match[1].toLowerCase();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function entryOf(response: Record<string, unknown>, flavor: WebdavFlavor): WebdavEntry | null {
|
||||
const href = textOf(response.href);
|
||||
if (!href) return null;
|
||||
|
||||
const props = propsOf(response);
|
||||
const path = decodeHrefPath(href);
|
||||
const segments = path.split('/');
|
||||
const length = textOf(props.getcontentlength);
|
||||
const modified = textOf(props.getlastmodified);
|
||||
const parsedModified = modified ? new Date(modified) : null;
|
||||
const mimeType = textOf(props.getcontenttype);
|
||||
const resourceType = asRecord(props.resourcetype);
|
||||
|
||||
return {
|
||||
href: hrefPath(href),
|
||||
path,
|
||||
name: segments[segments.length - 1] ?? '',
|
||||
isCollection: resourceType !== null && 'collection' in resourceType,
|
||||
etag: textOf(props.getetag) ?? '',
|
||||
fileId: normalizeFileId(textOf(props.fileid), flavor),
|
||||
size: length !== null && length !== '' && Number.isFinite(Number(length)) ? Number(length) : null,
|
||||
mimeType: mimeType ? mimeType.split(';')[0].trim() : null,
|
||||
lastModifiedIso:
|
||||
parsedModified && !Number.isNaN(parsedModified.getTime()) ? parsedModified.toISOString() : null,
|
||||
sha256: sha256Of(props.checksums),
|
||||
};
|
||||
}
|
||||
|
||||
function toDrive(value: unknown): WebdavDrive | null {
|
||||
const record = asRecord(value);
|
||||
const id = textOf(record?.id);
|
||||
if (!record || !id) return null;
|
||||
const webDavUrl = textOf(asRecord(record.root)?.webDavUrl);
|
||||
return {
|
||||
id,
|
||||
name: textOf(record.name) ?? id,
|
||||
driveType: textOf(record.driveType) ?? '',
|
||||
webDavPath: (webDavUrl ? davPathOf(webDavUrl) : null) ?? `/dav/spaces/${encodeURIComponent(id)}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** Text content for the PROPPATCH body, which is built as a string rather than parsed back. */
|
||||
function escapeXmlText(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class WebdavClient {
|
||||
/**
|
||||
* One authenticated request. Throws for anything that never reached the
|
||||
* application, and hands the caller the response otherwise: a 404 on DELETE
|
||||
* and a 405 on MKCOL are answers, not failures.
|
||||
*/
|
||||
private async send(
|
||||
creds: WebdavCreds,
|
||||
path: string,
|
||||
init: RequestInit,
|
||||
timeoutMs = PROVIDER_TIMEOUT_MS,
|
||||
): Promise<Response> {
|
||||
const auth = Buffer.from(`${creds.username}:${creds.password}`, 'utf8').toString('base64');
|
||||
const headers = new Headers(init.headers);
|
||||
headers.set('Authorization', `Basic ${auth}`);
|
||||
|
||||
return providerFetch(`${creds.origin}${path}`, { ...init, headers }, {
|
||||
timeoutMs,
|
||||
allowInsecureTls: creds.allowInsecureTls,
|
||||
onTransportFailure: ({ code, detail }) =>
|
||||
new WebdavError(code, 'Could not reach the WebDAV instance', undefined, detail),
|
||||
});
|
||||
}
|
||||
|
||||
/** The error for a response nobody wanted, with a bounded slice of its body as detail. */
|
||||
private async failed(response: Response, what: string): Promise<WebdavError> {
|
||||
const { text } = await readCappedText(response, 8 * 1024);
|
||||
return new WebdavError(
|
||||
classifyStatus(response.status),
|
||||
`${what} answered HTTP ${response.status}`,
|
||||
response.status,
|
||||
text.slice(0, 500),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* PROPFIND with the props the sync core reads.
|
||||
*
|
||||
* Depth 0 is the cursor probe: a write anywhere below a collection changes
|
||||
* that collection's ETag on both products, up to the root, so an unchanged
|
||||
* root ETag means the whole walk can be skipped. Depth 1 is the walk.
|
||||
*/
|
||||
async propfind(creds: WebdavCreds, path: string, depth: 0 | 1): Promise<WebdavPropfindResult> {
|
||||
const response = await this.send(creds, path, {
|
||||
method: 'PROPFIND',
|
||||
headers: { Depth: String(depth), 'Content-Type': 'application/xml; charset=utf-8' },
|
||||
body: PROPFIND_BODY,
|
||||
});
|
||||
if (response.status !== 207) {
|
||||
throw await this.failed(response, 'PROPFIND');
|
||||
}
|
||||
|
||||
const { text, truncated } = await readCappedText(response, PROVIDER_JSON_MAX_BYTES);
|
||||
const xml = truncated ? repairTruncatedMultistatus(text) : text;
|
||||
if (xml === null) {
|
||||
throw new WebdavError('too_large', 'The folder listing exceeded the size limit', response.status);
|
||||
}
|
||||
|
||||
const multistatus = asRecord(asRecord(propfindParser.parse(xml))?.multistatus);
|
||||
if (!multistatus) {
|
||||
throw new WebdavError(
|
||||
'provider_error',
|
||||
'The instance answered something that is not a WebDAV multistatus',
|
||||
response.status,
|
||||
text.slice(0, 200),
|
||||
);
|
||||
}
|
||||
|
||||
const entries: WebdavEntry[] = [];
|
||||
for (const element of asArray(multistatus.response)) {
|
||||
const record = asRecord(element);
|
||||
const entry = record ? entryOf(record, creds.flavor) : null;
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
if (entries.length === 0) {
|
||||
throw new WebdavError('provider_error', 'The instance answered an empty multistatus', response.status);
|
||||
}
|
||||
|
||||
// RFC 4918 puts the requested resource first, but it is matched by path
|
||||
// rather than trusted by position: the root must never end up in the
|
||||
// document list, and one misordered server would put it there.
|
||||
const wanted = decodeHrefPath(path);
|
||||
const selfIndex = Math.max(
|
||||
entries.findIndex((entry) => entry.path === wanted),
|
||||
0,
|
||||
);
|
||||
return {
|
||||
self: entries[selfIndex],
|
||||
children: entries.filter((_, index) => index !== selfIndex),
|
||||
truncated,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Write dead properties, which both products persist (Nextcloud in
|
||||
* `oc_properties`, OpenCloud in the space's metadata) and hand back through a
|
||||
* PROPFIND that asks for them. That is where the TREK anchors go: they are the
|
||||
* only thing that still identifies a document after a human has renamed it and
|
||||
* moved it somewhere else in the folder.
|
||||
*
|
||||
* Answers whether the server accepted them. A 207 is not enough on its own:
|
||||
* a refusal arrives as a non-2xx status inside the multistatus.
|
||||
*/
|
||||
async proppatch(creds: WebdavCreds, path: string, props: Record<string, string>): Promise<boolean> {
|
||||
const body =
|
||||
'<?xml version="1.0" encoding="UTF-8"?>' +
|
||||
`<d:propertyupdate xmlns:d="DAV:" xmlns:trek="${TREK_PROP_NS}"><d:set><d:prop>` +
|
||||
Object.entries(props)
|
||||
.map(([name, value]) => `<trek:${name}>${escapeXmlText(value)}</trek:${name}>`)
|
||||
.join('') +
|
||||
'</d:prop></d:set></d:propertyupdate>';
|
||||
|
||||
const response = await this.send(creds, path, {
|
||||
method: 'PROPPATCH',
|
||||
headers: { 'Content-Type': 'application/xml; charset=utf-8' },
|
||||
body,
|
||||
});
|
||||
if (response.status !== 207) {
|
||||
throw await this.failed(response, 'PROPPATCH');
|
||||
}
|
||||
const { text } = await readCappedText(response, PROVIDER_JSON_MAX_BYTES);
|
||||
const multistatus = asRecord(asRecord(propfindParser.parse(text))?.multistatus);
|
||||
for (const element of asArray(multistatus?.response)) {
|
||||
const record = asRecord(element);
|
||||
if (!record) continue;
|
||||
const accepted = propsOf(record);
|
||||
if (Object.keys(props).every((name) => name in accepted)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** MKCOL. `exists` rather than an error, because 405 is how both say "already there". */
|
||||
async mkcol(creds: WebdavCreds, path: string): Promise<'created' | 'exists'> {
|
||||
const response = await this.send(creds, path, { method: 'MKCOL' });
|
||||
void response.body?.cancel().catch(() => {});
|
||||
if (response.ok) return 'created';
|
||||
if (response.status === 405) return 'exists';
|
||||
throw await this.failed(response, 'MKCOL');
|
||||
}
|
||||
|
||||
async put(
|
||||
creds: WebdavCreds,
|
||||
path: string,
|
||||
body: Readable,
|
||||
options: WebdavPutOptions,
|
||||
): Promise<WebdavPutResult> {
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': options.mimeType,
|
||||
// Explicit, so the upload does not go out chunked: a reverse proxy in
|
||||
// front of a self-hosted instance is the common case and several refuse
|
||||
// a chunked PUT outright.
|
||||
'Content-Length': String(options.size),
|
||||
'X-OC-MTime': String(options.mtimeSeconds),
|
||||
};
|
||||
if (creds.flavor === 'nextcloud' && /^[0-9a-f]{64}$/i.test(options.sha256)) {
|
||||
headers['OC-Checksum'] = `SHA256:${options.sha256.toLowerCase()}`;
|
||||
}
|
||||
if (options.ifNoneMatch) {
|
||||
headers['If-None-Match'] = options.ifNoneMatch;
|
||||
}
|
||||
if (options.ifMatch) {
|
||||
headers['If-Match'] = options.ifMatch;
|
||||
}
|
||||
|
||||
const response = await this.send(
|
||||
creds,
|
||||
path,
|
||||
{ method: 'PUT', headers, body, duplex: 'half' },
|
||||
PROVIDER_TRANSFER_TIMEOUT_MS,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw await this.failed(response, 'PUT');
|
||||
}
|
||||
void response.body?.cancel().catch(() => {});
|
||||
return {
|
||||
fileId: normalizeFileId(response.headers.get('oc-fileid'), creds.flavor),
|
||||
etag: response.headers.get('oc-etag') ?? response.headers.get('etag'),
|
||||
created: response.status === 201,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET, handed back as a stream.
|
||||
*
|
||||
* Never buffered: the caller pipes it into storage, and holding a trip's
|
||||
* scanned passport in memory to count its bytes is the opposite of what a
|
||||
* cap is for. The guard counts them on the way through instead.
|
||||
*/
|
||||
async get(creds: WebdavCreds, path: string): Promise<WebdavDownload> {
|
||||
const response = await this.send(creds, path, { method: 'GET' }, PROVIDER_TRANSFER_TIMEOUT_MS);
|
||||
if (!response.ok) {
|
||||
throw await this.failed(response, 'GET');
|
||||
}
|
||||
const { body, size } = guardDownload(response, {
|
||||
maxBytes: DOWNLOAD_MAX_BYTES,
|
||||
tooLarge: (declared) =>
|
||||
new WebdavError(
|
||||
'too_large',
|
||||
'The document is larger than TREK will transfer',
|
||||
response.status,
|
||||
`content_length=${declared}`,
|
||||
),
|
||||
noBody: () =>
|
||||
new WebdavError('provider_error', 'The instance answered a download without a body', response.status),
|
||||
});
|
||||
const mimeType = response.headers.get('content-type');
|
||||
return {
|
||||
body,
|
||||
size,
|
||||
mimeType: mimeType ? mimeType.split(';')[0].trim() : null,
|
||||
etag: response.headers.get('oc-etag') ?? response.headers.get('etag') ?? '',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* MOVE within one instance. `Overwrite: F` on purpose: a rename that silently
|
||||
* replaced a namesake would destroy a document nobody asked about, and the
|
||||
* 412 it produces instead is a conflict the core can show someone.
|
||||
*/
|
||||
async move(creds: WebdavCreds, fromPath: string, toPath: string): Promise<{ etag: string | null }> {
|
||||
const response = await this.send(creds, fromPath, {
|
||||
method: 'MOVE',
|
||||
headers: { Destination: `${creds.origin}${toPath}`, Overwrite: 'F' },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw await this.failed(response, 'MOVE');
|
||||
}
|
||||
void response.body?.cancel().catch(() => {});
|
||||
return { etag: response.headers.get('oc-etag') ?? response.headers.get('etag') };
|
||||
}
|
||||
|
||||
/** DELETE, which lands in the owner's trash on Nextcloud and the space trash on OpenCloud. */
|
||||
async delete(creds: WebdavCreds, path: string): Promise<void> {
|
||||
const response = await this.send(creds, path, { method: 'DELETE' });
|
||||
void response.body?.cancel().catch(() => {});
|
||||
// A document that is already gone is the state the caller asked for.
|
||||
if (response.ok || response.status === 404) return;
|
||||
throw await this.failed(response, 'DELETE');
|
||||
}
|
||||
|
||||
private async graph(
|
||||
creds: WebdavCreds,
|
||||
method: 'GET' | 'POST',
|
||||
path: string,
|
||||
body?: unknown,
|
||||
): Promise<unknown> {
|
||||
const response = await this.send(creds, path, {
|
||||
method,
|
||||
headers:
|
||||
body === undefined
|
||||
? { Accept: 'application/json' }
|
||||
: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw await this.failed(response, 'The Graph API');
|
||||
}
|
||||
const parsed = await readCappedJson<unknown>(response, PROVIDER_JSON_MAX_BYTES);
|
||||
if (parsed === undefined) {
|
||||
throw new WebdavError(
|
||||
'provider_error',
|
||||
'The Graph API answered something that is not JSON. Check that the URL points at OpenCloud itself.',
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async listDrives(creds: WebdavCreds): Promise<WebdavDrive[]> {
|
||||
const parsed = asRecord(await this.graph(creds, 'GET', '/graph/v1.0/me/drives'));
|
||||
const drives: WebdavDrive[] = [];
|
||||
for (const value of asArray(parsed?.value)) {
|
||||
const drive = toDrive(value);
|
||||
if (drive) drives.push(drive);
|
||||
}
|
||||
return drives;
|
||||
}
|
||||
|
||||
/** One space, or null when it is gone, which is what makes `scope_missing` detectable. */
|
||||
async getDrive(creds: WebdavCreds, driveId: string): Promise<WebdavDrive | null> {
|
||||
try {
|
||||
return toDrive(await this.graph(creds, 'GET', `/graph/v1.0/drives/${encodeURIComponent(driveId)}`));
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof WebdavError && err.status === 404) return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async createDrive(creds: WebdavCreds, name: string): Promise<WebdavDrive> {
|
||||
const drive = toDrive(
|
||||
await this.graph(creds, 'POST', '/graph/v1.0/drives', { name, driveType: 'project' }),
|
||||
);
|
||||
if (!drive) {
|
||||
throw new WebdavError('provider_error', 'OpenCloud created a space it then described incompletely');
|
||||
}
|
||||
return drive;
|
||||
}
|
||||
|
||||
/** The account's own name, for the line the settings screen shows after a test. */
|
||||
async whoAmI(creds: WebdavCreds): Promise<string> {
|
||||
const me = asRecord(await this.graph(creds, 'GET', '/graph/v1.0/me'));
|
||||
return (
|
||||
textOf(me?.onPremisesSamAccountName) ??
|
||||
textOf(me?.displayName) ??
|
||||
textOf(me?.mail) ??
|
||||
creds.username
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether this account may manage webhooks at all.
|
||||
*
|
||||
* The endpoint is admin-only and a read is harmless, so asking is cheaper and
|
||||
* more honest than registering four subscriptions to find out. A non-admin
|
||||
* app password gets 403 here, which is exactly the case `webhook-manual`
|
||||
* exists for.
|
||||
*/
|
||||
async canRegisterWebhooks(creds: WebdavCreds): Promise<boolean> {
|
||||
const response = await this.send(creds, OCS_WEBHOOKS, { method: 'GET', headers: OCS_HEADERS });
|
||||
void response.body?.cancel().catch(() => {});
|
||||
return response.ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register one event, unfiltered.
|
||||
*
|
||||
* Form-encoded, because a JSON body arrives at the controller with every
|
||||
* argument null and the instance answers HTTP 500.
|
||||
*
|
||||
* No `eventFilter`, and that is measured rather than lazy. The filter is
|
||||
* evaluated by the app's own bundled PHPMongoQuery, which implements
|
||||
* `$all $e $in $lt $lte $gt $gte $ne $nin $exists $mod` and nothing else, so a
|
||||
* `$regex` prefix match on the changed path throws "Operator $regex is
|
||||
* unknown" inside the event listener. The damage does not stop at this
|
||||
* subscription: the throw aborts the listener for that event, so one filtered
|
||||
* webhook silently stops delivery for every other webhook on the same event,
|
||||
* including ones belonging to other applications. An unfiltered subscription
|
||||
* that occasionally says "look now" about an unrelated folder is the cheap
|
||||
* mistake; the filter is the expensive one.
|
||||
*/
|
||||
async createWebhook(
|
||||
creds: WebdavCreds,
|
||||
options: { uri: string; event: string; secretHeader: string; secret: string },
|
||||
): Promise<number> {
|
||||
const form = new URLSearchParams();
|
||||
form.set('httpMethod', 'POST');
|
||||
form.set('uri', options.uri);
|
||||
form.set('event', options.event);
|
||||
form.set('authMethod', 'header');
|
||||
form.set(`authData[${options.secretHeader}]`, options.secret);
|
||||
|
||||
const response = await this.send(creds, OCS_WEBHOOKS, {
|
||||
method: 'POST',
|
||||
headers: { ...OCS_HEADERS, 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: form.toString(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw await this.failed(response, 'The webhook API');
|
||||
}
|
||||
const parsed = await readCappedJson<unknown>(response, PROVIDER_JSON_MAX_BYTES);
|
||||
const id = Number(textOf(asRecord(asRecord(asRecord(parsed)?.ocs)?.data)?.id));
|
||||
if (!Number.isInteger(id) || id <= 0) {
|
||||
throw new WebdavError('provider_error', 'The webhook API answered without an id', response.status);
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
async deleteWebhook(creds: WebdavCreds, id: number): Promise<void> {
|
||||
const response = await this.send(creds, `${OCS_WEBHOOKS}/${id}`, {
|
||||
method: 'DELETE',
|
||||
headers: OCS_HEADERS,
|
||||
});
|
||||
void response.body?.cancel().catch(() => {});
|
||||
if (!response.ok && response.status !== 404) {
|
||||
throw new WebdavError(
|
||||
classifyStatus(response.status),
|
||||
`Removing the webhook answered HTTP ${response.status}`,
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,862 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type {
|
||||
DocResult,
|
||||
DocumentConnectionRef,
|
||||
DocumentProvider,
|
||||
DocumentProviderCapabilities,
|
||||
DocumentScopeOption,
|
||||
DocumentScopeRef,
|
||||
FetchResult,
|
||||
PushRequest,
|
||||
PushResult,
|
||||
RemoteDocument,
|
||||
} from '../document-provider';
|
||||
import { docFail, docFailed, docOk } from '../document-provider';
|
||||
import {
|
||||
TREK_PROP_DOC_UID,
|
||||
TREK_PROP_TRIP_UID,
|
||||
WebdavClient,
|
||||
WebdavError,
|
||||
davPathOf,
|
||||
encodePath,
|
||||
normalizeOrigin,
|
||||
normalizePath,
|
||||
type WebdavCreds,
|
||||
type WebdavEntry,
|
||||
type WebdavFlavor,
|
||||
} from './webdav.client';
|
||||
|
||||
/**
|
||||
* The document adapter for Nextcloud and OpenCloud.
|
||||
*
|
||||
* One implementation serves both, and which one it is talking to is read off
|
||||
* the connection: Nextcloud brings a login name, an app password and a folder,
|
||||
* OpenCloud a username, an app token and a space. Nothing below branches on the
|
||||
* registered `id`; the two subclasses at the end of this file exist only because
|
||||
* the registry keys providers by `id` and Nest keys them by class, so two
|
||||
* product ids need two tokens. Everything a product does differently lives in
|
||||
* `flavorOf()` and the handful of branches it feeds.
|
||||
*
|
||||
* The scope is a folder (`fileid:<oc:fileid>`) or a space (`drive:<driveId>`).
|
||||
* Both ids survive a rename and a move within the same storage, which is what
|
||||
* lets `stableId` be true and a trip binding survive someone tidying up.
|
||||
*
|
||||
* Deliberately NOT here: a change feed. `sync-collection` (RFC 6578) is not
|
||||
* implemented by either product (Nextcloud answers HTTP 415 with
|
||||
* `ReportNotSupported`), so the listing is a full enumeration, short-circuited
|
||||
* by the root ETag, exactly as `DocumentProvider.list` describes.
|
||||
*/
|
||||
|
||||
/** The four Nextcloud events that mean "something in this folder moved". */
|
||||
const WEBHOOK_EVENTS = [
|
||||
'OCP\\Files\\Events\\Node\\NodeCreatedEvent',
|
||||
'OCP\\Files\\Events\\Node\\NodeWrittenEvent',
|
||||
'OCP\\Files\\Events\\Node\\NodeDeletedEvent',
|
||||
'OCP\\Files\\Events\\Node\\NodeRenamedEvent',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Header the subscription carries the shared secret in.
|
||||
*
|
||||
* It has to be the one the endpoint reads: `doc-sync-webhook.controller.ts`
|
||||
* looks for `x-trek-docsync-secret`. It used to say `X-TREK-Docsync-Signature`
|
||||
* here, so every webhook Nextcloud actually sent arrived without a secret the
|
||||
* controller could find and was silently dropped: the subscription existed, the
|
||||
* calls arrived, and nothing ever came of them.
|
||||
*/
|
||||
const WEBHOOK_SECRET_HEADER = 'x-trek-docsync-secret';
|
||||
|
||||
/** Where a Nextcloud connection looks for trip folders when nobody said otherwise. */
|
||||
const DEFAULT_BASE_PATH = '/TREK';
|
||||
|
||||
/** Enough scopes for a busy install, and a ceiling so the cache cannot grow forever. */
|
||||
const MAX_CACHED_SCOPES = 200;
|
||||
|
||||
/**
|
||||
* A document whose entry carried no `oc:fileid`. Both products always send one,
|
||||
* but a reverse proxy stripping the `oc:` namespace would otherwise drop the
|
||||
* file from the trip silently; a path-shaped id keeps it visible at the cost of
|
||||
* looking renamed if someone renames it.
|
||||
*/
|
||||
const PATH_ID_PREFIX = 'path:';
|
||||
|
||||
/** Everything the adapter needs to address one bound scope. */
|
||||
interface ResolvedScope {
|
||||
creds: WebdavCreds;
|
||||
/** DAV path of the scope root, percent-encoded, on the connection's own origin. */
|
||||
rootPath: string;
|
||||
/** The same path decoded, for deriving a document's path inside the scope. */
|
||||
rootDecoded: string;
|
||||
/** The id inside the scope key: an `oc:fileid` or a `driveId`. */
|
||||
scopeId: string;
|
||||
}
|
||||
|
||||
function textSetting(values: Readonly<Record<string, string>>, key: string): string {
|
||||
const value = values[key];
|
||||
return typeof value === 'string' ? value.trim() : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Which product this connection describes.
|
||||
*
|
||||
* Read off the field set the admin filled in rather than off the provider id,
|
||||
* so a connection cannot end up being addressed as the product it is not
|
||||
* because two rows in `document_providers` were swapped.
|
||||
*/
|
||||
export function flavorOf(conn: DocumentConnectionRef): WebdavFlavor | null {
|
||||
if (textSetting(conn.settings, 'login_name') || textSetting(conn.secrets, 'app_password')) {
|
||||
return 'nextcloud';
|
||||
}
|
||||
if (textSetting(conn.settings, 'username') || textSetting(conn.secrets, 'app_token')) {
|
||||
return 'opencloud';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function credsOf(conn: DocumentConnectionRef): WebdavCreds | null {
|
||||
const flavor = flavorOf(conn);
|
||||
const origin = normalizeOrigin(conn.baseUrl);
|
||||
if (!flavor || !/^https?:\/\/[^/]+$/i.test(origin)) return null;
|
||||
|
||||
const username =
|
||||
flavor === 'nextcloud'
|
||||
? textSetting(conn.settings, 'login_name')
|
||||
: textSetting(conn.settings, 'username');
|
||||
const password =
|
||||
flavor === 'nextcloud'
|
||||
? textSetting(conn.secrets, 'app_password')
|
||||
: textSetting(conn.secrets, 'app_token');
|
||||
if (!username || !password) return null;
|
||||
|
||||
return { origin, username, password, allowInsecureTls: conn.allowInsecureTls, flavor };
|
||||
}
|
||||
|
||||
/** `/remote.php/dav/files/<login>`: the per-account root of Nextcloud's file DAV. */
|
||||
function filesRoot(creds: WebdavCreds): string {
|
||||
return `/remote.php/dav/files/${encodeURIComponent(creds.username)}`;
|
||||
}
|
||||
|
||||
function basePathOf(conn: DocumentConnectionRef): string {
|
||||
const configured = textSetting(conn.settings, 'base_path');
|
||||
return normalizePath(configured || DEFAULT_BASE_PATH);
|
||||
}
|
||||
|
||||
/** `fileid:437` / `drive:storage$space` → the id, for the kind this flavor uses. */
|
||||
function scopeIdOf(scopeKey: string, flavor: WebdavFlavor): string | null {
|
||||
const expected = flavor === 'nextcloud' ? 'fileid:' : 'drive:';
|
||||
if (!scopeKey.startsWith(expected)) return null;
|
||||
const id = scopeKey.slice(expected.length).trim();
|
||||
return id.length > 0 ? id : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Names both products accept.
|
||||
*
|
||||
* Nextcloud refuses `\ / < > : " | ? *` and a trailing dot or space outright, so
|
||||
* a trip called "Japan: 2026" would fail to get a folder at all. The replacement
|
||||
* is visible: the label in the returned option is the name that was really
|
||||
* used, not the one that was asked for.
|
||||
*/
|
||||
function sanitizeName(name: string): string {
|
||||
const cleaned = name
|
||||
// eslint-disable-next-line no-control-regex
|
||||
.replace(/[\\/<>:"|?*\u0000-\u001f]/g, '-')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.replace(/[. ]+$/, '');
|
||||
return cleaned.slice(0, 200);
|
||||
}
|
||||
|
||||
function parentOf(encodedPath: string): string {
|
||||
const cut = encodedPath.lastIndexOf('/');
|
||||
return cut > 0 ? encodedPath.slice(0, cut) : '';
|
||||
}
|
||||
|
||||
/** The document's path inside its scope, decoded, with no leading slash. */
|
||||
function relativeOf(entry: WebdavEntry, rootDecoded: string): string {
|
||||
return entry.path.startsWith(`${rootDecoded}/`)
|
||||
? entry.path.slice(rootDecoded.length + 1)
|
||||
: entry.name;
|
||||
}
|
||||
|
||||
function documentOf(entry: WebdavEntry, rootDecoded: string): RemoteDocument {
|
||||
const relative = relativeOf(entry, rootDecoded);
|
||||
return {
|
||||
remoteId: entry.fileId ?? `${PATH_ID_PREFIX}${relative}`,
|
||||
name: entry.name,
|
||||
size: entry.size,
|
||||
mimeType: entry.mimeType,
|
||||
// An ETag is what both products change on every write. When one is missing
|
||||
// the pair below still differs after a write, which is all the core reads
|
||||
// out of this field.
|
||||
remoteVersion: entry.etag || `mtime:${entry.lastModifiedIso ?? ''}:${entry.size ?? ''}`,
|
||||
contentHash: entry.sha256,
|
||||
remoteModifiedAt: entry.lastModifiedIso,
|
||||
// WebDAV has no tombstone in a listing: a deleted file is simply absent, and
|
||||
// the core's own mass-delete guard is what stands between that and a trip
|
||||
// emptying itself.
|
||||
isDeleted: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Anything that goes wrong, as a result rather than an exception.
|
||||
*
|
||||
* `probe` and its siblings always resolve because the settings screen renders
|
||||
* the answer inline. An error that is not a WebdavError is a bug in this file
|
||||
* rather than a provider problem, and it still has to come back as data: one
|
||||
* broken adapter must not take down the run for every other link.
|
||||
*/
|
||||
function failureOf<T>(err: unknown): DocResult<T> {
|
||||
if (err instanceof WebdavError) {
|
||||
return docFail<T>(err.code, err.detail ?? err.message, err.status);
|
||||
}
|
||||
return docFail<T>('unknown', err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class WebdavDocumentProvider implements DocumentProvider {
|
||||
readonly id: string = 'nextcloud';
|
||||
|
||||
/**
|
||||
* Scope key → (remote id → still-encoded href), refilled by every listing.
|
||||
*
|
||||
* WebDAV addresses resources by path while the core hands back the stable id
|
||||
* it stored, and neither product can PROPFIND a Nextcloud file id directly.
|
||||
* Without this, every download, rename and trash would re-walk the folder
|
||||
* first. It is a cache, not state: a miss costs one PROPFIND, never a wrong
|
||||
* answer.
|
||||
*/
|
||||
private readonly hrefCache = new Map<string, Map<string, string>>();
|
||||
|
||||
constructor(private readonly client: WebdavClient) {}
|
||||
|
||||
capabilities(conn: DocumentConnectionRef): DocumentProviderCapabilities {
|
||||
return {
|
||||
// Conservative on purpose: most Nextcloud connections are an ordinary
|
||||
// account's app password, and `webhook_listeners` is admin-only. `probe`
|
||||
// upgrades this to `webhook-self-registered` once it has asked the
|
||||
// instance. OpenCloud has no HTTP subscription API at all (its change
|
||||
// events live on an internal NATS bus that no trip admin can point at
|
||||
// TREK), so it is `none` rather than a `webhook-manual` that would send
|
||||
// someone looking for a settings page that does not exist.
|
||||
push: flavorOf(conn) === 'nextcloud' ? 'webhook-manual' : 'none',
|
||||
stableId: true,
|
||||
remoteTrash: true,
|
||||
replaceInPlace: true,
|
||||
// False for both: Nextcloud stores whatever checksum the uploader claimed
|
||||
// without verifying it, and OpenCloud computes SHA1/MD5/ADLER32 but never
|
||||
// SHA256. See the header of webdav.client.ts.
|
||||
contentHashInListing: false,
|
||||
maxUploadBytes: null,
|
||||
acceptedMimeTypes: null,
|
||||
canCreateScope: true,
|
||||
};
|
||||
}
|
||||
|
||||
async probe(
|
||||
conn: DocumentConnectionRef,
|
||||
): Promise<DocResult<{ account: string; capabilities: DocumentProviderCapabilities }>> {
|
||||
const creds = credsOf(conn);
|
||||
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
|
||||
|
||||
try {
|
||||
const capabilities = this.capabilities(conn);
|
||||
if (creds.flavor === 'opencloud') {
|
||||
const account = await this.client.whoAmI(creds);
|
||||
return docOk({ account, capabilities });
|
||||
}
|
||||
|
||||
// The WebDAV root rather than an OCS endpoint: it is the surface every
|
||||
// other call uses, and a login name that does not match the account
|
||||
// answers 404 here while the password checks out fine.
|
||||
try {
|
||||
await this.client.propfind(creds, filesRoot(creds), 0);
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof WebdavError && err.status === 404) {
|
||||
return docFail(
|
||||
'not_found',
|
||||
`No WebDAV home for login name "${creds.username}". Nextcloud shows the exact spelling under Settings → Personal → Security.`,
|
||||
404,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const canRegister = await this.client.canRegisterWebhooks(creds);
|
||||
return docOk({
|
||||
account: creds.username,
|
||||
capabilities: {
|
||||
...capabilities,
|
||||
push: canRegister ? 'webhook-self-registered' : 'webhook-manual',
|
||||
},
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async listScopes(conn: DocumentConnectionRef, query?: string): Promise<DocResult<DocumentScopeOption[]>> {
|
||||
const creds = credsOf(conn);
|
||||
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
|
||||
const needle = query?.trim().toLowerCase() ?? '';
|
||||
|
||||
try {
|
||||
const options =
|
||||
creds.flavor === 'nextcloud'
|
||||
? await this.nextcloudScopes(conn, creds)
|
||||
: await this.opencloudScopes(creds);
|
||||
return docOk(needle ? options.filter((option) => option.label.toLowerCase().includes(needle)) : options);
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
private async nextcloudScopes(
|
||||
conn: DocumentConnectionRef,
|
||||
creds: WebdavCreds,
|
||||
): Promise<DocumentScopeOption[]> {
|
||||
const basePath = basePathOf(conn);
|
||||
let listing;
|
||||
try {
|
||||
listing = await this.client.propfind(creds, `${filesRoot(creds)}${encodePath(basePath)}`, 1);
|
||||
} catch (err: unknown) {
|
||||
// The base folder does not exist yet on a fresh instance. An empty picker
|
||||
// plus a working "create" is a better answer than an error that reads
|
||||
// like the credentials are wrong.
|
||||
if (err instanceof WebdavError && err.status === 404) return [];
|
||||
throw err;
|
||||
}
|
||||
|
||||
const options: DocumentScopeOption[] = [];
|
||||
for (const child of listing.children) {
|
||||
if (!child.isCollection || !child.fileId) continue;
|
||||
options.push({
|
||||
scopeKey: `fileid:${child.fileId}`,
|
||||
label: child.name,
|
||||
remoteRootId: child.fileId,
|
||||
remoteRootPath: `${basePath}/${child.name}`,
|
||||
});
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
private async opencloudScopes(creds: WebdavCreds): Promise<DocumentScopeOption[]> {
|
||||
const drives = await this.client.listDrives(creds);
|
||||
return drives
|
||||
// `virtual` is the synthetic "Shares" drive, which has no storage of its
|
||||
// own and cannot hold an upload.
|
||||
.filter((drive) => drive.driveType !== 'virtual')
|
||||
.map((drive) => ({
|
||||
scopeKey: `drive:${drive.id}`,
|
||||
label: drive.name,
|
||||
remoteRootId: drive.id,
|
||||
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
|
||||
}));
|
||||
}
|
||||
|
||||
async createScope(conn: DocumentConnectionRef, name: string): Promise<DocResult<DocumentScopeOption>> {
|
||||
const creds = credsOf(conn);
|
||||
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
|
||||
const safeName = sanitizeName(name);
|
||||
if (!safeName) return docFail('provider_error', 'The name has no characters the provider accepts');
|
||||
|
||||
try {
|
||||
if (creds.flavor === 'opencloud') {
|
||||
const drive = await this.client.createDrive(creds, safeName);
|
||||
return docOk({
|
||||
scopeKey: `drive:${drive.id}`,
|
||||
label: drive.name,
|
||||
remoteRootId: drive.id,
|
||||
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
|
||||
});
|
||||
}
|
||||
|
||||
const basePath = basePathOf(conn);
|
||||
const path = `${filesRoot(creds)}${encodePath(`${basePath}/${safeName}`)}`;
|
||||
let created: 'created' | 'exists';
|
||||
try {
|
||||
created = await this.client.mkcol(creds, path);
|
||||
} catch (err: unknown) {
|
||||
// 409 is Sabre's "parent node does not exist", i.e. the base folder has
|
||||
// never been created. Making it is the whole point of this call.
|
||||
if (!(err instanceof WebdavError) || err.status !== 409) throw err;
|
||||
await this.client.mkcol(creds, `${filesRoot(creds)}${encodePath(basePath)}`);
|
||||
created = await this.client.mkcol(creds, path);
|
||||
}
|
||||
if (created === 'exists') {
|
||||
return docFail(
|
||||
'conflict',
|
||||
`"${safeName}" already exists under ${basePath}. Pick it in the list instead of creating it again.`,
|
||||
405,
|
||||
);
|
||||
}
|
||||
|
||||
// MKCOL answers an OC-FileId header, but only on Nextcloud and only in the
|
||||
// padded spelling; one PROPFIND gets the id in the form every listing
|
||||
// will use, which is the form the scope key has to carry.
|
||||
const folder = await this.client.propfind(creds, path, 0);
|
||||
if (!folder.self.fileId) {
|
||||
return docFail('provider_error', 'The new folder came back without an oc:fileid');
|
||||
}
|
||||
return docOk({
|
||||
scopeKey: `fileid:${folder.self.fileId}`,
|
||||
label: safeName,
|
||||
remoteRootId: folder.self.fileId,
|
||||
remoteRootPath: `${basePath}/${safeName}`,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async resolveScope(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<DocumentScopeOption>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
const { creds, rootPath, scopeId } = resolved.data;
|
||||
|
||||
try {
|
||||
if (creds.flavor === 'opencloud') {
|
||||
const drive = await this.client.getDrive(creds, scopeId);
|
||||
if (!drive) return docFail('scope_missing', `The space ${scopeId} no longer exists`);
|
||||
return docOk({
|
||||
scopeKey: `drive:${drive.id}`,
|
||||
label: drive.name,
|
||||
remoteRootId: drive.id,
|
||||
remoteRootPath: `${creds.origin}${drive.webDavPath}`,
|
||||
});
|
||||
}
|
||||
|
||||
const folder = await this.client.propfind(creds, rootPath, 0);
|
||||
// The path still resolves but the id behind it changed: someone deleted
|
||||
// the folder and made a new one with the same name, and its contents have
|
||||
// nothing to do with this trip.
|
||||
if (folder.self.fileId !== scopeId) {
|
||||
return docFail(
|
||||
'scope_missing',
|
||||
`${scope.remoteRootPath ?? rootPath} is a different folder now (oc:fileid ${folder.self.fileId ?? 'missing'} instead of ${scopeId})`,
|
||||
);
|
||||
}
|
||||
return docOk({
|
||||
scopeKey: `fileid:${scopeId}`,
|
||||
label: folder.self.name,
|
||||
remoteRootId: scopeId,
|
||||
remoteRootPath: scope.remoteRootPath,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof WebdavError && err.status === 404) {
|
||||
return docFail('scope_missing', err.detail ?? err.message, 404);
|
||||
}
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): Promise<DocResult<{
|
||||
documents: RemoteDocument[];
|
||||
cursor: string | null;
|
||||
cursorUnchanged: boolean;
|
||||
truncated: boolean;
|
||||
}>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
const { creds, rootPath, rootDecoded } = resolved.data;
|
||||
|
||||
try {
|
||||
// A write anywhere below a collection changes that collection's ETag up to
|
||||
// the root on both products, so one Depth:0 request answers "has anything
|
||||
// at all happened here" for a folder of any size.
|
||||
if (scope.cursor) {
|
||||
const head = await this.client.propfind(creds, rootPath, 0);
|
||||
if (head.self.etag && head.self.etag === scope.cursor) {
|
||||
return docOk({ documents: [], cursor: scope.cursor, cursorUnchanged: true, truncated: false });
|
||||
}
|
||||
}
|
||||
|
||||
const listing = await this.client.propfind(creds, rootPath, 1);
|
||||
const documents: RemoteDocument[] = [];
|
||||
const hrefs = new Map<string, string>();
|
||||
for (const child of listing.children) {
|
||||
// Sub-folders are not documents and are not descended into: Depth:1 is
|
||||
// one request per run whatever the folder holds, and a recursive walk
|
||||
// would trade that for one request per directory.
|
||||
if (child.isCollection) continue;
|
||||
const document = documentOf(child, rootDecoded);
|
||||
documents.push(document);
|
||||
hrefs.set(document.remoteId, child.href);
|
||||
}
|
||||
this.rememberHrefs(conn, scope, hrefs);
|
||||
|
||||
return docOk({
|
||||
documents,
|
||||
cursor: listing.self.etag || null,
|
||||
cursorUnchanged: false,
|
||||
truncated: listing.truncated,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof WebdavError && err.status === 404) {
|
||||
return docFail('scope_missing', err.detail ?? err.message, 404);
|
||||
}
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async fetch(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<FetchResult>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
|
||||
try {
|
||||
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
|
||||
if (!href) return docFail('not_found', `${remoteId} is no longer in this folder`);
|
||||
const download = await this.client.get(resolved.data.creds, href);
|
||||
return docOk({
|
||||
body: download.body,
|
||||
size: download.size,
|
||||
mimeType: download.mimeType,
|
||||
remoteVersion: download.etag,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async push(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
req: PushRequest,
|
||||
): Promise<DocResult<PushResult>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
const { creds, rootPath } = resolved.data;
|
||||
|
||||
const name = sanitizeName(req.fileName);
|
||||
if (!name) return docFail('provider_error', 'The file name has no characters the provider accepts');
|
||||
|
||||
try {
|
||||
let target: string;
|
||||
if (req.remoteId) {
|
||||
const href = await this.hrefFor(conn, scope, resolved.data, req.remoteId);
|
||||
// Replacing something that is gone is not the same as creating it: the
|
||||
// core records the document as missing and asks a human, rather than
|
||||
// resurrecting a file somebody deleted upstream on purpose.
|
||||
if (!href) return docFail('not_found', `${req.remoteId} is no longer in this folder`);
|
||||
target = href;
|
||||
} else {
|
||||
target = `${rootPath}/${encodeURIComponent(name)}`;
|
||||
}
|
||||
|
||||
const result = await this.client.put(creds, target, req.body, {
|
||||
size: req.size,
|
||||
mimeType: req.mimeType,
|
||||
// Without this the file TREK just uploaded looks freshly changed
|
||||
// upstream on the next run, and the two sides push it back and forth.
|
||||
mtimeSeconds: req.mtimeSeconds,
|
||||
sha256: req.sha256,
|
||||
ifMatch: req.expectedRemoteVersion,
|
||||
// Creating means creating: an unconditional PUT to a name that is
|
||||
// already taken overwrites somebody else's document without a word.
|
||||
// Two trips bound to one folder, or a person who put a `receipt.pdf`
|
||||
// there by hand, are enough for that to happen.
|
||||
ifNoneMatch: req.remoteId ? undefined : '*',
|
||||
});
|
||||
|
||||
// Best effort, and deliberately not fatal: the bytes are already stored,
|
||||
// so failing here would make the core upload them again on every run. The
|
||||
// anchors are a recovery aid, not part of the document's identity.
|
||||
try {
|
||||
await this.client.proppatch(creds, target, {
|
||||
[TREK_PROP_DOC_UID]: req.trekDocUid,
|
||||
[TREK_PROP_TRIP_UID]: req.trekTripUid,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (!(err instanceof WebdavError)) throw err;
|
||||
}
|
||||
|
||||
let { fileId, etag } = result;
|
||||
// A reverse proxy that drops the OC-* headers leaves the upload done and
|
||||
// unidentifiable, so ask for what it swallowed.
|
||||
if (!fileId || !etag) {
|
||||
const stored = await this.client.propfind(creds, target, 0);
|
||||
fileId ??= stored.self.fileId;
|
||||
etag ||= stored.self.etag;
|
||||
}
|
||||
if (!fileId || !etag) {
|
||||
return docFail('provider_error', 'The upload succeeded but the instance never named the stored file');
|
||||
}
|
||||
|
||||
this.rememberHrefs(conn, scope, new Map([[fileId, target]]));
|
||||
return docOk({
|
||||
remoteId: fileId,
|
||||
remoteVersion: etag,
|
||||
remoteModifiedAt: new Date(req.mtimeSeconds * 1000).toISOString(),
|
||||
// Neither product de-duplicates by content; a second upload of the same
|
||||
// bytes is a second file.
|
||||
deduplicated: false,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async rename(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
name: string,
|
||||
): Promise<DocResult<{ remoteVersion: string }>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
const safeName = sanitizeName(name);
|
||||
if (!safeName) return docFail('provider_error', 'The new name has no characters the provider accepts');
|
||||
|
||||
try {
|
||||
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
|
||||
if (!href) return docFail('not_found', `${remoteId} is no longer in this folder`);
|
||||
|
||||
// Within the same parent: a rename moves a document inside the trip's
|
||||
// folder, it never moves it out of the scope the trip is bound to.
|
||||
const destination = `${parentOf(href)}/${encodeURIComponent(safeName)}`;
|
||||
const moved = await this.client.move(resolved.data.creds, href, destination);
|
||||
this.rememberHrefs(conn, scope, new Map([[remoteId, destination]]));
|
||||
|
||||
if (moved.etag) return docOk({ remoteVersion: moved.etag });
|
||||
const renamed = await this.client.propfind(resolved.data.creds, destination, 0);
|
||||
return docOk({ remoteVersion: renamed.self.etag });
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async trash(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
remoteId: string,
|
||||
): Promise<DocResult<void>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
|
||||
try {
|
||||
const href = await this.hrefFor(conn, scope, resolved.data, remoteId);
|
||||
// Already gone is the state the caller asked for.
|
||||
if (!href) return docOk(undefined);
|
||||
await this.client.delete(resolved.data.creds, href);
|
||||
this.forgetHref(conn, scope, remoteId);
|
||||
return docOk(undefined);
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to the four file events, one subscription each.
|
||||
*
|
||||
* A failure here is never a broken connection: the endpoint is admin-only, so
|
||||
* the ordinary case of a non-admin app password is a plain "no", the link
|
||||
* stays on polling, and `probe` already reported `webhook-manual` for exactly
|
||||
* this account. Partial registrations are rolled back so a retry does not
|
||||
* leave a pile of orphaned subscriptions behind.
|
||||
*
|
||||
* Two things a self-hoster has to know, neither of which this call can fix:
|
||||
* Nextcloud delivers webhooks through its background-job queue, so the
|
||||
* notification is as prompt as that instance's cron and no prompter, and
|
||||
* `allow_local_remote_servers` must be on or a callback to a LAN address is
|
||||
* refused at delivery time while the subscription still looks healthy.
|
||||
*/
|
||||
async registerWebhook(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
callbackUrl: string,
|
||||
secret: string,
|
||||
): Promise<DocResult<{ subscriptionId: string }>> {
|
||||
const resolved = this.resolve(conn, scope);
|
||||
if (docFailed(resolved)) return resolved;
|
||||
const { creds } = resolved.data;
|
||||
if (creds.flavor === 'opencloud') {
|
||||
return docFail('forbidden', 'OpenCloud has no webhook API a trip admin can subscribe to');
|
||||
}
|
||||
|
||||
const created: number[] = [];
|
||||
try {
|
||||
if (!(await this.client.canRegisterWebhooks(creds))) {
|
||||
return docFail(
|
||||
'forbidden',
|
||||
'This Nextcloud account may not manage webhooks, so the link keeps polling instead',
|
||||
403,
|
||||
);
|
||||
}
|
||||
// Deliberately unscoped: the obvious `eventFilter` on the changed path
|
||||
// needs `$regex`, which the app's query engine does not implement, and the
|
||||
// exception it throws stops delivery for every other webhook on the same
|
||||
// event. A subscription that says "look now" about a folder this trip does
|
||||
// not care about costs one listing; the filter costs the whole channel.
|
||||
for (const event of WEBHOOK_EVENTS) {
|
||||
created.push(
|
||||
await this.client.createWebhook(creds, {
|
||||
uri: callbackUrl,
|
||||
event,
|
||||
secretHeader: WEBHOOK_SECRET_HEADER,
|
||||
secret,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return docOk({ subscriptionId: created.join(',') });
|
||||
} catch (err: unknown) {
|
||||
for (const id of created) {
|
||||
await this.client.deleteWebhook(creds, id).catch(() => undefined);
|
||||
}
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
async unregisterWebhook(conn: DocumentConnectionRef, subscriptionId: string): Promise<DocResult<void>> {
|
||||
const creds = credsOf(conn);
|
||||
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
|
||||
|
||||
try {
|
||||
for (const part of subscriptionId.split(',')) {
|
||||
const id = Number(part.trim());
|
||||
if (Number.isInteger(id) && id > 0) {
|
||||
await this.client.deleteWebhook(creds, id);
|
||||
}
|
||||
}
|
||||
return docOk(undefined);
|
||||
} catch (err: unknown) {
|
||||
return failureOf(err);
|
||||
}
|
||||
}
|
||||
|
||||
/** Connection plus scope key turned into an addressable root, or the reason it is not one. */
|
||||
private resolve(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
): DocResult<ResolvedScope> {
|
||||
const creds = credsOf(conn);
|
||||
if (!creds) return docFail('unauthorized', 'The connection is missing a URL, a username or a password');
|
||||
|
||||
const scopeId = scopeIdOf(scope.scopeKey, creds.flavor);
|
||||
if (!scopeId) {
|
||||
return docFail('scope_missing', `"${scope.scopeKey}" is not a scope key this provider issued`);
|
||||
}
|
||||
|
||||
if (creds.flavor === 'nextcloud') {
|
||||
const userPath = scope.remoteRootPath ? normalizePath(scope.remoteRootPath) : null;
|
||||
if (!userPath || userPath === '/') {
|
||||
return docFail('scope_missing', 'The binding has no folder path, and a file id cannot be addressed over WebDAV');
|
||||
}
|
||||
const rootPath = `${filesRoot(creds)}${encodePath(userPath)}`;
|
||||
return docOk({ creds, rootPath, rootDecoded: `${filesRoot(creds)}${userPath}`, scopeId });
|
||||
}
|
||||
|
||||
// The stored root is the space's own webDavUrl. Only its path is used, and
|
||||
// it is re-based on the connection's origin: the URL came out of a provider
|
||||
// response, and following the host in it would hand an attacker who can
|
||||
// answer for that instance a way past the SSRF guard.
|
||||
const path = scope.remoteRootPath ? davPathOf(scope.remoteRootPath) : null;
|
||||
const rootPath = path ?? `/dav/spaces/${encodeURIComponent(scopeId)}`;
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = normalizePath(decodeURIComponent(rootPath));
|
||||
} catch {
|
||||
// A stored path with a stray percent sign is still usable as-is; failing
|
||||
// the whole binding over a decoding quirk would be worse than the quirk.
|
||||
decoded = rootPath;
|
||||
}
|
||||
return docOk({ creds, rootPath, rootDecoded: decoded, scopeId });
|
||||
}
|
||||
|
||||
private cacheKey(conn: DocumentConnectionRef, scope: DocumentScopeRef): string {
|
||||
return `${conn.connectionId}:${scope.scopeKey}`;
|
||||
}
|
||||
|
||||
private rememberHrefs(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
hrefs: Map<string, string>,
|
||||
): void {
|
||||
const key = this.cacheKey(conn, scope);
|
||||
const existing = this.hrefCache.get(key);
|
||||
if (existing) {
|
||||
for (const [id, href] of hrefs) existing.set(id, href);
|
||||
return;
|
||||
}
|
||||
if (this.hrefCache.size >= MAX_CACHED_SCOPES) {
|
||||
// Map iterates in insertion order, so the first key is the oldest scope.
|
||||
const oldest = this.hrefCache.keys().next();
|
||||
if (!oldest.done) this.hrefCache.delete(oldest.value);
|
||||
}
|
||||
this.hrefCache.set(key, new Map(hrefs));
|
||||
}
|
||||
|
||||
private forgetHref(conn: DocumentConnectionRef, scope: DocumentScopeRef, remoteId: string): void {
|
||||
this.hrefCache.get(this.cacheKey(conn, scope))?.delete(remoteId);
|
||||
}
|
||||
|
||||
/** The href for a remote id, re-walking the folder once when the cache cannot say. */
|
||||
private async hrefFor(
|
||||
conn: DocumentConnectionRef,
|
||||
scope: DocumentScopeRef,
|
||||
resolved: ResolvedScope,
|
||||
remoteId: string,
|
||||
): Promise<string | null> {
|
||||
if (remoteId.startsWith(PATH_ID_PREFIX)) {
|
||||
const relative = remoteId.slice(PATH_ID_PREFIX.length);
|
||||
// The path comes out of a listing, which is the provider's word and not
|
||||
// TREK's. A `..` in it would address a file outside the folder the trip is
|
||||
// bound to. Every other id here is opaque, this one is a path and has to
|
||||
// be treated like one.
|
||||
if (!relative || relative.split('/').some(seg => seg === '..' || seg === '.')) return null;
|
||||
return `${resolved.rootPath}/${encodePath(relative)}`;
|
||||
}
|
||||
|
||||
const cached = this.hrefCache.get(this.cacheKey(conn, scope))?.get(remoteId);
|
||||
if (cached) return cached;
|
||||
|
||||
const listing = await this.client.propfind(resolved.creds, resolved.rootPath, 1);
|
||||
const hrefs = new Map<string, string>();
|
||||
for (const child of listing.children) {
|
||||
if (child.isCollection) continue;
|
||||
hrefs.set(child.fileId ?? `${PATH_ID_PREFIX}${relativeOf(child, resolved.rootDecoded)}`, child.href);
|
||||
}
|
||||
this.rememberHrefs(conn, scope, hrefs);
|
||||
return hrefs.get(remoteId) ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The two registrations.
|
||||
*
|
||||
* They carry an id and nothing else. The registry maps a connection's
|
||||
* `provider_id` onto a provider instance and Nest maps a class onto a token, so
|
||||
* two product ids need two classes, but both run the implementation above, and
|
||||
* that implementation never asks which one it is.
|
||||
*
|
||||
* The constructors are spelled out rather than inherited: Nest reads
|
||||
* `design:paramtypes` off the concrete class, and a subclass without its own
|
||||
* constructor is emitted without that metadata and resolved with no arguments.
|
||||
*/
|
||||
@Injectable()
|
||||
export class NextcloudDocumentProvider extends WebdavDocumentProvider {
|
||||
override readonly id = 'nextcloud';
|
||||
|
||||
constructor(client: WebdavClient) {
|
||||
super(client);
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class OpencloudDocumentProvider extends WebdavDocumentProvider {
|
||||
override readonly id = 'opencloud';
|
||||
|
||||
constructor(client: WebdavClient) {
|
||||
super(client);
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
splitIntoRuns,
|
||||
spillChains,
|
||||
dayWindow,
|
||||
parseClock,
|
||||
effectiveRangeKm,
|
||||
parseAvoid,
|
||||
formatDistance,
|
||||
@@ -34,6 +33,7 @@ interface StoredDay {
|
||||
}
|
||||
interface VisitRow {
|
||||
check_in: string | null;
|
||||
/** The drive does not read these two (#2357): they are the booking as get_roadtrip_context reports it. */
|
||||
check_out: string | null;
|
||||
checkout_day: number | null;
|
||||
id: number;
|
||||
@@ -43,6 +43,7 @@ interface VisitRow {
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
time: string | null;
|
||||
end_time: string | null;
|
||||
duration_minutes: number | null;
|
||||
end_day: number;
|
||||
leg_transport_mode: string | null;
|
||||
@@ -72,7 +73,8 @@ export class RoadtripPlanService {
|
||||
);
|
||||
const visits = this.db.all<VisitRow>(
|
||||
`SELECT a.id, a.day_id, a.place_id, p.name, p.lat, p.lng,
|
||||
COALESCE(a.assignment_time, p.place_time) AS time, p.duration_minutes, a.end_day,
|
||||
COALESCE(a.assignment_time, p.place_time) AS time, COALESCE(a.assignment_end_time, p.end_time) AS end_time,
|
||||
p.duration_minutes, a.end_day,
|
||||
a.leg_transport_mode, a.incoming_leg_transport_mode, p.stop_type, p.fill_percent, stay.check_in, stay.check_out, checkout.day_number AS checkout_day
|
||||
FROM day_assignments a JOIN days d ON d.id = a.day_id JOIN places p ON p.id = a.place_id
|
||||
LEFT JOIN day_accommodations stay ON stay.id = (SELECT id FROM day_accommodations WHERE place_id = p.id AND start_day_id = d.id ORDER BY id LIMIT 1)
|
||||
@@ -117,8 +119,10 @@ export class RoadtripPlanService {
|
||||
lat: v.lat!,
|
||||
lng: v.lng!,
|
||||
time: v.time,
|
||||
// The visit's end time is when the drive leaves it, the same reading the
|
||||
// planner makes in the browser (useRoadtripRoutes).
|
||||
leaveAt: v.end_time,
|
||||
checkInTime: v.check_in,
|
||||
...(v.checkout_day != null && parseClock(v.check_out) !== null ? { checkoutAt: v.checkout_day * 1440 + parseClock(v.check_out)! } : {}),
|
||||
dwellMinutes: v.duration_minutes,
|
||||
endDay: v.end_day === 1,
|
||||
legMode: v.leg_transport_mode,
|
||||
|
||||
@@ -27,7 +27,7 @@ export class RoadtripPlanningMcp {
|
||||
@Tool({
|
||||
name: 'get_roadtrip_context',
|
||||
description:
|
||||
'Read the saved roadtrip days, visits, pinned times, stays, stop types, fill levels, travel modes, via points, followed tracks and manual day endings. Includes the shared driving preferences for this trip. No routing request and no browser needed. Coordinates missing from a visit prevent it from being routed. Use calculate_roadtrip to get the derived day layout. Edit visits with the existing place and assignment tools; change their saved order with reorder_day_assignments. Settings, visits and manual boundaries all belong to the shared trip.',
|
||||
'Read the saved roadtrip days, visits, pinned times, end times (when the drive leaves a visit), stays, stop types, fill levels, travel modes, via points, followed tracks and manual day endings. Includes the shared driving preferences for this trip. No routing request and no browser needed. Coordinates missing from a visit prevent it from being routed. Use calculate_roadtrip to get the derived day layout. Edit visits with the existing place and assignment tools; change their saved order with reorder_day_assignments. Settings, visits and manual boundaries all belong to the shared trip.',
|
||||
inputSchema: { tripId: z.number().int().positive() },
|
||||
annotations: TOOL_ANNOTATIONS_READONLY,
|
||||
access: { group: 'trips', mode: 'read' },
|
||||
@@ -40,7 +40,7 @@ export class RoadtripPlanningMcp {
|
||||
@Tool({
|
||||
name: 'calculate_roadtrip',
|
||||
description:
|
||||
'Calculate a saved trip without an open browser, using the same daily scheduling and vehicle-range rules as the planner. Returns derived days, arrivals, departures, driving limits, fuel warnings, automatic pauses and routing failures. Fixed visit times retain priority; no accommodation or place is created. Optional settings are a read-only preview, not saved. Distances are metres, durations seconds, dwell times minutes. A failed leg or conflicting window means the output is incomplete: never present its totals as a complete route. Up to 150 legs; requests are paced for public routing servers and can take time. Existing place/assignment tools edit stops, stays and ordering, update_roadtrip_settings saves preferences, set_assignment_end_day and set_day_boundary set exceptions. Recalculate after changes. Plugin profiles follow installed route-provider permissions.',
|
||||
'Calculate a saved trip without an open browser, using the same daily scheduling and vehicle-range rules as the planner. Returns derived days, arrivals, departures, driving limits, fuel warnings, automatic pauses and routing failures. Fixed visit times retain priority. The end time of a visit is when the drive leaves it, in place of its stay; reaching the visit after it is reported as a missedLeave warning. No accommodation or place is created. Optional settings are a read-only preview, not saved. Distances are metres, durations seconds, dwell times minutes. A failed leg or conflicting window means the output is incomplete: never present its totals as a complete route. Up to 150 legs; requests are paced for public routing servers and can take time. Existing place/assignment tools edit stops, stays and ordering, update_roadtrip_settings saves preferences, set_assignment_end_day and set_day_boundary set exceptions. Recalculate after changes. Plugin profiles follow installed route-provider permissions.',
|
||||
inputSchema: roadtripPlanRequestSchema.shape,
|
||||
annotations: TOOL_ANNOTATIONS_READONLY,
|
||||
access: { group: 'trips', mode: 'read' },
|
||||
|
||||
Reference in New Issue
Block a user