mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
fix(docsync): an idle WebDAV binding read as an emptied folder, and two ids nobody checked
The audit's worst find, reproduced against a live Nextcloud before and after the
fix: the WebDAV adapters answer "nothing changed here" by returning an EMPTY
listing plus `cursorUnchanged`, and nothing read that flag — `grep -rn
cursorUnchanged src/` found it set by four adapters and consulted by none. The
core therefore saw a folder that had lost everything. Run 1 and 2 look fine
because the run's own uploads move the ETag; from run 3 on, every run of an
idle binding ends `partial` with `mass_delete_guard` and six documents reported
missing, forever, until failure_count opens the circuit and the scheduler drops
the binding entirely. That is the ordinary case — a binding nobody is touching.
planReconcile now takes `remoteUnchanged` and plans only what happened in TREK.
Two ids were authorised by trip and never checked against it:
- resolveConflict took an item id on trust, so the owner of any trip could
decide a conflict belonging to somebody else's. Both are plain integers and
nothing compared them.
- The connection test merged the STORED credential into a probe of whatever
baseUrl the caller named — a way to have TREK post another account's API
token at a server of the caller's choosing. Stored secrets are now reused
only for the address they were stored against.
The phone's connect form was wrong in three ways at once, each of which the
desktop had right: it read the address as `baseUrl` while the field is
`base_url` (so every connection it saved carried an empty address), it drew
`allow_insecure_tls` as a text box and always sent false, and it built label
keys as `docsync.provider${cap(label)}` when the label column already holds the
full suffix — `docsync.providerProviderApiKey`, rendered raw on screen. Those
rules are not markup, so they now live in `useConnectForm`, which both shells
use. Its test fixture invented the field shape and so agreed with the broken
sheet; it now mirrors document_provider_fields.
Also: the scheduler read its interval once at bootstrap while this file's own
comment promised a per-tick read, so changing it did nothing until a restart;
and dueLinks ordered by a column that a successful run clears, which left every
binding tied and SQLite answering in insertion order — on a database with 170
bindings the same twenty ran on every tick and nothing created after them was
ever synced automatically.
This commit is contained in:
@@ -3,6 +3,7 @@ import { AlertTriangle, Check, Loader2, ShieldAlert } from 'lucide-react'
|
||||
import Modal from '../../shared/Modal'
|
||||
import ToggleSwitch from '../../Settings/ToggleSwitch'
|
||||
import { useTranslation } from '../../../i18n/TranslationContext'
|
||||
import { useConnectForm } from './useConnectForm'
|
||||
import { Badge } from './DocSyncBits'
|
||||
import { DOCUMENT_PROVIDER_ICONS } from '../../shared/DocumentProviderIcons'
|
||||
import type { DocSyncProvider, useDocSync } from './useDocSync'
|
||||
@@ -30,35 +31,13 @@ export default function DocSyncConnectModal({
|
||||
const existing = sync.connectionFor(provider.id)
|
||||
const Icon = DOCUMENT_PROVIDER_ICONS[provider.id]
|
||||
|
||||
const [values, setValues] = useState<Record<string, string>>({})
|
||||
const [insecure, setInsecure] = useState<boolean | null>(null)
|
||||
const form = useConnectForm(provider, existing, sync)
|
||||
const [verdict, setVerdict] = useState<{ connected: boolean; account?: string; error?: string } | null>(null)
|
||||
|
||||
// Null until touched, so a connection that loads after the dialog opened is
|
||||
// still reflected. Seeding the state at mount showed a stored "allow
|
||||
// self-signed" as off and wrote that back on the next save — a security
|
||||
// switch silently turning itself off is the worst version of this bug.
|
||||
const insecureOn = insecure ?? existing?.allowInsecureTls ?? false
|
||||
|
||||
const baseUrl = values.base_url ?? existing?.baseUrl ?? ''
|
||||
// The address is a column of its own, not a settings entry, so it has to be
|
||||
// read from there — otherwise the field sits empty while the form submits the
|
||||
// stored value behind it.
|
||||
const shown = (key: string) =>
|
||||
values[key] ?? (key === 'base_url' ? existing?.baseUrl : existing?.settings?.[key]) ?? ''
|
||||
const required = provider.fields.filter(f => f.required)
|
||||
const canSubmit = required.every(f =>
|
||||
f.field_key === 'base_url'
|
||||
? baseUrl.trim().length > 0
|
||||
: f.secret
|
||||
? (values[f.field_key] ?? '').length > 0 || !!existing?.secrets?.[f.field_key]
|
||||
: shown(f.field_key).trim().length > 0,
|
||||
)
|
||||
|
||||
const test = async () => setVerdict(await sync.testConnection(provider.id, baseUrl, values, insecureOn))
|
||||
const test = async () => setVerdict(await form.probe())
|
||||
|
||||
const save = async () => {
|
||||
if (await sync.saveConnection(provider.id, baseUrl, values, insecureOn)) onConnected(provider.id)
|
||||
if (await form.save()) onConnected(provider.id)
|
||||
}
|
||||
|
||||
return (
|
||||
@@ -79,7 +58,7 @@ export default function DocSyncConnectModal({
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void test()}
|
||||
disabled={!canSubmit || sync.busy === 'test'}
|
||||
disabled={!form.complete || sync.busy === 'test'}
|
||||
className="rounded-lg border border-edge px-3.5 py-2 text-body text-content-secondary transition-colors hover:bg-surface-hover disabled:opacity-50"
|
||||
>
|
||||
{t('docsync.test')}
|
||||
@@ -87,7 +66,7 @@ export default function DocSyncConnectModal({
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void save()}
|
||||
disabled={!canSubmit || sync.busy === 'save'}
|
||||
disabled={!form.complete || sync.busy === 'save'}
|
||||
className="flex items-center gap-2 rounded-lg bg-accent px-3.5 py-2 text-body font-medium text-accent-text transition-opacity hover:opacity-90 disabled:opacity-50"
|
||||
>
|
||||
{sync.busy === 'save' && <Loader2 size={14} className="animate-spin" />}
|
||||
@@ -100,20 +79,18 @@ export default function DocSyncConnectModal({
|
||||
<div className="space-y-4">
|
||||
<p className="text-caption text-content-muted">{t(`docsync.connect.about.${provider.id}`)}</p>
|
||||
|
||||
{provider.fields.filter(f => f.field_key !== 'allow_insecure_tls').map(f => (
|
||||
{form.fields.map(f => (
|
||||
<Field
|
||||
key={f.field_key}
|
||||
label={t(`docsync.${f.label}`)}
|
||||
hint={f.hint ? t(`docsync.${f.hint}`) : undefined}
|
||||
label={t(form.labelKey(f))}
|
||||
hint={form.hintKey(f) ? t(form.hintKey(f) as string) : undefined}
|
||||
required={f.required}
|
||||
>
|
||||
<input
|
||||
type={f.input_type === 'password' ? 'password' : 'text'}
|
||||
value={f.secret ? (values[f.field_key] ?? '') : shown(f.field_key)}
|
||||
// A stored secret never comes back, so an empty field with a dotted
|
||||
// placeholder means "the one already saved" rather than "blank".
|
||||
placeholder={f.secret && existing?.secrets?.[f.field_key] ? '••••••••' : f.placeholder || ''}
|
||||
onChange={e => setValues({ ...values, [f.field_key]: e.target.value })}
|
||||
value={form.valueOf(f.field_key)}
|
||||
placeholder={form.placeholderOf(f)}
|
||||
onChange={e => form.setValue(f.field_key, e.target.value)}
|
||||
className="w-full rounded-lg border border-edge bg-surface-input px-3 py-2.5 text-body text-content ring-accent transition-shadow placeholder:text-content-faint focus:outline-none focus:ring-2"
|
||||
/>
|
||||
</Field>
|
||||
@@ -127,7 +104,7 @@ export default function DocSyncConnectModal({
|
||||
</span>
|
||||
</span>
|
||||
<span className="shrink-0 pt-0.5">
|
||||
<ToggleSwitch on={insecureOn} onToggle={() => setInsecure(!insecureOn)} />
|
||||
<ToggleSwitch on={form.insecureTls} onToggle={() => form.setInsecureTls(!form.insecureTls)} />
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { useState } from 'react'
|
||||
import type { DocSyncConnection, DocSyncProvider, useDocSync } from './useDocSync'
|
||||
|
||||
/**
|
||||
* The credential form's rules, for both shells.
|
||||
*
|
||||
* Which field carries the address, which one must not be drawn as a text box,
|
||||
* how a label becomes a translation key, what counts as filled in when a secret
|
||||
* is already stored: every one of those is a rule, and every one of them was
|
||||
* wrong in the phone sheet while being right on the desktop. The phone read the
|
||||
* address under `baseUrl` while the server names the field `base_url`, so every
|
||||
* connection it saved carried an empty address; it drew `allow_insecure_tls` as
|
||||
* a text input and always sent false; and it built label keys a different way,
|
||||
* so the labels rendered as raw keys.
|
||||
*
|
||||
* None of that is markup, so none of it belongs in a component.
|
||||
*/
|
||||
|
||||
/** The field that holds the address rather than a credential. */
|
||||
export const BASE_URL_FIELD = 'base_url'
|
||||
|
||||
/** Rendered as a switch beside the fields, never as one of them. */
|
||||
export const INSECURE_TLS_FIELD = 'allow_insecure_tls'
|
||||
|
||||
export interface ConnectForm {
|
||||
/** The fields to draw, in order, minus the ones with their own control. */
|
||||
fields: DocSyncProvider['fields']
|
||||
/** Translation key for a field's label, and for its hint where it has one. */
|
||||
labelKey: (field: DocSyncProvider['fields'][number]) => string
|
||||
hintKey: (field: DocSyncProvider['fields'][number]) => string | undefined
|
||||
/** What to show in a field: the typed value, else what is stored. */
|
||||
valueOf: (fieldKey: string) => string
|
||||
/** A stored secret never comes back, so its box shows dots instead. */
|
||||
placeholderOf: (field: DocSyncProvider['fields'][number]) => string
|
||||
setValue: (fieldKey: string, value: string) => void
|
||||
baseUrl: string
|
||||
credentials: Record<string, string>
|
||||
insecureTls: boolean
|
||||
setInsecureTls: (on: boolean) => void
|
||||
/** Every required field is either filled in or already stored. */
|
||||
complete: boolean
|
||||
probe: () => Promise<{ connected: boolean; account?: string; error?: string }>
|
||||
save: () => Promise<boolean>
|
||||
}
|
||||
|
||||
export function useConnectForm(
|
||||
provider: DocSyncProvider,
|
||||
existing: DocSyncConnection | null,
|
||||
sync: ReturnType<typeof useDocSync>,
|
||||
): ConnectForm {
|
||||
const [values, setValues] = useState<Record<string, string>>({})
|
||||
// Null until touched, so a connection that arrives after the form opened is
|
||||
// still reflected — seeding from `existing` at mount showed a stored "allow
|
||||
// self-signed" as off and wrote that back on the next save.
|
||||
const [insecure, setInsecure] = useState<boolean | null>(null)
|
||||
|
||||
const insecureTls = insecure ?? existing?.allowInsecureTls ?? false
|
||||
|
||||
/**
|
||||
* What a box shows.
|
||||
*
|
||||
* A secret shows only what was typed into it this time, never what is stored:
|
||||
* a stored credential must not travel back to a browser, and a value that
|
||||
* ended up in `settings` by mistake must not become the one place it leaks.
|
||||
* The address lives in its own column rather than in settings, so it is read
|
||||
* from there; everything else comes out of settings.
|
||||
*/
|
||||
const secretKeys = new Set(provider.fields.filter(f => f.secret).map(f => f.field_key))
|
||||
const valueOf = (key: string) => {
|
||||
if (secretKeys.has(key)) return values[key] ?? ''
|
||||
return values[key] ?? (key === BASE_URL_FIELD ? existing?.baseUrl : existing?.settings?.[key]) ?? ''
|
||||
}
|
||||
|
||||
const baseUrl = valueOf(BASE_URL_FIELD)
|
||||
// The address stays in here as well as in its own argument: the server walks
|
||||
// the provider's fields over `credentials` and would otherwise store no
|
||||
// `base_url` setting. It special-cases the key when checking completeness, so
|
||||
// dropping it changes nothing it needs — but it is an established request
|
||||
// shape and this is not the change to alter it in.
|
||||
const credentials = Object.fromEntries(
|
||||
Object.entries(values).filter(([k]) => k !== INSECURE_TLS_FIELD),
|
||||
)
|
||||
|
||||
const fields = provider.fields.filter(f => f.field_key !== INSECURE_TLS_FIELD)
|
||||
|
||||
const complete = fields.every(f => {
|
||||
if (!f.required) return true
|
||||
if (f.secret) return (values[f.field_key] ?? '').length > 0 || !!existing?.secrets?.[f.field_key]
|
||||
return valueOf(f.field_key).trim().length > 0
|
||||
})
|
||||
|
||||
return {
|
||||
fields,
|
||||
// The label and hint columns store a key suffix, not text.
|
||||
labelKey: f => `docsync.${f.label}`,
|
||||
hintKey: f => (f.hint ? `docsync.${f.hint}` : undefined),
|
||||
valueOf,
|
||||
placeholderOf: f =>
|
||||
f.secret && existing?.secrets?.[f.field_key] ? '••••••••' : f.placeholder || '',
|
||||
setValue: (key, value) => setValues(v => ({ ...v, [key]: value })),
|
||||
baseUrl,
|
||||
credentials,
|
||||
insecureTls,
|
||||
setInsecureTls: setInsecure,
|
||||
complete,
|
||||
probe: () => sync.testConnection(provider.id, baseUrl, credentials, insecureTls),
|
||||
save: () => sync.saveConnection(provider.id, baseUrl, credentials, insecureTls),
|
||||
}
|
||||
}
|
||||
@@ -14,9 +14,20 @@ import MDocSyncSheet from './MDocSyncSheet'
|
||||
|
||||
const TRIP_ID = 3
|
||||
|
||||
/**
|
||||
* Shaped like a row of document_provider_fields, which matters more than it looks.
|
||||
*
|
||||
* `field_key` is snake_case and `label` already carries the full key suffix
|
||||
* (`providerApiKey`, not `apiKey`) — see server/src/db/document-provider-seed.ts.
|
||||
* An earlier version of this fixture invented both, which made a sheet that
|
||||
* built label keys the wrong way look correct: the fixture's `apiKey` and the
|
||||
* sheet's `docsync.provider${cap(...)}` happened to meet in the middle, while
|
||||
* the real data produced `docsync.providerProviderApiKey` and rendered the raw
|
||||
* key on screen.
|
||||
*/
|
||||
const field = (over: Partial<DocSyncProvider['fields'][number]>): DocSyncProvider['fields'][number] => ({
|
||||
field_key: 'apiToken',
|
||||
label: 'apiToken',
|
||||
field_key: 'api_token',
|
||||
label: 'providerApiToken',
|
||||
input_type: 'text',
|
||||
placeholder: null,
|
||||
hint: null,
|
||||
@@ -32,7 +43,7 @@ const provider = (over: Partial<DocSyncProvider>): DocSyncProvider => ({
|
||||
icon: 'paperless',
|
||||
available: true,
|
||||
fields: [
|
||||
field({ field_key: 'baseUrl', label: 'url', input_type: 'url', placeholder: 'https://paperless.example', secret: false }),
|
||||
field({ field_key: 'base_url', label: 'providerUrl', input_type: 'url', placeholder: 'https://paperless.example', secret: false }),
|
||||
field({}),
|
||||
],
|
||||
...over,
|
||||
@@ -102,8 +113,8 @@ beforeEach(() => {
|
||||
onClose.mockClear()
|
||||
providers = [
|
||||
provider({}),
|
||||
provider({ id: 'nextcloud', name: 'Nextcloud', fields: [field({ field_key: 'appPassword', label: 'appPassword' })] }),
|
||||
provider({ id: 'papra', name: 'Papra', fields: [field({ field_key: 'apiKey', label: 'apiKey' })] }),
|
||||
provider({ id: 'nextcloud', name: 'Nextcloud', fields: [field({ field_key: 'app_password', label: 'providerAppPassword' })] }),
|
||||
provider({ id: 'papra', name: 'Papra', fields: [field({ field_key: 'api_key', label: 'providerApiKey' })] }),
|
||||
]
|
||||
connections = [connection({}), connection({ id: 6, providerId: 'nextcloud' })]
|
||||
links = [link()]
|
||||
|
||||
@@ -14,6 +14,7 @@ import TrekMark from '../../../../components/shared/TrekMark'
|
||||
import {
|
||||
useDocSync, type DocSyncLink, type DocSyncProvider,
|
||||
} from '../../../../components/Files/docsync/useDocSync'
|
||||
import { useConnectForm } from '../../../../components/Files/docsync/useConnectForm'
|
||||
import { relativeTime } from '../../../../utils/relativeTime'
|
||||
|
||||
/**
|
||||
@@ -344,7 +345,14 @@ function DetailView({
|
||||
)
|
||||
}
|
||||
|
||||
/** Credentials for a store this instance offers but the trip has not used yet. */
|
||||
/**
|
||||
* Credentials for a store this instance offers but the trip has not used yet.
|
||||
*
|
||||
* The rules — which field is the address, which one is a switch, how a label
|
||||
* becomes a key — live in `useConnectForm`, the same one the desktop dialog
|
||||
* uses. Written out here a second time they came out wrong in four separate
|
||||
* ways, including an address this form never actually sent.
|
||||
*/
|
||||
function ConnectView({
|
||||
provider,
|
||||
sync,
|
||||
@@ -355,45 +363,49 @@ function ConnectView({
|
||||
onDone: () => void
|
||||
}) {
|
||||
const { t } = useTranslation()
|
||||
const [values, setValues] = useState<Record<string, string>>({})
|
||||
const form = useConnectForm(provider, sync.connectionFor(provider.id), sync)
|
||||
const [verdict, setVerdict] = useState<{ connected: boolean; account?: string; error?: string } | null>(null)
|
||||
|
||||
const baseUrl = values.baseUrl ?? ''
|
||||
const credentials = Object.fromEntries(Object.entries(values).filter(([k]) => k !== 'baseUrl'))
|
||||
const missing = provider.fields.some(f => f.required && !(values[f.field_key] || '').trim())
|
||||
|
||||
const probe = async () => {
|
||||
setVerdict(await sync.testConnection(provider.id, baseUrl, credentials, false))
|
||||
}
|
||||
|
||||
const save = async () => {
|
||||
if (await sync.saveConnection(provider.id, baseUrl, credentials, false)) onDone()
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mt-3 flex flex-col gap-3">
|
||||
<p className="font-geist text-[0.71875rem] leading-snug text-m-muted">{t(`docsync.connect.about.${provider.id}`)}</p>
|
||||
|
||||
{provider.fields.map(f => (
|
||||
{form.fields.map(f => (
|
||||
<label key={f.field_key} className="flex flex-col gap-[6px]">
|
||||
<span className="font-geist text-[0.6875rem] font-bold text-m-muted">
|
||||
{t(`docsync.provider${cap(f.label)}`)}
|
||||
{t(form.labelKey(f))}
|
||||
{f.required && <span className="text-[color:var(--m-st-danger)]"> *</span>}
|
||||
</span>
|
||||
<input
|
||||
type={f.secret ? 'password' : f.input_type === 'url' ? 'url' : 'text'}
|
||||
value={values[f.field_key] ?? ''}
|
||||
onChange={e => { setValues(v => ({ ...v, [f.field_key]: e.target.value })); setVerdict(null) }}
|
||||
placeholder={f.placeholder ?? ''}
|
||||
type={f.input_type === 'password' ? 'password' : 'text'}
|
||||
value={form.valueOf(f.field_key)}
|
||||
onChange={e => { form.setValue(f.field_key, e.target.value); setVerdict(null) }}
|
||||
placeholder={form.placeholderOf(f)}
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
spellCheck={false}
|
||||
className="h-11 rounded-2xl border border-[color:var(--m-inbr)] bg-[color:var(--m-inner)] px-3 text-[0.8125rem] text-m-ink outline-none"
|
||||
/>
|
||||
{f.hint && <span className="font-geist text-[0.625rem] text-m-faint">{f.hint}</span>}
|
||||
{form.hintKey(f) && (
|
||||
<span className="font-geist text-[0.625rem] text-m-faint">{t(form.hintKey(f) as string)}</span>
|
||||
)}
|
||||
</label>
|
||||
))}
|
||||
|
||||
<div className="flex items-center justify-between gap-3 rounded-2xl border border-[color:var(--m-rowbr)] bg-m-sheetop px-3 py-[9px]">
|
||||
<span className="min-w-0 flex-1">
|
||||
<span className="block text-[0.78125rem] font-semibold text-m-ink">{t('docsync.allowInsecureTls')}</span>
|
||||
<span className="mt-[2px] block font-geist text-[0.625rem] leading-snug text-m-faint">
|
||||
{t('docsync.connect.insecureHint')}
|
||||
</span>
|
||||
</span>
|
||||
<MToggle
|
||||
checked={form.insecureTls}
|
||||
ariaLabel={t('docsync.allowInsecureTls')}
|
||||
onChange={form.setInsecureTls}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{verdict && (
|
||||
<p
|
||||
className={`rounded-2xl px-3 py-2 font-geist text-[0.6875rem] ${
|
||||
@@ -408,19 +420,25 @@ function ConnectView({
|
||||
</p>
|
||||
)}
|
||||
|
||||
{sync.error && !verdict && (
|
||||
<p className="rounded-2xl bg-[color:var(--m-ic)] px-3 py-2 font-geist text-[0.6875rem] text-[color:var(--m-st-danger)]">
|
||||
{t(`docsync.error.${sync.error}`)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void probe()}
|
||||
disabled={missing || sync.busy === 'test'}
|
||||
onClick={() => void form.probe().then(setVerdict)}
|
||||
disabled={!form.complete || sync.busy === 'test'}
|
||||
className="h-11 flex-1 rounded-2xl border border-[color:var(--m-rowbr)] text-[0.8125rem] font-bold text-m-ink disabled:opacity-50"
|
||||
>
|
||||
{sync.busy === 'test' ? <Loader2 size={15} className="mx-auto animate-spin" /> : t('docsync.test')}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void save()}
|
||||
disabled={missing || sync.busy === 'save'}
|
||||
onClick={() => void form.save().then(ok => { if (ok) onDone() })}
|
||||
disabled={!form.complete || sync.busy === 'save'}
|
||||
className="h-11 flex-1 rounded-2xl bg-m-act text-[0.8125rem] font-bold text-m-actfg disabled:opacity-50"
|
||||
>
|
||||
{sync.busy === 'save' ? <Loader2 size={15} className="mx-auto animate-spin" /> : t('common.save')}
|
||||
@@ -701,11 +719,6 @@ function Empty({ text, hint }: { text: string; hint?: string }) {
|
||||
)
|
||||
}
|
||||
|
||||
/** `apiToken` → `ApiToken`, so the field's stored label suffix becomes its key. */
|
||||
function cap(s: string): string {
|
||||
return s.charAt(0).toUpperCase() + s.slice(1)
|
||||
}
|
||||
|
||||
/** A folder name from the trip's own title, so nobody has to invent one. */
|
||||
function slugFor(title: string | undefined, tripId: number | string): string {
|
||||
const base = (title || 'trek')
|
||||
|
||||
@@ -150,7 +150,13 @@ export class DocSyncController {
|
||||
const existing = this.config
|
||||
.listConnections(Number(tripId))
|
||||
.find((c) => c.provider_id === body.providerId);
|
||||
const stored = existing ? this.config.toRef(existing) : null;
|
||||
// Only for the address the credential was stored against. Merging it into a
|
||||
// probe of an arbitrary baseUrl turns this route into a way to have TREK
|
||||
// post a stored API token at a server of the caller's choosing — which is
|
||||
// exactly what somebody who inherited a trip but not its credentials would
|
||||
// reach for. Same host, same scheme, same port, or the caller types it in.
|
||||
const sameTarget = !!existing && sameOrigin(existing.base_url, urlCheck.data.url);
|
||||
const stored = sameTarget && existing ? this.config.toRef(existing) : null;
|
||||
const fields = this.config.providerFields(body.providerId);
|
||||
const secrets: Record<string, string> = { ...(stored?.secrets ?? {}) };
|
||||
const settings: Record<string, string> = { ...(stored?.settings ?? {}) };
|
||||
@@ -356,12 +362,25 @@ export class DocSyncController {
|
||||
@Body() body: DocsyncResolveConflictDto,
|
||||
) {
|
||||
this.assertOwner(tripId, user);
|
||||
const ok = await this.sync.resolveConflict(Number(itemId), body.keep);
|
||||
const ok = await this.sync.resolveConflict(Number(itemId), body.keep, Number(tripId));
|
||||
if (!ok) throw new HttpException('Item is not in conflict', 400);
|
||||
return { success: true };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/** Whether two URLs address the same server, for deciding if a stored secret may be reused. */
|
||||
function sameOrigin(a: string | null | undefined, b: string): boolean {
|
||||
if (!a) return false;
|
||||
try {
|
||||
const x = new URL(a);
|
||||
const y = new URL(b);
|
||||
return x.protocol === y.protocol && x.host === y.host;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The origin a provider has to call back on.
|
||||
*
|
||||
|
||||
@@ -156,6 +156,18 @@ export interface ReconcileInput {
|
||||
direction: 'both' | 'pull' | 'push';
|
||||
/** A truncated listing must never be read as "the rest was deleted". */
|
||||
remoteTruncated: boolean;
|
||||
/**
|
||||
* The provider says its side has not moved since the stored cursor.
|
||||
*
|
||||
* Every adapter derives that cursor from the scope's contents — a digest over
|
||||
* the documents for Paperless, Papra and Synology, the root ETag for WebDAV —
|
||||
* so the flag means "nothing changed upstream" for all of them. The WebDAV
|
||||
* adapter acts on it by returning an EMPTY list instead of walking the folder,
|
||||
* which is only safe if this is read: otherwise a quiet binding looks like a
|
||||
* folder somebody emptied. It was not read, and the third run of every idle
|
||||
* Nextcloud or OpenCloud binding tripped the mass-delete guard for good.
|
||||
*/
|
||||
remoteUnchanged: boolean;
|
||||
/** Providers without stable ids need the rename heuristic. */
|
||||
stableRemoteIds: boolean;
|
||||
/** How many failures a row gets before it is shelved for a person to look at. */
|
||||
@@ -223,6 +235,25 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
if (it.fileId !== null) itemsByFile.set(it.fileId, it);
|
||||
}
|
||||
|
||||
// Upstream is unchanged: only what happened in TREK can need doing, and the
|
||||
// remote half of the plan would be reading a list the adapter did not fetch.
|
||||
if (input.remoteUnchanged) {
|
||||
for (const it of items) {
|
||||
if (it.fileId === null || !it.remoteId) continue;
|
||||
const l = localById.get(it.fileId);
|
||||
if (l?.deletedAt) add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
}
|
||||
if (direction !== 'pull') {
|
||||
for (const l of local) {
|
||||
if (l.deletedAt) continue;
|
||||
const it = itemsByFile.get(l.fileId);
|
||||
if (it?.remoteId) continue;
|
||||
add({ kind: 'push', local: l, itemId: it?.id ?? null, remoteId: null });
|
||||
}
|
||||
}
|
||||
return { actions, massDeleteGuardTripped: false, missingCount: 0 };
|
||||
}
|
||||
|
||||
// A listing that lost most of what it had last time is far more likely to be
|
||||
// a broken mount, a revoked token or a moved folder than a real mass delete.
|
||||
const known = items.filter((i) => i.remoteId && i.state === 'synced');
|
||||
|
||||
@@ -30,6 +30,12 @@ import {
|
||||
* the Dawarich job on purpose: this one is driven by a per-trip binding an
|
||||
* ordinary user creates, and telling them "restart your server" is not an
|
||||
* answer.
|
||||
*
|
||||
* Which is why the cron itself runs every minute and the tick decides whether it
|
||||
* is due: baking the interval into the cron expression at bootstrap — as this
|
||||
* did until the claim above was checked against the code — meant a changed
|
||||
* interval did nothing until a restart, quietly, while the admin screen said
|
||||
* otherwise.
|
||||
*/
|
||||
@Injectable()
|
||||
export class DocSyncJob implements OnApplicationBootstrap {
|
||||
@@ -41,12 +47,30 @@ export class DocSyncJob implements OnApplicationBootstrap {
|
||||
private readonly registrar: CronRegistrarService,
|
||||
) {}
|
||||
|
||||
/** When the last pass started; null until the first one, which is never skipped. */
|
||||
private lastRunAt: number | null = null;
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
if (!this.registrar.isEnabled()) return;
|
||||
const seconds = this.intervalSeconds();
|
||||
const minutes = Math.max(1, Math.round(seconds / 60));
|
||||
logInfo(`Document sync: scheduled every ${minutes}m`);
|
||||
this.registrar.register('docsync', `*/${minutes} * * * *`, () => this.tick());
|
||||
logInfo(`Document sync: polling every ${this.intervalSeconds()}s`);
|
||||
this.registrar.register('docsync', '* * * * *', () => this.tick());
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether enough time has passed for another pass.
|
||||
*
|
||||
* The minute the cron wakes up on is not the unit the setting is in, so a
|
||||
* 90-second interval must not become one minute or two depending on where the
|
||||
* boundaries fall. Comparing against the last run keeps the setting's own
|
||||
* resolution; the cost of the extra wake-ups is one read of app_settings.
|
||||
*/
|
||||
private isDue(now: number): boolean {
|
||||
// Null rather than 0: comparing against the epoch means "due" only once the
|
||||
// clock has passed the interval since 1970, which is true in production and
|
||||
// false for any test that picks a small timestamp — a difference that would
|
||||
// have hidden here rather than in the behaviour it is supposed to describe.
|
||||
if (this.lastRunAt === null) return true;
|
||||
return now - this.lastRunAt >= this.intervalSeconds() * 1000;
|
||||
}
|
||||
|
||||
private intervalSeconds(): number {
|
||||
@@ -64,6 +88,10 @@ export class DocSyncJob implements OnApplicationBootstrap {
|
||||
// default; only an explicit 'false' stops the sync.
|
||||
if (killSwitch === 'false') return;
|
||||
|
||||
const now = Date.now();
|
||||
if (!this.isDue(now)) return;
|
||||
this.lastRunAt = now;
|
||||
|
||||
// Cheap, and it catches a binding whose owner left the trip through a
|
||||
// path that has no hook to attach to — a transfer, a direct DB edit.
|
||||
const orphaned = this.config.markOrphanedLinks();
|
||||
|
||||
@@ -74,7 +74,17 @@ export class DocSyncService {
|
||||
|
||||
// ── Entry points ───────────────────────────────────────────────────────────
|
||||
|
||||
/** Links the scheduler should look at now. */
|
||||
/**
|
||||
* Links the scheduler should look at now, longest-waiting first.
|
||||
*
|
||||
* The second sort key is what makes the limit fair. A successful run clears
|
||||
* `next_attempt_at`, so on a healthy instance every binding sorts equal on the
|
||||
* first key and SQLite falls back to insertion order — which means the twenty
|
||||
* oldest bindings were picked on every tick and everything created after them
|
||||
* was never synced automatically at all. Found on a dev database with 170
|
||||
* bindings, where a freshly created one was still untouched minutes later
|
||||
* while the first twenty ran again and again.
|
||||
*/
|
||||
dueLinks(limit = 20): LinkRow[] {
|
||||
return this.db.connection
|
||||
.prepare(
|
||||
@@ -82,7 +92,9 @@ export class DocSyncService {
|
||||
WHERE sync_enabled = 1
|
||||
AND failure_count < ?
|
||||
AND (next_attempt_at IS NULL OR next_attempt_at <= CURRENT_TIMESTAMP)
|
||||
ORDER BY COALESCE(next_attempt_at, '1970-01-01') ASC
|
||||
ORDER BY COALESCE(next_attempt_at, '1970-01-01') ASC,
|
||||
COALESCE(last_sync_at, '1970-01-01') ASC,
|
||||
id ASC
|
||||
LIMIT ?`,
|
||||
)
|
||||
.all(LINK_CIRCUIT_OPEN_AFTER, limit) as LinkRow[];
|
||||
@@ -161,6 +173,7 @@ export class DocSyncService {
|
||||
local,
|
||||
direction: link.direction as 'both' | 'pull' | 'push',
|
||||
remoteTruncated: listing.data.truncated,
|
||||
remoteUnchanged: listing.data.cursorUnchanged === true,
|
||||
stableRemoteIds: provider.capabilities(ref).stableId,
|
||||
maxAttempts: ITEM_MAX_ATTEMPTS,
|
||||
});
|
||||
@@ -659,11 +672,21 @@ export class DocSyncService {
|
||||
* overwriting either side, which is the only outcome that cannot lose work
|
||||
* and is therefore what the UI offers first.
|
||||
*/
|
||||
async resolveConflict(itemId: number, keep: 'trek' | 'provider' | 'both'): Promise<boolean> {
|
||||
/**
|
||||
* Decide a conflict.
|
||||
*
|
||||
* `tripId` is not decoration: the route authorises the caller against the trip
|
||||
* in its URL, and without checking the row against the same trip an owner of
|
||||
* any trip could resolve a conflict in somebody else's — the id is a plain
|
||||
* integer and nothing else tied the two together. Same rule as everywhere in
|
||||
* this codebase: every referenced id must exist AND belong to the same trip.
|
||||
*/
|
||||
async resolveConflict(itemId: number, keep: 'trek' | 'provider' | 'both', tripId?: number): Promise<boolean> {
|
||||
const item = this.db.connection
|
||||
.prepare('SELECT * FROM document_sync_items WHERE id = ?')
|
||||
.get(itemId) as Record<string, unknown> | undefined;
|
||||
if (!item || item.state !== 'conflict') return false;
|
||||
if (tripId !== undefined && Number(item.trip_id) !== Number(tripId)) return false;
|
||||
const link = this.config.getLink(Number(item.link_id));
|
||||
if (!link) return false;
|
||||
|
||||
|
||||
@@ -691,3 +691,20 @@ describe('a manual run and the shelved rows', () => {
|
||||
expect(sync.retryShelvedItems).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The trip the route was authorised against is now passed on to the service.
|
||||
*
|
||||
* `assertOwner` proves the caller owns the trip in the URL and says nothing
|
||||
* about whether the item id in the path belongs to that trip — the id is a
|
||||
* plain integer, and nothing tied the two together. The check itself lives in
|
||||
* the service (where the row is), so what belongs here is that the trip reaches
|
||||
* it at all.
|
||||
*/
|
||||
describe('resolving a conflict carries its trip', () => {
|
||||
it('passes the trip from the URL to the service', async () => {
|
||||
sync.resolveConflict.mockClear();
|
||||
await controller.resolve(String(tripId), '42', owner, { keep: 'trek' } as never);
|
||||
expect(sync.resolveConflict).toHaveBeenCalledWith(42, 'trek', tripId);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -64,6 +64,7 @@ const plan = (over: Partial<Parameters<typeof planReconcile>[0]> = {}) =>
|
||||
local: [],
|
||||
direction: 'both',
|
||||
remoteTruncated: false,
|
||||
remoteUnchanged: false,
|
||||
stableRemoteIds: true,
|
||||
maxAttempts: 6,
|
||||
...over,
|
||||
@@ -399,3 +400,92 @@ describe('planReconcile > shelving a row that keeps failing', () => {
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* An unchanged upstream, which the WebDAV adapters report by NOT fetching.
|
||||
*
|
||||
* Their root-ETag probe answers "nothing has happened here" in one request, and
|
||||
* they then return an empty document list. The core ignored the flag and read
|
||||
* that emptiness as a folder somebody had emptied: from the third run of every
|
||||
* idle Nextcloud or OpenCloud binding onward, every run ended in the mass-delete
|
||||
* guard — `partial`, `missing: 6`, forever, until the failure counter opened the
|
||||
* circuit and the scheduler dropped the binding entirely. Reproduced against a
|
||||
* live Nextcloud before this was written.
|
||||
*/
|
||||
describe('planReconcile > upstream reported as unchanged', () => {
|
||||
it('does not read an empty listing as a mass deletion', () => {
|
||||
const p = plan({
|
||||
items: [item({ id: 1, remoteId: 'r1' }), item({ id: 2, fileId: 2, remoteId: 'r2' })],
|
||||
remote: [],
|
||||
local: [local({ fileId: 1 }), local({ fileId: 2 })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.massDeleteGuardTripped).toBe(false);
|
||||
expect(p.missingCount).toBe(0);
|
||||
});
|
||||
|
||||
it('marks nothing as gone from the provider', () => {
|
||||
const p = plan({
|
||||
items: [item({ id: 1, remoteId: 'r1' })],
|
||||
remote: [],
|
||||
local: [local({ fileId: 1 })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions.map(a => a.kind)).not.toContain('mark_remote_missing');
|
||||
});
|
||||
|
||||
it('still pushes a document TREK gained in the meantime', () => {
|
||||
const p = plan({
|
||||
items: [],
|
||||
remote: [],
|
||||
local: [local({ fileId: 7, name: 'new.pdf' })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions).toEqual([{ kind: 'push', local: expect.objectContaining({ fileId: 7 }), itemId: null, remoteId: null }]);
|
||||
});
|
||||
|
||||
it('still reports a document TREK deleted in the meantime', () => {
|
||||
const p = plan({
|
||||
items: [item({ id: 5, fileId: 3, remoteId: 'r3' })],
|
||||
remote: [],
|
||||
local: [local({ fileId: 3, deletedAt: '2026-09-18 10:00:00' })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions).toEqual([{ kind: 'local_deleted', itemId: 5, remoteId: 'r3' }]);
|
||||
});
|
||||
|
||||
it('does not push out of a pull-only binding', () => {
|
||||
const p = plan({
|
||||
items: [],
|
||||
remote: [],
|
||||
local: [local({ fileId: 7 })],
|
||||
direction: 'pull',
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('plans nothing at all when neither side moved', () => {
|
||||
const p = plan({
|
||||
items: [item({ id: 1, remoteId: 'r1' })],
|
||||
remote: [],
|
||||
local: [local({ fileId: 1 })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('works the old way when the provider does report a change', () => {
|
||||
// The same shape with the flag off is exactly the case that used to break:
|
||||
// an empty listing then really does mean everything vanished.
|
||||
const many = Array.from({ length: 6 }, (_, i) =>
|
||||
item({ id: i + 1, fileId: i + 1, remoteId: `r${i + 1}` }));
|
||||
const p = plan({
|
||||
items: many,
|
||||
remote: [],
|
||||
local: many.map((_, i) => local({ fileId: i + 1 })),
|
||||
remoteUnchanged: false,
|
||||
});
|
||||
expect(p.massDeleteGuardTripped).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* '5' and '999999' both have to land somewhere the cron parser accepts. And one
|
||||
* unreachable NAS must not take the other trips' bindings down with it.
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
const log = vi.hoisted(() => ({
|
||||
LOG_LEVEL: 'error',
|
||||
@@ -102,11 +102,16 @@ describe('DocSyncJob bootstrap', () => {
|
||||
expect(log.logInfo).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('registers one cron under a name of its own and announces the cadence', () => {
|
||||
it('registers one cron under a name of its own, on the minute', () => {
|
||||
// Every minute, with the tick deciding whether it is due. Baking the
|
||||
// interval into the expression at bootstrap meant a changed setting did
|
||||
// nothing until a restart — which is the opposite of what this job's own
|
||||
// comment promises, and nothing re-registers it (auto-backup has a start()
|
||||
// its settings save calls; this has no such path).
|
||||
const { job, registrar } = makeJob();
|
||||
job.onApplicationBootstrap();
|
||||
expect(registrar.register).toHaveBeenCalledWith('docsync', '*/5 * * * *', expect.any(Function));
|
||||
expect(log.logInfo).toHaveBeenCalledWith('Document sync: scheduled every 5m');
|
||||
expect(registrar.register).toHaveBeenCalledWith('docsync', '* * * * *', expect.any(Function));
|
||||
expect(log.logInfo).toHaveBeenCalledWith('Document sync: polling every 300s');
|
||||
});
|
||||
|
||||
it('reads the interval from its own app_settings key', () => {
|
||||
@@ -115,32 +120,6 @@ describe('DocSyncJob bootstrap', () => {
|
||||
expect(db.get).toHaveBeenCalledWith('SELECT value FROM app_settings WHERE key = ?', SETTING_POLL_INTERVAL);
|
||||
});
|
||||
|
||||
it('clamps a hand-typed interval to between a minute and an hour', () => {
|
||||
for (const [setting, minutes] of [
|
||||
[undefined, 5], // no row: the 300s default
|
||||
['', 5],
|
||||
['not-a-number', 5],
|
||||
['0', 1], // below the floor, and a `*/0` expression the parser would reject
|
||||
['10', 1],
|
||||
['60', 1], // the floor itself
|
||||
['600', 10],
|
||||
['3600', 60], // the ceiling itself
|
||||
['86400', 60],
|
||||
['-300', 1], // a minus sign parses fine and must not become a negative cron
|
||||
] as const) {
|
||||
vi.clearAllMocks();
|
||||
const { job, registrar } = makeJob({ interval: setting });
|
||||
job.onApplicationBootstrap();
|
||||
expect(registrar.register).toHaveBeenCalledWith('docsync', `*/${minutes} * * * *`, expect.any(Function));
|
||||
}
|
||||
});
|
||||
|
||||
it('rounds a sub-minute remainder to the nearest whole minute rather than down to nothing', () => {
|
||||
const { job, registrar } = makeJob({ interval: '90' });
|
||||
job.onApplicationBootstrap();
|
||||
expect(registrar.register).toHaveBeenCalledWith('docsync', '*/2 * * * *', expect.any(Function));
|
||||
});
|
||||
|
||||
it('hands the registrar the tick itself, so a fired cron reaches the sync', async () => {
|
||||
const { job, sync, takeTick } = makeJob({ links: [link(1)] });
|
||||
job.onApplicationBootstrap();
|
||||
@@ -202,11 +181,11 @@ describe('DocSyncJob tick', () => {
|
||||
});
|
||||
|
||||
it('re-reads the kill switch per tick instead of remembering the first answer', async () => {
|
||||
const { job, db, sync } = makeJob({ links: [link(1)] });
|
||||
const { job, db } = makeJob({ links: [link(1)] });
|
||||
await job.tick();
|
||||
await job.tick();
|
||||
expect(db.get).toHaveBeenCalledTimes(2);
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
const killSwitchReads = db.get.mock.calls.filter((c) => c[1] === SETTING_SYNC_ENABLED).length;
|
||||
expect(killSwitchReads).toBe(2);
|
||||
});
|
||||
|
||||
it('sweeps bindings whose credential owner left the trip before it syncs anything', async () => {
|
||||
@@ -261,15 +240,27 @@ describe('DocSyncJob tick', () => {
|
||||
expect(log.logError).toHaveBeenCalledWith('Document sync tick failed: SQLITE_BUSY');
|
||||
});
|
||||
|
||||
it('recovers on the next tick after a failed one', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)] });
|
||||
sync.dueLinks.mockImplementationOnce(() => {
|
||||
throw new Error('down');
|
||||
});
|
||||
await job.tick();
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
expect(log.logError).toHaveBeenCalledTimes(1);
|
||||
it('recovers on the next due tick after a failed one', async () => {
|
||||
// A failed pass still spends its interval, on purpose: a tick that throws
|
||||
// every time would otherwise run on every minute the cron fires. It does not
|
||||
// block anything permanently — the next due tick works normally.
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
vi.setSystemTime(new Date(10_000_000));
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
|
||||
sync.dueLinks.mockImplementationOnce(() => {
|
||||
throw new Error('down');
|
||||
});
|
||||
await job.tick();
|
||||
expect(sync.syncLink).not.toHaveBeenCalled();
|
||||
expect(log.logError).toHaveBeenCalledTimes(1);
|
||||
|
||||
vi.setSystemTime(new Date(10_300_000));
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('leaves the choice of what is due to the sync service', async () => {
|
||||
@@ -282,3 +273,108 @@ describe('DocSyncJob tick', () => {
|
||||
expect(sync.syncLink.mock.calls.map((c) => c[0])).toEqual(links);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The interval, now that the cron fires every minute.
|
||||
*
|
||||
* It used to be baked into the cron expression at bootstrap, so the setting did
|
||||
* nothing until a restart while this file's own header said otherwise. The tick
|
||||
* now decides, which also means the clamp has to hold here instead of in the
|
||||
* expression: an interval of 0 must not turn the minute cron into a run every
|
||||
* minute, and a huge one must not park the job for a day.
|
||||
*/
|
||||
describe('DocSyncJob due-ness', () => {
|
||||
// Fake system time rather than a Date.now spy: the spy has to be re-applied
|
||||
// after the suite-wide clearAllMocks, and a cleared spy answers undefined,
|
||||
// which reads as "no time has passed" and fails the case for the wrong reason.
|
||||
const at = (ms: number) => vi.setSystemTime(new Date(ms));
|
||||
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it('runs the first tick it is given', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)] });
|
||||
at(1_000_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does nothing on a tick inside the interval', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
|
||||
at(1_000_000);
|
||||
await job.tick();
|
||||
at(1_000_000 + 299_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('runs again once the interval has passed', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: '300' });
|
||||
at(1_000_000);
|
||||
await job.tick();
|
||||
at(1_000_000 + 300_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('keeps the floor: a zero or negative setting still waits a minute', async () => {
|
||||
for (const setting of ['0', '-300', '10']) {
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: setting });
|
||||
at(2_000_000);
|
||||
await job.tick();
|
||||
at(2_000_000 + 59_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink, `interval=${setting} ran again inside the floor`).toHaveBeenCalledTimes(1);
|
||||
at(2_000_000 + 60_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink, `interval=${setting} did not run at the floor`).toHaveBeenCalledTimes(2);
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the ceiling: an absurd setting waits an hour, not a day', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: '86400' });
|
||||
at(3_000_000);
|
||||
await job.tick();
|
||||
at(3_000_000 + 3_600_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('takes a changed setting without a restart, which is the whole point', async () => {
|
||||
const settings = new Map<string, string>([[SETTING_POLL_INTERVAL, '3600']]);
|
||||
const db = {
|
||||
get: vi.fn((_sql: string, key?: unknown) => {
|
||||
const value = settings.get(String(key));
|
||||
return value === undefined ? undefined : { value };
|
||||
}),
|
||||
};
|
||||
const sync = {
|
||||
dueLinks: vi.fn(() => [link(1)]),
|
||||
syncLink: vi.fn(async () => RUN),
|
||||
};
|
||||
const job = new DocSyncJob(
|
||||
db as unknown as DatabaseService,
|
||||
sync as unknown as DocSyncService,
|
||||
{ markOrphanedLinks: vi.fn(() => 0) } as unknown as DocSyncConfigService,
|
||||
{ isAddonEnabled: vi.fn(() => true) } as unknown as AddonsService,
|
||||
{ isEnabled: vi.fn(() => true), register: vi.fn(), unregister: vi.fn() } as unknown as CronRegistrarService,
|
||||
);
|
||||
|
||||
at(4_000_000);
|
||||
await job.tick();
|
||||
at(4_000_000 + 120_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1); // still inside the hour
|
||||
|
||||
settings.set(SETTING_POLL_INTERVAL, '60');
|
||||
await job.tick();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not count a tick the kill switch stopped, so switching back on runs at once', async () => {
|
||||
const { job, sync } = makeJob({ links: [link(1)], interval: '3600', killSwitch: 'false' });
|
||||
at(5_000_000);
|
||||
await job.tick();
|
||||
expect(sync.syncLink).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -185,6 +185,7 @@ function makeLink(over: {
|
||||
syncEnabled?: number;
|
||||
failureCount?: number;
|
||||
nextAttemptAt?: string | null;
|
||||
lastSyncAt?: string | null;
|
||||
} = {}): LinkRow {
|
||||
scopeSeq += 1;
|
||||
const info = testDb
|
||||
@@ -199,6 +200,10 @@ function makeLink(over: {
|
||||
over.direction ?? 'both', over.deletePolicy ?? 'unlink',
|
||||
over.syncEnabled ?? 1, over.failureCount ?? 0, over.nextAttemptAt ?? null, ownerId,
|
||||
);
|
||||
if (over.lastSyncAt !== undefined) {
|
||||
testDb.prepare('UPDATE trip_document_links SET last_sync_at = ? WHERE id = ?')
|
||||
.run(over.lastSyncAt, Number(info.lastInsertRowid));
|
||||
}
|
||||
return config.getLink(Number(info.lastInsertRowid));
|
||||
}
|
||||
|
||||
@@ -314,6 +319,36 @@ describe('DocSyncService', () => {
|
||||
expect(ids).toEqual([dueNow.id, overdue.id].sort((a, b) => a - b));
|
||||
});
|
||||
|
||||
it('puts the longest-waiting binding first, so the limit cannot starve the newer ones', () => {
|
||||
// A successful run clears next_attempt_at, so on a healthy instance every
|
||||
// binding ties on the first sort key. Without a second key SQLite answers
|
||||
// in insertion order and the same oldest bindings win every tick: found on
|
||||
// a dev database with 170 of them, where a binding created minutes earlier
|
||||
// had never been synced while the first twenty ran over and over.
|
||||
const recent = makeLink({ lastSyncAt: sqlTime('-1 minute') });
|
||||
const stale = makeLink({ lastSyncAt: sqlTime('-2 hours') });
|
||||
const never = makeLink({ lastSyncAt: null });
|
||||
|
||||
expect(service.dueLinks().map((l) => l.id)).toEqual([never.id, stale.id, recent.id]);
|
||||
});
|
||||
|
||||
it('serves ordinary bindings before one that is only due again after a backoff', () => {
|
||||
// COALESCE(next_attempt_at, '1970-01-01') is what orders these: a binding
|
||||
// with no backoff counts as due since the epoch and therefore goes first,
|
||||
// while one whose backoff has merely expired sorts by when it expired.
|
||||
// Normal work ahead of a retry, which is the right way round.
|
||||
const backedOff = makeLink({ nextAttemptAt: sqlTime('-1 hour'), lastSyncAt: sqlTime('-1 minute') });
|
||||
const never = makeLink({ lastSyncAt: null });
|
||||
|
||||
expect(service.dueLinks().map((l) => l.id)).toEqual([never.id, backedOff.id]);
|
||||
});
|
||||
|
||||
it('honours the limit', () => {
|
||||
for (let i = 0; i < 25; i += 1) makeLink({ lastSyncAt: null });
|
||||
expect(service.dueLinks().length).toBe(20);
|
||||
expect(service.dueLinks(5).length).toBe(5);
|
||||
});
|
||||
|
||||
it('lets a binding back in one failure before the circuit opens', () => {
|
||||
const nearly = makeLink({ failureCount: LINK_CIRCUIT_OPEN_AFTER - 1 });
|
||||
expect(service.dueLinks().map((l) => l.id)).toEqual([nearly.id]);
|
||||
@@ -914,4 +949,41 @@ describe('DocSyncService', () => {
|
||||
expect(states).toEqual([...decidable].sort());
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A conflict may only be decided from the trip it belongs to.
|
||||
*
|
||||
* The route authorises the caller against the trip in its URL and then took the
|
||||
* item id on trust, so the owner of any trip could resolve a conflict in
|
||||
* somebody else's: both are plain integers and nothing compared them. Found by
|
||||
* an audit rather than by a failing test, which is why the check lives here,
|
||||
* next to the row it is about.
|
||||
*/
|
||||
describe('resolveConflict across trips', () => {
|
||||
function conflictedItem(linkId: number, tripOfItem: number): number {
|
||||
return Number(testDb.prepare(
|
||||
`INSERT INTO document_sync_items (link_id, trip_id, trek_doc_uid, remote_id, state)
|
||||
VALUES (?, ?, 'uid-conflict', 'r-conflict', 'conflict')`,
|
||||
).run(linkId, tripOfItem).lastInsertRowid);
|
||||
}
|
||||
it('refuses when the row belongs to another trip', async () => {
|
||||
const link = makeLink();
|
||||
const itemId = conflictedItem(link.id, link.trip_id);
|
||||
expect(await service.resolveConflict(itemId, 'trek', link.trip_id + 999)).toBe(false);
|
||||
const after = testDb.prepare('SELECT state FROM document_sync_items WHERE id = ?').get(itemId) as { state: string };
|
||||
expect(after.state).toBe('conflict');
|
||||
});
|
||||
it('resolves when the row belongs to the trip', async () => {
|
||||
const link = makeLink();
|
||||
const itemId = conflictedItem(link.id, link.trip_id);
|
||||
expect(await service.resolveConflict(itemId, 'trek', link.trip_id)).toBe(true);
|
||||
const after = testDb.prepare('SELECT state FROM document_sync_items WHERE id = ?').get(itemId) as { state: string };
|
||||
expect(after.state).not.toBe('conflict');
|
||||
});
|
||||
it('still works for a caller that names no trip, so nothing else breaks', async () => {
|
||||
const link = makeLink();
|
||||
const itemId = conflictedItem(link.id, link.trip_id);
|
||||
expect(await service.resolveConflict(itemId, 'trek')).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user