fix(docsync): stop a push from stealing another file's pairing

Papra answers a push of bytes it already holds with the document that has
them. Two trip files with the same content therefore pushed to one document,
and the unique index on (link_id, remote_id) aborted that run and every run
after it with a constraint error. The pairing now stays with the file that
owns it and the second file is recorded as a conflict.

Also in this pass:
- Paperless no longer adopts a same-bytes document that lacks the scope's tag
  when a consume outlives the poll budget; that document belongs to someone
  else's filing and delete-through would have trashed it.
- The pull path writes the file row, the retirement of the copy it replaces
  and the pairing in one transaction, and announces them only once committed.
- PATCH links stops accepting remoteRootId/remoteRootPath, which it read and
  silently dropped.
- The MCP sync tool clears shelved items before running, as the REST route
  does.
This commit is contained in:
Maurice
2026-09-19 00:44:43 +02:00
parent 7237a1881d
commit e6a5ca1521
11 changed files with 267 additions and 39 deletions
@@ -371,3 +371,40 @@ describe('the admin switches', () => {
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
});
/**
* A nudge that lands on a run already in flight.
*
* `syncLink` answers `busy` and returns, so the nudge was thrown away — and the
* changes it was about may well have landed after the running pass read the
* folder, which means waiting out a whole poll interval for them. Asked again
* once, and only once, so two clients cannot keep each other going.
*/
describe('a nudge that arrives mid-run', () => {
it('asks again once when the run was busy', async () => {
sync.syncLink.mockResolvedValueOnce({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
await Promise.resolve();
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('gives up after that one retry rather than chasing itself', async () => {
sync.syncLink.mockResolvedValue({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
for (let i = 0; i < 5; i += 1) { settle(); await Promise.resolve(); }
expect(sync.syncLink).toHaveBeenCalledTimes(2);
});
it('does not retry a run that worked', async () => {
// Set explicitly: clearAllMocks drops the calls but keeps the
// implementation the previous case installed.
sync.syncLink.mockResolvedValue({ state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
settle();
await Promise.resolve();
settle();
expect(sync.syncLink).toHaveBeenCalledTimes(1);
});
});
@@ -570,3 +570,31 @@ describe('planReconcile > a pairing whose download never landed', () => {
expect(p.actions.map(a => a.kind)).toEqual(['touch']);
});
});
/**
* Names are compared through the same sanitiser the local one went through.
*
* A provider name TREK had to clean up — a slash, a control character, a
* leading dot — otherwise reads as "TREK renamed this document" on the very
* next run, and the provider's copy is renamed to the cleaned version without
* anybody asking for it.
*/
describe('planReconcile > a provider name TREK had to clean up', () => {
it('does not read the cleaning as a local rename', () => {
const p = plan({
items: [item({ remoteName: 'a/b.pdf', contentSha256: 'aaa' })],
remote: [remote({ remoteId: 'r1', name: 'a/b.pdf', remoteVersion: 'v1' })],
local: [local({ fileId: 1, name: 'b.pdf' })], // what sanitizeIncomingName made of it
});
expect(p.actions.map(a => a.kind)).toEqual(['touch']);
});
it('still follows a real rename at the provider', () => {
const p = plan({
items: [item({ remoteName: 'b.pdf', contentSha256: 'aaa' })],
remote: [remote({ remoteId: 'r1', name: 'invoice.pdf', remoteVersion: 'v1' })],
local: [local({ fileId: 1, name: 'b.pdf' })],
});
expect(p.actions.map(a => a.kind)).toEqual(['rename_local']);
});
});
@@ -49,6 +49,7 @@ function makeMcp(over: Partial<Setup> = {}) {
status: vi.fn(() => setup.status),
issues: vi.fn(() => setup.issues),
syncLink: vi.fn(async () => RUN),
retryShelvedItems: vi.fn(),
};
const addons = { isAddonEnabled: vi.fn(() => setup.addonOn) };
const mcp = new DocSyncMcp(
@@ -220,6 +221,16 @@ describe('sync_trip_documents', () => {
]);
});
it('un-shelves the documents that gave up before running, as the REST route does', async () => {
// "Run it now" means "try again", including the items parked after too many
// failures. A tool that skipped this reported "in sync" while leaving them
// parked, and the only way out was the button in the web UI.
const links = [link(1, 'paperless'), link(2, 'nextcloud')];
const { mcp, sync } = makeMcp({ links });
await mcp.syncNow({ tripId: 3 }, ctx);
expect(sync.retryShelvedItems.mock.calls).toEqual([[1], [2]]);
});
it('asks for the cheap incremental run unless the caller says otherwise', async () => {
const { mcp, sync } = makeMcp({ links: [link(1)] });
await mcp.syncNow({ tripId: 3 }, ctx);
@@ -66,6 +66,7 @@ import type {
PushResult,
RemoteDocument,
} from '../../../../src/nest/doc-sync/document-provider';
import type { AddonsService } from '../../../../src/nest/addons/addons.service';
import type { FilesService } from '../../../../src/nest/files/files.service';
import type { StorageService } from '../../../../src/nest/storage/storage.service';
import type { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
@@ -272,6 +273,12 @@ const fileRows = () =>
const sqlTime = (modifier: string): string =>
(testDb.prepare("SELECT datetime('now', ?) AS t").get(modifier) as { t: string }).t;
/**
* The provider's own addon switch. Every binding asks it before it runs, so a
* double that always says yes would hide the one case worth a test.
*/
const addons = { isAddonEnabled: vi.fn(() => true) };
// ── Suite ────────────────────────────────────────────────────────────────────
describe('DocSyncService', () => {
@@ -293,7 +300,16 @@ describe('DocSyncService', () => {
const dbs = new DatabaseService(testDb);
const registry = new DocumentProviderRegistry([provider as unknown as DocumentProvider]);
config = new DocSyncConfigService(dbs, registry);
service = new DocSyncService(dbs, config, registry, storage, files, new AllowedFileTypesService(dbs), realtime);
service = new DocSyncService(
dbs,
config,
registry,
storage,
files,
new AllowedFileTypesService(dbs),
realtime,
addons as unknown as AddonsService,
);
});
afterAll(() => {
@@ -548,6 +564,49 @@ describe('DocSyncService', () => {
expect(itemRows()[0].state).toBe('synced');
});
it('refuses to move a pairing onto a document another file already owns', async () => {
// Papra answers a push of bytes it already holds with the document that
// has them. Two trip files with the same content therefore push to ONE
// document — and the unique index on (link_id, remote_id) turned that
// into a constraint error that aborted this run and every run after it.
const link = makeLink();
makeFile({ name: 'boarding.pdf' });
makeFile({ name: 'boarding-copy.pdf', storageKey: 'key-copy' });
provider.list.mockResolvedValue(ok(listing([])));
const run = await service.syncLink(link);
// Counted as a conflict, which is what it is — a document two files want
// — and reported the same way as any other conflict rather than as a
// failed run.
expect(run.pushed).toBe(1);
expect(run.conflicts).toBe(1);
const rows = itemRows();
expect(rows).toHaveLength(2);
// The first file keeps the document. The second is on record as the one
// that could not have it, with its bytes still in TREK.
expect(rows.filter((r) => r.remote_id === 'r-pushed')).toHaveLength(1);
const blocked = rows.find((r) => r.state === 'error');
expect(blocked?.error_code).toBe('conflict');
expect(blocked?.remote_id).toBeNull();
});
it('stands down while the provider is switched off in the admin panel', async () => {
// Switching a provider off stopped new connections and left every
// existing binding running, which is not what "off" means to the person
// who switched it. Not a failure either: nothing is wrong with the
// binding, an admin has closed the door.
const link = makeLink();
addons.isAddonEnabled.mockReturnValueOnce(false);
const res = await service.syncLink(link);
expect(res.state).toBe('disabled');
expect(provider.list).not.toHaveBeenCalled();
expect(linkRow(link.id).failure_count).toBe(0);
});
it('counts a failure and puts the binding off until later', async () => {
const link = makeLink();
provider.list.mockResolvedValueOnce(fail('unreachable'));
@@ -743,6 +743,20 @@ describe('PaperlessProvider — pushing bytes', () => {
expect(lastRequest('GET /api/documents/').url.searchParams.get('checksum__iexact')).toBe(HASH_A);
});
it('PAPERLESS-057a: a document with the same bytes but another tag is not adopted', async () => {
// The tag goes on at upload, so only a tagged document can be the one this
// push made. Taking the untagged twin instead put a stranger's document in
// the trip — and delete-through would have trashed it on a later run.
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
);
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
on('GET /api/documents/', reply(page([docRow({ id: 21, tags: [99] })])));
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('timeout');
expect(requests('GET /api/documents/21/')).toHaveLength(0);
});
it('PAPERLESS-057: when the checksum finds nothing either, the timeout is reported', async () => {
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),