mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
fix(docsync): stop a push from stealing another file's pairing
Papra answers a push of bytes it already holds with the document that has them. Two trip files with the same content therefore pushed to one document, and the unique index on (link_id, remote_id) aborted that run and every run after it with a constraint error. The pairing now stays with the file that owns it and the second file is recorded as a conflict. Also in this pass: - Paperless no longer adopts a same-bytes document that lacks the scope's tag when a consume outlives the poll budget; that document belongs to someone else's filing and delete-through would have trashed it. - The pull path writes the file row, the retirement of the copy it replaces and the pairing in one transaction, and announces them only once committed. - PATCH links stops accepting remoteRootId/remoteRootPath, which it read and silently dropped. - The MCP sync tool clears shelved items before running, as the REST route does.
This commit is contained in:
@@ -371,3 +371,40 @@ describe('the admin switches', () => {
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A nudge that lands on a run already in flight.
|
||||
*
|
||||
* `syncLink` answers `busy` and returns, so the nudge was thrown away — and the
|
||||
* changes it was about may well have landed after the running pass read the
|
||||
* folder, which means waiting out a whole poll interval for them. Asked again
|
||||
* once, and only once, so two clients cannot keep each other going.
|
||||
*/
|
||||
describe('a nudge that arrives mid-run', () => {
|
||||
it('asks again once when the run was busy', async () => {
|
||||
sync.syncLink.mockResolvedValueOnce({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
|
||||
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
|
||||
settle();
|
||||
await Promise.resolve();
|
||||
settle();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('gives up after that one retry rather than chasing itself', async () => {
|
||||
sync.syncLink.mockResolvedValue({ state: 'busy', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
|
||||
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
|
||||
for (let i = 0; i < 5; i += 1) { settle(); await Promise.resolve(); }
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not retry a run that worked', async () => {
|
||||
// Set explicitly: clearAllMocks drops the calls but keeps the
|
||||
// implementation the previous case installed.
|
||||
sync.syncLink.mockResolvedValue({ state: 'ok', pulled: 0, pushed: 0, conflicts: 0, missing: 0 });
|
||||
controller.nudge('tok-live', makeReq({ 'x-trek-docsync-secret': SECRET }));
|
||||
settle();
|
||||
await Promise.resolve();
|
||||
settle();
|
||||
expect(sync.syncLink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -570,3 +570,31 @@ describe('planReconcile > a pairing whose download never landed', () => {
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['touch']);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Names are compared through the same sanitiser the local one went through.
|
||||
*
|
||||
* A provider name TREK had to clean up — a slash, a control character, a
|
||||
* leading dot — otherwise reads as "TREK renamed this document" on the very
|
||||
* next run, and the provider's copy is renamed to the cleaned version without
|
||||
* anybody asking for it.
|
||||
*/
|
||||
describe('planReconcile > a provider name TREK had to clean up', () => {
|
||||
it('does not read the cleaning as a local rename', () => {
|
||||
const p = plan({
|
||||
items: [item({ remoteName: 'a/b.pdf', contentSha256: 'aaa' })],
|
||||
remote: [remote({ remoteId: 'r1', name: 'a/b.pdf', remoteVersion: 'v1' })],
|
||||
local: [local({ fileId: 1, name: 'b.pdf' })], // what sanitizeIncomingName made of it
|
||||
});
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['touch']);
|
||||
});
|
||||
|
||||
it('still follows a real rename at the provider', () => {
|
||||
const p = plan({
|
||||
items: [item({ remoteName: 'b.pdf', contentSha256: 'aaa' })],
|
||||
remote: [remote({ remoteId: 'r1', name: 'invoice.pdf', remoteVersion: 'v1' })],
|
||||
local: [local({ fileId: 1, name: 'b.pdf' })],
|
||||
});
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['rename_local']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -49,6 +49,7 @@ function makeMcp(over: Partial<Setup> = {}) {
|
||||
status: vi.fn(() => setup.status),
|
||||
issues: vi.fn(() => setup.issues),
|
||||
syncLink: vi.fn(async () => RUN),
|
||||
retryShelvedItems: vi.fn(),
|
||||
};
|
||||
const addons = { isAddonEnabled: vi.fn(() => setup.addonOn) };
|
||||
const mcp = new DocSyncMcp(
|
||||
@@ -220,6 +221,16 @@ describe('sync_trip_documents', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('un-shelves the documents that gave up before running, as the REST route does', async () => {
|
||||
// "Run it now" means "try again", including the items parked after too many
|
||||
// failures. A tool that skipped this reported "in sync" while leaving them
|
||||
// parked, and the only way out was the button in the web UI.
|
||||
const links = [link(1, 'paperless'), link(2, 'nextcloud')];
|
||||
const { mcp, sync } = makeMcp({ links });
|
||||
await mcp.syncNow({ tripId: 3 }, ctx);
|
||||
expect(sync.retryShelvedItems.mock.calls).toEqual([[1], [2]]);
|
||||
});
|
||||
|
||||
it('asks for the cheap incremental run unless the caller says otherwise', async () => {
|
||||
const { mcp, sync } = makeMcp({ links: [link(1)] });
|
||||
await mcp.syncNow({ tripId: 3 }, ctx);
|
||||
|
||||
@@ -66,6 +66,7 @@ import type {
|
||||
PushResult,
|
||||
RemoteDocument,
|
||||
} from '../../../../src/nest/doc-sync/document-provider';
|
||||
import type { AddonsService } from '../../../../src/nest/addons/addons.service';
|
||||
import type { FilesService } from '../../../../src/nest/files/files.service';
|
||||
import type { StorageService } from '../../../../src/nest/storage/storage.service';
|
||||
import type { RealtimeService } from '../../../../src/nest/realtime/realtime.service';
|
||||
@@ -272,6 +273,12 @@ const fileRows = () =>
|
||||
const sqlTime = (modifier: string): string =>
|
||||
(testDb.prepare("SELECT datetime('now', ?) AS t").get(modifier) as { t: string }).t;
|
||||
|
||||
/**
|
||||
* The provider's own addon switch. Every binding asks it before it runs, so a
|
||||
* double that always says yes would hide the one case worth a test.
|
||||
*/
|
||||
const addons = { isAddonEnabled: vi.fn(() => true) };
|
||||
|
||||
// ── Suite ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('DocSyncService', () => {
|
||||
@@ -293,7 +300,16 @@ describe('DocSyncService', () => {
|
||||
const dbs = new DatabaseService(testDb);
|
||||
const registry = new DocumentProviderRegistry([provider as unknown as DocumentProvider]);
|
||||
config = new DocSyncConfigService(dbs, registry);
|
||||
service = new DocSyncService(dbs, config, registry, storage, files, new AllowedFileTypesService(dbs), realtime);
|
||||
service = new DocSyncService(
|
||||
dbs,
|
||||
config,
|
||||
registry,
|
||||
storage,
|
||||
files,
|
||||
new AllowedFileTypesService(dbs),
|
||||
realtime,
|
||||
addons as unknown as AddonsService,
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
@@ -548,6 +564,49 @@ describe('DocSyncService', () => {
|
||||
expect(itemRows()[0].state).toBe('synced');
|
||||
});
|
||||
|
||||
it('refuses to move a pairing onto a document another file already owns', async () => {
|
||||
// Papra answers a push of bytes it already holds with the document that
|
||||
// has them. Two trip files with the same content therefore push to ONE
|
||||
// document — and the unique index on (link_id, remote_id) turned that
|
||||
// into a constraint error that aborted this run and every run after it.
|
||||
const link = makeLink();
|
||||
makeFile({ name: 'boarding.pdf' });
|
||||
makeFile({ name: 'boarding-copy.pdf', storageKey: 'key-copy' });
|
||||
provider.list.mockResolvedValue(ok(listing([])));
|
||||
|
||||
const run = await service.syncLink(link);
|
||||
|
||||
// Counted as a conflict, which is what it is — a document two files want
|
||||
// — and reported the same way as any other conflict rather than as a
|
||||
// failed run.
|
||||
expect(run.pushed).toBe(1);
|
||||
expect(run.conflicts).toBe(1);
|
||||
|
||||
const rows = itemRows();
|
||||
expect(rows).toHaveLength(2);
|
||||
// The first file keeps the document. The second is on record as the one
|
||||
// that could not have it, with its bytes still in TREK.
|
||||
expect(rows.filter((r) => r.remote_id === 'r-pushed')).toHaveLength(1);
|
||||
const blocked = rows.find((r) => r.state === 'error');
|
||||
expect(blocked?.error_code).toBe('conflict');
|
||||
expect(blocked?.remote_id).toBeNull();
|
||||
});
|
||||
|
||||
it('stands down while the provider is switched off in the admin panel', async () => {
|
||||
// Switching a provider off stopped new connections and left every
|
||||
// existing binding running, which is not what "off" means to the person
|
||||
// who switched it. Not a failure either: nothing is wrong with the
|
||||
// binding, an admin has closed the door.
|
||||
const link = makeLink();
|
||||
addons.isAddonEnabled.mockReturnValueOnce(false);
|
||||
|
||||
const res = await service.syncLink(link);
|
||||
|
||||
expect(res.state).toBe('disabled');
|
||||
expect(provider.list).not.toHaveBeenCalled();
|
||||
expect(linkRow(link.id).failure_count).toBe(0);
|
||||
});
|
||||
|
||||
it('counts a failure and puts the binding off until later', async () => {
|
||||
const link = makeLink();
|
||||
provider.list.mockResolvedValueOnce(fail('unreachable'));
|
||||
|
||||
@@ -743,6 +743,20 @@ describe('PaperlessProvider — pushing bytes', () => {
|
||||
expect(lastRequest('GET /api/documents/').url.searchParams.get('checksum__iexact')).toBe(HASH_A);
|
||||
});
|
||||
|
||||
it('PAPERLESS-057a: a document with the same bytes but another tag is not adopted', async () => {
|
||||
// The tag goes on at upload, so only a tagged document can be the one this
|
||||
// push made. Taking the untagged twin instead put a stranger's document in
|
||||
// the trip — and delete-through would have trashed it on a later run.
|
||||
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
|
||||
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
|
||||
);
|
||||
on('GET /api/custom_fields/', reply(page([{ id: 1, name: TRIP_UID_FIELD_NAME, data_type: 'string' }])));
|
||||
on('POST /api/documents/post_document/', reply('"task-uuid-1"'));
|
||||
on('GET /api/documents/', reply(page([docRow({ id: 21, tags: [99] })])));
|
||||
expect(expectFail(await provider.push(CONN, SCOPE, pushRequest())).code).toBe('timeout');
|
||||
expect(requests('GET /api/documents/21/')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('PAPERLESS-057: when the checksum finds nothing either, the timeout is reported', async () => {
|
||||
vi.spyOn(client, 'awaitConsume').mockRejectedValue(
|
||||
new PaperlessError('timeout', 'Paperless is still processing the uploaded file'),
|
||||
|
||||
Reference in New Issue
Block a user