mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
fix(docsync): act on a deletion once, and know what to do when it is undone
Both planner branches planned local_deleted for as long as a file sat in TREK's trash. Under unlink nothing showed, until the binding was switched to trash: the next run then binned every document ever deleted in TREK, months back included. A binding on trash from the start asked the provider to bin the same copy on every run. A local_deleted row is now left alone, so the policy is applied once, under the policy in force at that moment. That guard also meant a file taken back out of TREK's trash stayed on local_deleted for good, which it already did before: no transfer, no upload, no flag, and a second deletion of the same file would never have propagated. The row now leaves that state by what became of the provider copy: - still there: the pairing counts again and the row is judged like any synced one, so an upstream edit made meanwhile comes down; - binned by TREK: the pairing is dropped and the file goes up again as a new document through the ordinary push, keeping its trek_doc_uid; - deleted by somebody else: flagged remote_missing, never uploaded over their deletion. Telling the last two apart needs a record of what TREK did, so the new remote_trashed_at column (migration 239) is written when TREK bins a copy and cleared when the pairing is back, or when the binned copy is listed again because somebody restored it at the provider. A file purged from the trash came straight back as well: file_id is ON DELETE SET NULL, and the row read as a download that never landed. A tombstone now stays a tombstone, and a purge no run had seen yet is treated as the deletion it is. Also drops snapshotHash, which nothing read.
This commit is contained in:
Maurice
@@ -5125,6 +5125,27 @@ function runMigrations(db: Database.Database): void {
|
||||
CREATE INDEX IF NOT EXISTS idx_document_sync_items_due ON document_sync_items(link_id, next_attempt_at);
|
||||
`);
|
||||
},
|
||||
|
||||
/*
|
||||
* Document providers: when TREK itself put a provider copy in the bin.
|
||||
*
|
||||
* A file deleted in TREK under the `trash` policy takes its provider copy
|
||||
* with it. Taken back out of TREK's trash, it has to go up again; a copy
|
||||
* somebody else deleted in the meantime must not. Both leave the same gap
|
||||
* in a listing, so the difference is written down when TREK acts rather
|
||||
* than guessed at later.
|
||||
*
|
||||
* No backfill from the binding's current policy: that policy may not be
|
||||
* the one the deletion ran under, and reading it back is the retroactive
|
||||
* mistake this column exists to avoid. A row left NULL is treated like a
|
||||
* copy somebody else deleted, which flags it instead of uploading it.
|
||||
*/
|
||||
() => {
|
||||
const hasColumn = db
|
||||
.prepare("SELECT 1 FROM pragma_table_info('document_sync_items') WHERE name = 'remote_trashed_at'")
|
||||
.get();
|
||||
if (!hasColumn) db.exec('ALTER TABLE document_sync_items ADD COLUMN remote_trashed_at TEXT');
|
||||
},
|
||||
];
|
||||
|
||||
if (currentVersion < migrations.length) {
|
||||
|
||||
@@ -14,20 +14,7 @@ import type { RemoteDocument } from './document-provider';
|
||||
* service does nothing but execute the plan.
|
||||
*/
|
||||
|
||||
// ── Identity and fingerprints ────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* A stable fingerprint over an explicit field order.
|
||||
*
|
||||
* The order is written out rather than derived from the object for the reason
|
||||
* dawarich.helpers.ts gives: a hash over `Object.keys` changes when someone
|
||||
* adds a field or a provider reorders its JSON, and a changed hash means
|
||||
* "changed upstream" to every later run. Explicit means a reviewer can see what
|
||||
* participates.
|
||||
*/
|
||||
export function snapshotHash(parts: readonly (string | number | null | undefined)[]): string {
|
||||
return crypto.createHash('sha256').update(parts.map((p) => String(p ?? '')).join('\u0000')).digest('hex');
|
||||
}
|
||||
// ── Identity and incoming names ──────────────────────────────────────────────
|
||||
|
||||
/** Per-document anchor, written into provider metadata where there is room. */
|
||||
export function newTrekDocUid(): string {
|
||||
@@ -108,6 +95,13 @@ export interface SyncItemState {
|
||||
/** When the backoff lets this row be tried again; null once it is shelved. */
|
||||
nextAttemptAt: string | null;
|
||||
remoteMissingAt: string | null;
|
||||
/**
|
||||
* When TREK itself put the provider copy in the recycle bin, under the
|
||||
* `trash` policy. A copy binned by TREK and one deleted by somebody else
|
||||
* leave the same gap in a listing; only this tells them apart once the file
|
||||
* comes back out of TREK's trash.
|
||||
*/
|
||||
remoteTrashedAt: string | null;
|
||||
}
|
||||
|
||||
/** What TREK currently holds for this trip. */
|
||||
@@ -138,6 +132,12 @@ export type PlanAction =
|
||||
| { kind: 'mark_remote_missing'; itemId: number }
|
||||
/** Deleted in TREK; what happens upstream is the link's delete policy. */
|
||||
| { kind: 'local_deleted'; itemId: number; remoteId: string }
|
||||
/** Back out of TREK's trash while the provider copy stayed: the pairing counts again. */
|
||||
| { kind: 'local_restored'; itemId: number }
|
||||
/** Back out of TREK's trash after TREK binned the copy: unpaired, so it goes up as a new document. */
|
||||
| { kind: 'detach'; itemId: number }
|
||||
/** The copy TREK binned is listed again: somebody restored it at the provider. */
|
||||
| { kind: 'remote_restored'; itemId: number }
|
||||
/** Nothing to do but the row should stop looking stale. */
|
||||
| { kind: 'touch'; itemId: number; remote: RemoteDocument | null };
|
||||
|
||||
@@ -195,7 +195,7 @@ export interface ReconcileInput {
|
||||
* or ping-pong forever.
|
||||
*/
|
||||
export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
const { items, remote, local, direction, remoteTruncated, stableRemoteIds, maxAttempts, conflictPolicy } = input;
|
||||
const { remote, local, direction, remoteTruncated, stableRemoteIds, maxAttempts, conflictPolicy } = input;
|
||||
const now = input.now ?? new Date().toISOString().replace('T', ' ').slice(0, 19);
|
||||
const actions: PlanAction[] = [];
|
||||
|
||||
@@ -213,7 +213,7 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
* which is what "waits for a person" means.
|
||||
*/
|
||||
const blocked = new Set<number>();
|
||||
for (const it of items) {
|
||||
for (const it of input.items) {
|
||||
if (it.state !== 'error') continue;
|
||||
if (it.attempts >= maxAttempts) { blocked.add(it.id); continue; }
|
||||
if (it.nextAttemptAt !== null && it.nextAttemptAt > now) blocked.add(it.id);
|
||||
@@ -260,6 +260,39 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
const localById = new Map<number, LocalDocument>();
|
||||
for (const l of local) localById.set(l.fileId, l);
|
||||
|
||||
/**
|
||||
* A file taken back out of TREK's trash.
|
||||
*
|
||||
* A deletion TREK has acted on is otherwise left alone for good (see the
|
||||
* pair loop), so this is the one way out of `local_deleted`, and which way
|
||||
* depends on the provider copy. If it stayed, the pairing counts again and
|
||||
* the row is judged like any synced one from here on: an edit made upstream
|
||||
* in the meantime comes down, and a copy somebody else deleted is flagged as
|
||||
* missing instead of being uploaded over their deletion. If TREK binned it,
|
||||
* TREK took it away and the person asked for it back, so the pairing is
|
||||
* dropped and the file goes up as a new document through the ordinary push.
|
||||
*
|
||||
* The returned row is what the rest of the plan sees, so it has to match
|
||||
* what the executor writes for the action added here.
|
||||
*/
|
||||
const reopen = (it: SyncItemState): SyncItemState => {
|
||||
if (it.state !== 'local_deleted' || !it.remoteId || it.fileId === null) return it;
|
||||
const l = localById.get(it.fileId);
|
||||
if (!l || l.deletedAt) return it;
|
||||
const listed = !input.remoteUnchanged && remoteById.has(it.remoteId);
|
||||
if (it.remoteTrashedAt === null || listed) {
|
||||
add({ kind: 'local_restored', itemId: it.id });
|
||||
return { ...it, state: 'synced', remoteMissingAt: null, remoteTrashedAt: null };
|
||||
}
|
||||
// A pull-only binding cannot upload, and a gap in a truncated listing says
|
||||
// nothing about the copy. The row waits for a run that can decide.
|
||||
if (direction === 'pull' || remoteTruncated) return it;
|
||||
add({ kind: 'detach', itemId: it.id });
|
||||
return { ...it, state: 'pending', remoteId: null, remoteVersion: null, remoteTrashedAt: null };
|
||||
};
|
||||
const items: SyncItemState[] = [];
|
||||
for (const it of input.items) items.push(reopen(it));
|
||||
|
||||
const itemsByRemote = new Map<string, SyncItemState>();
|
||||
const itemsByFile = new Map<number, SyncItemState>();
|
||||
for (const it of items) {
|
||||
@@ -271,9 +304,14 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
// remote half of the plan would be reading a list the adapter did not fetch.
|
||||
if (input.remoteUnchanged) {
|
||||
for (const it of items) {
|
||||
if (it.fileId === null || !it.remoteId) continue;
|
||||
const l = localById.get(it.fileId);
|
||||
if (l?.deletedAt) add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
// A deletion already acted on stays acted on, as in the pair loop. A copy
|
||||
// on record as gone is absent from a full listing too, so a full run
|
||||
// plans nothing for it until it comes back; binning it here would also
|
||||
// write the gap down as TREK's doing, and a restore would then upload
|
||||
// what somebody else deleted.
|
||||
if (!it.remoteId || it.state === 'local_deleted' || it.state === 'remote_missing') continue;
|
||||
const l = it.fileId !== null ? localById.get(it.fileId) : undefined;
|
||||
if (l?.deletedAt || lostItsFile(it)) add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
}
|
||||
if (direction !== 'pull') {
|
||||
for (const l of local) {
|
||||
@@ -315,7 +353,21 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
|
||||
const localChanged = !!l && l.sha256 !== null && it.contentSha256 !== null && l.sha256 !== it.contentSha256;
|
||||
|
||||
if (l?.deletedAt) {
|
||||
/**
|
||||
* A deletion is acted on once, under the policy in force at that moment.
|
||||
*
|
||||
* This used to be planned again on every run for as long as the file sat
|
||||
* in TREK's trash. Under `unlink` nothing showed; switching the binding to
|
||||
* `trash` later then binned every document ever deleted in TREK, months
|
||||
* back included, and a binding on `trash` from the start asked the
|
||||
* provider to bin the same copy on every run. A row still here is one
|
||||
* `reopen` did not take back: its file is still in the trash or purged.
|
||||
*/
|
||||
if (it.state === 'local_deleted') {
|
||||
if (it.remoteTrashedAt !== null) add({ kind: 'remote_restored', itemId: it.id });
|
||||
continue;
|
||||
}
|
||||
if (l?.deletedAt || lostItsFile(it)) {
|
||||
add({ kind: 'local_deleted', itemId: it.id, remoteId: it.remoteId });
|
||||
continue;
|
||||
}
|
||||
@@ -444,6 +496,19 @@ export function planReconcile(input: ReconcileInput): ReconcilePlan {
|
||||
return { actions, massDeleteGuardTripped: guardTripped, missingCount: vanished.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* A file purged from TREK's trash before any run saw it deleted.
|
||||
*
|
||||
* `file_id` is ON DELETE SET NULL, so the row outlives the file with nothing
|
||||
* but a NULL to show for it. Read as a download that never landed, which is
|
||||
* what happened until now, the document came straight back and the delete
|
||||
* policy never ran. Only states a row holds together with a file count here: a
|
||||
* first download that failed has no file either, and is retried.
|
||||
*/
|
||||
function lostItsFile(it: SyncItemState): boolean {
|
||||
return it.fileId === null && (it.state === 'synced' || it.state === 'conflict' || it.state === 'remote_missing');
|
||||
}
|
||||
|
||||
/** Backoff lookup that saturates instead of running off the end of the curve. */
|
||||
export function backoffSeconds(curve: readonly number[], failures: number): number {
|
||||
if (failures <= 0) return curve[0];
|
||||
|
||||
@@ -277,12 +277,54 @@ export class DocSyncService {
|
||||
return 'ok';
|
||||
}
|
||||
case 'local_deleted': {
|
||||
if (ctx.link.delete_policy === 'trash') {
|
||||
const binned = ctx.link.delete_policy === 'trash';
|
||||
if (binned) {
|
||||
const res = await ctx.provider.trash(ctx.ref, ctx.scope, action.remoteId);
|
||||
if (docFailed(res)) return res.error.code;
|
||||
}
|
||||
// The policy is applied here and never again: the planner leaves a
|
||||
// `local_deleted` row alone, so a policy changed later cannot reach
|
||||
// back. Whether TREK binned the copy is written down with it, because a
|
||||
// restore in TREK later has to know whether the gap upstream is TREK's.
|
||||
this.db.connection
|
||||
.prepare("UPDATE document_sync_items SET state = 'local_deleted', last_seen_at = CURRENT_TIMESTAMP WHERE id = ?")
|
||||
.prepare(
|
||||
`UPDATE document_sync_items
|
||||
SET state = 'local_deleted', remote_missing_at = NULL,
|
||||
remote_trashed_at = CASE WHEN ? = 1 THEN CURRENT_TIMESTAMP ELSE NULL END,
|
||||
last_seen_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(binned ? 1 : 0, action.itemId);
|
||||
return 'ok';
|
||||
}
|
||||
case 'local_restored': {
|
||||
this.db.connection
|
||||
.prepare(
|
||||
`UPDATE document_sync_items
|
||||
SET state = 'synced', remote_missing_at = NULL, remote_trashed_at = NULL, last_seen_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(action.itemId);
|
||||
return 'ok';
|
||||
}
|
||||
case 'detach': {
|
||||
// The trek_doc_uid stays: it is the same document to TREK, only the
|
||||
// provider copy is new. The push the planner queued after this fills
|
||||
// the pairing in again, and if that push fails the row is an ordinary
|
||||
// unpaired file that the next run retries.
|
||||
this.db.connection
|
||||
.prepare(
|
||||
`UPDATE document_sync_items
|
||||
SET state = 'pending', remote_id = NULL, remote_version = NULL, remote_trashed_at = NULL,
|
||||
error_code = NULL, last_seen_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?`,
|
||||
)
|
||||
.run(action.itemId);
|
||||
return 'ok';
|
||||
}
|
||||
case 'remote_restored': {
|
||||
this.db.connection
|
||||
.prepare('UPDATE document_sync_items SET remote_trashed_at = NULL, last_seen_at = CURRENT_TIMESTAMP WHERE id = ?')
|
||||
.run(action.itemId);
|
||||
return 'ok';
|
||||
}
|
||||
@@ -546,7 +588,7 @@ export class DocSyncService {
|
||||
const rows = this.db.connection
|
||||
.prepare(
|
||||
`SELECT id, file_id, trek_doc_uid, remote_id, remote_version, remote_name, content_sha256,
|
||||
pushed_sha256, state, attempts, next_attempt_at, remote_missing_at
|
||||
pushed_sha256, state, attempts, next_attempt_at, remote_missing_at, remote_trashed_at
|
||||
FROM document_sync_items WHERE link_id = ?`,
|
||||
)
|
||||
.all(linkId) as Array<Record<string, unknown>>;
|
||||
@@ -563,6 +605,7 @@ export class DocSyncService {
|
||||
attempts: Number(r.attempts ?? 0),
|
||||
nextAttemptAt: r.next_attempt_at === null || r.next_attempt_at === undefined ? null : String(r.next_attempt_at),
|
||||
remoteMissingAt: r.remote_missing_at === null ? null : String(r.remote_missing_at),
|
||||
remoteTrashedAt: r.remote_trashed_at === null ? null : String(r.remote_trashed_at),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -913,7 +956,8 @@ export class DocSyncService {
|
||||
.prepare(
|
||||
`SELECT link_id,
|
||||
SUM(CASE WHEN file_id IS NOT NULL AND remote_id IS NOT NULL AND state = 'synced' THEN 1 ELSE 0 END) AS paired,
|
||||
SUM(CASE WHEN remote_id IS NOT NULL AND state != 'remote_missing' THEN 1 ELSE 0 END) AS atProvider,
|
||||
SUM(CASE WHEN remote_id IS NOT NULL AND state != 'remote_missing' AND remote_trashed_at IS NULL
|
||||
THEN 1 ELSE 0 END) AS atProvider,
|
||||
SUM(CASE WHEN state = 'remote_missing' THEN 1 ELSE 0 END) AS missing
|
||||
FROM document_sync_items
|
||||
WHERE trip_id = ?
|
||||
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
isBlockedName,
|
||||
planReconcile,
|
||||
sanitizeIncomingName,
|
||||
snapshotHash,
|
||||
type LocalDocument,
|
||||
type SyncItemState,
|
||||
} from '../../../../src/nest/doc-sync/doc-sync.helpers';
|
||||
@@ -54,6 +53,7 @@ const item = (over: Partial<SyncItemState> = {}): SyncItemState => ({
|
||||
attempts: 0,
|
||||
nextAttemptAt: null,
|
||||
remoteMissingAt: null,
|
||||
remoteTrashedAt: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
@@ -329,18 +329,6 @@ describe('isAllowedByOperator', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('snapshotHash', () => {
|
||||
it('is stable for the same field order and differs when a field changes', () => {
|
||||
expect(snapshotHash(['a', 1, null])).toBe(snapshotHash(['a', 1, null]));
|
||||
expect(snapshotHash(['a', 1, null])).not.toBe(snapshotHash(['a', 2, null]));
|
||||
});
|
||||
|
||||
it('does not collide when field boundaries shift', () => {
|
||||
// Joining without a separator would make ['ab','c'] and ['a','bc'] equal.
|
||||
expect(snapshotHash(['ab', 'c'])).not.toBe(snapshotHash(['a', 'bc']));
|
||||
});
|
||||
});
|
||||
|
||||
describe('backoffSeconds', () => {
|
||||
it('saturates at the end of the curve instead of running off it', () => {
|
||||
expect(backoffSeconds([60, 300], 1)).toBe(60);
|
||||
@@ -636,3 +624,152 @@ describe('planReconcile > a provider name TREK had to clean up', () => {
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['rename_local']);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A deletion in TREK is acted on once.
|
||||
*
|
||||
* Both branches used to plan `local_deleted` for as long as the file sat in
|
||||
* TREK's trash, so the policy was applied again on every run. Under `unlink`
|
||||
* that was invisible until the binding was switched to `trash`: the next run
|
||||
* then binned every document ever deleted in TREK.
|
||||
*/
|
||||
describe('planReconcile > a deletion TREK already acted on', () => {
|
||||
const settled = (over: Partial<SyncItemState> = {}) =>
|
||||
item({ id: 50, fileId: 5, remoteId: 'r5', state: 'local_deleted', ...over });
|
||||
const inTrash = local({ fileId: 5, deletedAt: '2026-09-18 08:00:00' });
|
||||
|
||||
it('plans nothing more while the file sits in the trash', () => {
|
||||
const p = plan({ items: [settled()], remote: [remote({ remoteId: 'r5' })], local: [inTrash] });
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('plans nothing more under an unchanged upstream either', () => {
|
||||
const p = plan({ items: [settled()], remote: [], local: [inTrash], remoteUnchanged: true });
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('does not download the document again once the file is purged', () => {
|
||||
// file_id is ON DELETE SET NULL, and a row without a file used to read as
|
||||
// a download that never landed.
|
||||
const p = plan({ items: [settled({ fileId: null })], remote: [remote({ remoteId: 'r5' })], local: [] });
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('reads a file purged before any run saw it deleted as a deletion', () => {
|
||||
const p = plan({ items: [item({ fileId: null, state: 'synced' })], remote: [remote()], local: [] });
|
||||
expect(p.actions).toEqual([{ kind: 'local_deleted', itemId: 10, remoteId: 'r1' }]);
|
||||
});
|
||||
|
||||
it('does the same under an unchanged upstream', () => {
|
||||
const p = plan({ items: [item({ fileId: null, state: 'synced' })], remote: [], local: [], remoteUnchanged: true });
|
||||
expect(p.actions).toEqual([{ kind: 'local_deleted', itemId: 10, remoteId: 'r1' }]);
|
||||
});
|
||||
|
||||
it('leaves a copy already on record as gone alone under an unchanged upstream', () => {
|
||||
// Binning it would record the gap as TREK's doing, and a restore would
|
||||
// later upload a document somebody else deleted.
|
||||
const p = plan({
|
||||
items: [item({ state: 'remote_missing', remoteMissingAt: '2026-09-17 08:00:00' })],
|
||||
remote: [],
|
||||
local: [local({ deletedAt: '2026-09-18 08:00:00' })],
|
||||
remoteUnchanged: true,
|
||||
});
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('notices when the copy TREK binned is listed again', () => {
|
||||
const p = plan({
|
||||
items: [settled({ remoteTrashedAt: '2026-09-18 08:05:00' })],
|
||||
remote: [remote({ remoteId: 'r5' })],
|
||||
local: [inTrash],
|
||||
});
|
||||
expect(p.actions).toEqual([{ kind: 'remote_restored', itemId: 50 }]);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A file taken back out of TREK's trash.
|
||||
*
|
||||
* Which way the row goes depends on what became of the provider copy while
|
||||
* TREK's sat in the bin: still there, binned by TREK itself, or deleted by
|
||||
* somebody else.
|
||||
*/
|
||||
describe('planReconcile > a file taken back out of TREK trash', () => {
|
||||
const settled = (over: Partial<SyncItemState> = {}) =>
|
||||
item({ id: 50, fileId: 5, remoteId: 'r5', state: 'local_deleted', ...over });
|
||||
const binnedByTrek = (over: Partial<SyncItemState> = {}) =>
|
||||
settled({ remoteTrashedAt: '2026-09-18 08:05:00', ...over });
|
||||
const back = local({ fileId: 5 });
|
||||
|
||||
it('resumes the pairing when the provider copy stayed', () => {
|
||||
const p = plan({ items: [settled()], remote: [remote({ remoteId: 'r5' })], local: [back] });
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['local_restored', 'touch']);
|
||||
});
|
||||
|
||||
it('brings down an edit made upstream in the meantime, as for any synced row', () => {
|
||||
const p = plan({ items: [settled()], remote: [remote({ remoteId: 'r5', remoteVersion: 'v2' })], local: [back] });
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['local_restored', 'pull_update']);
|
||||
});
|
||||
|
||||
it('flags a copy somebody else deleted meanwhile instead of uploading it', () => {
|
||||
const p = plan({ items: [settled()], remote: [], local: [back] });
|
||||
expect(p.actions).toEqual([
|
||||
{ kind: 'local_restored', itemId: 50 },
|
||||
{ kind: 'mark_remote_missing', itemId: 50 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('resumes under an unchanged upstream and leaves the listing to a later run', () => {
|
||||
const p = plan({ items: [settled()], remote: [], local: [back], remoteUnchanged: true });
|
||||
expect(p.actions).toEqual([{ kind: 'local_restored', itemId: 50 }]);
|
||||
});
|
||||
|
||||
it('uploads it again when TREK was the one that binned the copy', () => {
|
||||
const p = plan({ items: [binnedByTrek()], remote: [], local: [back] });
|
||||
expect(p.actions).toEqual([
|
||||
{ kind: 'detach', itemId: 50 },
|
||||
{ kind: 'push', local: back, itemId: 50, remoteId: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it('does so under an unchanged upstream too', () => {
|
||||
const p = plan({ items: [binnedByTrek()], remote: [], local: [back], remoteUnchanged: true });
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['detach', 'push']);
|
||||
});
|
||||
|
||||
it('is not held back by the echo guard: nothing about it is a remote change', () => {
|
||||
const p = plan({ items: [binnedByTrek({ pushedSha256: 'aaa' })], remote: [], local: [local({ fileId: 5, sha256: 'aaa' })] });
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['detach', 'push']);
|
||||
});
|
||||
|
||||
it('waits on a pull-only binding, which cannot upload', () => {
|
||||
const p = plan({ items: [binnedByTrek()], remote: [], local: [back], direction: 'pull' });
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('waits on a truncated listing, where a gap proves nothing', () => {
|
||||
const p = plan({ items: [binnedByTrek()], remote: [], local: [back], remoteTruncated: true });
|
||||
expect(p.actions).toEqual([]);
|
||||
});
|
||||
|
||||
it('resumes instead when the binned copy was restored at the provider as well', () => {
|
||||
const p = plan({ items: [binnedByTrek()], remote: [remote({ remoteId: 'r5' })], local: [back] });
|
||||
expect(p.actions.map(a => a.kind)).toEqual(['local_restored', 'touch']);
|
||||
});
|
||||
|
||||
it('reads many restores after TREK binned them as uploads, not as a mass deletion', () => {
|
||||
const rows = Array.from({ length: 10 }, (_, i) =>
|
||||
binnedByTrek({ id: 100 + i, fileId: 100 + i, remoteId: `r${100 + i}` }));
|
||||
const p = plan({ items: rows, remote: [], local: rows.map((r) => local({ fileId: r.fileId as number })) });
|
||||
expect(p.massDeleteGuardTripped).toBe(false);
|
||||
expect(p.actions.filter(a => a.kind === 'detach')).toHaveLength(10);
|
||||
expect(p.actions.filter(a => a.kind === 'push')).toHaveLength(10);
|
||||
});
|
||||
|
||||
it('counts many restored rows whose copies vanished like any synced rows, guard included', () => {
|
||||
const rows = Array.from({ length: 10 }, (_, i) =>
|
||||
settled({ id: 100 + i, fileId: 100 + i, remoteId: `r${100 + i}` }));
|
||||
const p = plan({ items: rows, remote: [], local: rows.map((r) => local({ fileId: r.fileId as number })) });
|
||||
expect(p.massDeleteGuardTripped).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1245,6 +1245,283 @@ describe('DocSyncService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A deletion in TREK is acted on once, under the policy of that moment.
|
||||
*
|
||||
* Both planner branches used to emit `local_deleted` for as long as the file
|
||||
* sat in TREK's trash. Under `unlink` that was invisible, until somebody
|
||||
* switched the binding to `trash`: the next run then binned every document
|
||||
* ever deleted in TREK, months back included.
|
||||
*/
|
||||
describe('a deletion in TREK', () => {
|
||||
function deletedPairing(link: LinkRow): { itemId: number; fileId: number } {
|
||||
const fileId = makeFile({ name: 'boarding.pdf', deletedAt: '2026-09-18 08:00:00' });
|
||||
const itemId = seedItem(link, {
|
||||
remoteId: 'r1', remoteName: 'boarding.pdf', remoteVersion: 'v1', fileId, state: 'synced',
|
||||
contentSha256: FILE_SHA, pushedSha256: FILE_SHA,
|
||||
});
|
||||
return { itemId, fileId };
|
||||
}
|
||||
|
||||
it('does not reach back when the binding is switched from unlink to trash later', async () => {
|
||||
const link = makeLink({ deletePolicy: 'unlink' });
|
||||
const { itemId } = deletedPairing(link);
|
||||
provider.list.mockResolvedValue(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
|
||||
await service.syncLink(link);
|
||||
expect(itemRow(itemId).state).toBe('local_deleted');
|
||||
expect(itemRow(itemId).remote_trashed_at).toBeNull();
|
||||
|
||||
testDb.prepare("UPDATE trip_document_links SET delete_policy = 'trash' WHERE id = ?").run(link.id);
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.trash).not.toHaveBeenCalled();
|
||||
expect(itemRow(itemId).state).toBe('local_deleted');
|
||||
});
|
||||
|
||||
it('bins the provider copy exactly once, even while the provider keeps listing it', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const { itemId } = deletedPairing(link);
|
||||
provider.list.mockResolvedValue(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
|
||||
await service.syncLink(link);
|
||||
expect(itemRow(itemId).remote_trashed_at).not.toBeNull();
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.trash).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not ask again under an unchanged upstream either', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
deletedPairing(link);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
await service.syncLink(link);
|
||||
|
||||
provider.list.mockResolvedValue(ok(listing([], { cursorUnchanged: true })));
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.trash).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Taking a file back out of TREK's trash.
|
||||
*
|
||||
* `FilesService.restoreFile` only clears `deleted_at`; nothing in the files
|
||||
* domain knows about sync, so the next run is what finds out. Three cases,
|
||||
* told apart by what happened to the provider copy while TREK's sat in the
|
||||
* bin: still there, binned by TREK itself, or gone by somebody else's hand.
|
||||
*/
|
||||
describe('a file taken back out of TREK trash', () => {
|
||||
const restore = (fileId: number) =>
|
||||
testDb.prepare('UPDATE trip_files SET deleted_at = NULL WHERE id = ?').run(fileId);
|
||||
|
||||
/** A synced pairing whose file was deleted in TREK and whose deletion one run has seen. */
|
||||
async function deletedAndSeen(link: LinkRow): Promise<{ itemId: number; fileId: number }> {
|
||||
const fileId = makeFile({ name: 'boarding.pdf', deletedAt: '2026-09-18 08:00:00' });
|
||||
const itemId = seedItem(link, {
|
||||
remoteId: 'r1', remoteName: 'boarding.pdf', remoteVersion: 'v1', fileId, state: 'synced',
|
||||
contentSha256: FILE_SHA, pushedSha256: FILE_SHA,
|
||||
});
|
||||
provider.list.mockResolvedValueOnce(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
await service.syncLink(link);
|
||||
expect(itemRow(itemId).state).toBe('local_deleted');
|
||||
return { itemId, fileId };
|
||||
}
|
||||
|
||||
it('resumes the pairing without a transfer when the provider copy is still there', async () => {
|
||||
const link = makeLink({ deletePolicy: 'unlink' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(run.state).toBe('ok');
|
||||
expect(provider.fetch).not.toHaveBeenCalled();
|
||||
expect(provider.push).not.toHaveBeenCalled();
|
||||
expect(itemRow(itemId).state).toBe('synced');
|
||||
expect(fileRows().map((f) => [f.id, f.deleted_at])).toEqual([[fileId, null]]);
|
||||
});
|
||||
|
||||
it('lets the ordinary rules bring down an edit made upstream in the meantime', async () => {
|
||||
const link = makeLink({ deletePolicy: 'unlink' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([remoteDoc({ remoteId: 'r1', remoteVersion: 'v2' })])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(run.pulled).toBe(1);
|
||||
const row = itemRow(itemId);
|
||||
expect(row.state).toBe('synced');
|
||||
expect(row.file_id).not.toBe(fileId);
|
||||
expect(fileRows().filter((f) => f.deleted_at === null)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('puts the file back at the provider when TREK was the one that binned it', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
expect(provider.trash).toHaveBeenCalledTimes(1);
|
||||
const uid = itemRow(itemId).trek_doc_uid;
|
||||
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(run.pushed).toBe(1);
|
||||
expect(provider.push.mock.calls[0][2]).toMatchObject({ fileName: 'boarding.pdf', remoteId: undefined, trekDocUid: uid });
|
||||
const row = itemRow(itemId);
|
||||
expect(row.state).toBe('synced');
|
||||
expect(row.remote_id).toBe('r-pushed');
|
||||
expect(row.file_id).toBe(fileId);
|
||||
expect(row.remote_trashed_at).toBeNull();
|
||||
expect(itemRows()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not mistake the re-upload for a foreign change on the run after', async () => {
|
||||
// The echo guard must still recognise TREK's own write once the provider
|
||||
// reports it back under a new version marker.
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([])));
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
provider.list.mockResolvedValueOnce(ok(listing([
|
||||
remoteDoc({ remoteId: 'r-pushed', name: 'boarding.pdf', remoteVersion: 'v2', contentHash: FILE_SHA }),
|
||||
])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(run.pulled).toBe(0);
|
||||
expect(provider.fetch).not.toHaveBeenCalled();
|
||||
expect(provider.push).toHaveBeenCalledTimes(1);
|
||||
expect(itemRow(itemId).state).toBe('synced');
|
||||
});
|
||||
|
||||
it('retries a re-upload that failed like any other unpaired file', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValue(ok(listing([])));
|
||||
provider.push.mockResolvedValueOnce(fail('unreachable'));
|
||||
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
const failed = itemRow(itemId);
|
||||
expect(failed.state).toBe('error');
|
||||
expect(failed.remote_id).toBeNull();
|
||||
|
||||
testDb.prepare('UPDATE document_sync_items SET next_attempt_at = NULL WHERE id = ?').run(itemId);
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.push).toHaveBeenCalledTimes(2);
|
||||
expect(itemRow(itemId).state).toBe('synced');
|
||||
expect(itemRow(itemId).remote_id).toBe('r-pushed');
|
||||
});
|
||||
|
||||
it('uploads many files TREK had binned instead of reading them as a mass deletion', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const fileIds = Array.from({ length: 10 }, (_, i) =>
|
||||
makeFile({ name: `doc-${i}.pdf`, storageKey: `key-${i}`, deletedAt: '2026-09-18 08:00:00' }));
|
||||
fileIds.forEach((fileId, i) =>
|
||||
seedItem(link, { remoteId: `r${i}`, remoteName: `doc-${i}.pdf`, remoteVersion: 'v1', fileId, state: 'synced' }));
|
||||
provider.list.mockResolvedValueOnce(ok(listing(fileIds.map((_, i) => remoteDoc({ remoteId: `r${i}`, name: `doc-${i}.pdf` })))));
|
||||
await service.syncLink(link);
|
||||
expect(provider.trash).toHaveBeenCalledTimes(10);
|
||||
|
||||
for (const fileId of fileIds) restore(fileId);
|
||||
let seq = 0;
|
||||
provider.push.mockImplementation(async () => {
|
||||
seq += 1;
|
||||
return ok({ remoteId: `r-new-${seq}`, remoteVersion: 'v1', remoteModifiedAt: null, deduplicated: false });
|
||||
});
|
||||
provider.list.mockResolvedValueOnce(ok(listing([])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(run.errorCode).toBeUndefined();
|
||||
expect(run.state).toBe('ok');
|
||||
expect(run.pushed).toBe(10);
|
||||
expect(itemRows().every((r) => r.state === 'synced')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not upload a copy TREK binned that was restored at the provider meanwhile', async () => {
|
||||
// Seen while TREK's file was still in the trash, so the note that TREK
|
||||
// binned it is dropped. Without that, a later restore under an unchanged
|
||||
// upstream (no listing to look at) would upload a second copy.
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
|
||||
provider.list.mockResolvedValueOnce(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
expect(itemRow(itemId).remote_trashed_at).toBeNull();
|
||||
expect(provider.fetch).not.toHaveBeenCalled();
|
||||
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([], { cursorUnchanged: true })));
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.push).not.toHaveBeenCalled();
|
||||
expect(itemRow(itemId).state).toBe('synced');
|
||||
expect(itemRow(itemId).remote_id).toBe('r1');
|
||||
});
|
||||
|
||||
it('flags the copy as missing when it vanished upstream by somebody else', async () => {
|
||||
const link = makeLink({ deletePolicy: 'unlink' });
|
||||
const { itemId, fileId } = await deletedAndSeen(link);
|
||||
|
||||
restore(fileId);
|
||||
provider.list.mockResolvedValueOnce(ok(listing([])));
|
||||
const run = await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.push).not.toHaveBeenCalled();
|
||||
expect(run.missing).toBe(1);
|
||||
expect(itemRow(itemId).state).toBe('remote_missing');
|
||||
expect(service.issues(tripId).map((r) => r.id)).toContain(itemId);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Purging leaves the row behind with `file_id` NULL (ON DELETE SET NULL), and
|
||||
* that row read as a download that never landed: under `unlink` the document
|
||||
* came back on the next run, and a purge no run had seen yet skipped the
|
||||
* delete policy entirely.
|
||||
*/
|
||||
describe('a file purged from TREK trash', () => {
|
||||
const purge = (fileId: number) => testDb.prepare('DELETE FROM trip_files WHERE id = ?').run(fileId);
|
||||
|
||||
it('does not download a document again once its deletion was seen', async () => {
|
||||
const link = makeLink({ deletePolicy: 'unlink' });
|
||||
const fileId = makeFile({ name: 'boarding.pdf', deletedAt: '2026-09-18 08:00:00' });
|
||||
const itemId = seedItem(link, { remoteId: 'r1', remoteName: 'boarding.pdf', remoteVersion: 'v1', fileId, state: 'synced' });
|
||||
provider.list.mockResolvedValue(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
await service.syncLink(link);
|
||||
|
||||
purge(fileId);
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.fetch).not.toHaveBeenCalled();
|
||||
expect(fileRows()).toHaveLength(0);
|
||||
expect(itemRow(itemId).state).toBe('local_deleted');
|
||||
});
|
||||
|
||||
it('applies the delete policy when the purge came before any run saw the deletion', async () => {
|
||||
const link = makeLink({ deletePolicy: 'trash' });
|
||||
const fileId = makeFile({ name: 'boarding.pdf', deletedAt: '2026-09-18 08:00:00' });
|
||||
const itemId = seedItem(link, { remoteId: 'r1', remoteName: 'boarding.pdf', remoteVersion: 'v1', fileId, state: 'synced' });
|
||||
purge(fileId);
|
||||
provider.list.mockResolvedValue(ok(listing([remoteDoc({ remoteId: 'r1' })])));
|
||||
|
||||
await service.syncLink(link);
|
||||
await service.syncLink(config.getLink(link.id));
|
||||
|
||||
expect(provider.fetch).not.toHaveBeenCalled();
|
||||
expect(provider.trash).toHaveBeenCalledTimes(1);
|
||||
expect(fileRows()).toHaveLength(0);
|
||||
expect(itemRow(itemId).state).toBe('local_deleted');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveConflict across trips', () => {
|
||||
function conflictedItem(linkId: number, tripOfItem: number): number {
|
||||
return Number(testDb.prepare(
|
||||
|
||||
Reference in New Issue
Block a user