mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-06-19 13:21:46 +00:00
fix(db): scope, evict, and cap the offline blob cache (H3) (#1178)
Blob cache previously leaked forever: clearTripData omitted it, entries had no trip discriminator, and there was no size/count bound, so file blobs survived trip eviction and could starve the map-tile cache for quota. - BlobCacheEntry gains tripId + bytes; Dexie v3 adds a tripId index with a backfill upgrade (legacy rows -> tripId -1, bytes from blob.size) - clearTripData purges the trip's blobs in-transaction - enforceBlobBudget() evicts oldest-by-cachedAt past 200 entries / 100 MB - tripSyncManager threads tripId/bytes into puts and enforces the budget
This commit is contained in:
@@ -47,7 +47,15 @@ export interface SyncMeta {
|
|||||||
export interface BlobCacheEntry {
|
export interface BlobCacheEntry {
|
||||||
/** Relative URL, e.g. "/api/files/42/download" */
|
/** Relative URL, e.g. "/api/files/42/download" */
|
||||||
url: string;
|
url: string;
|
||||||
|
/**
|
||||||
|
* Trip this blob belongs to, so it is evicted together with the trip in
|
||||||
|
* clearTripData. Legacy rows cached before v3 carry the sentinel -1.
|
||||||
|
*/
|
||||||
|
tripId: number;
|
||||||
blob: Blob;
|
blob: Blob;
|
||||||
|
/** Byte size captured at insert time — Blob.size is not reliably preserved
|
||||||
|
* across IndexedDB round-trips, so the LRU budget reads this instead. */
|
||||||
|
bytes: number;
|
||||||
mime: string;
|
mime: string;
|
||||||
cachedAt: number;
|
cachedAt: number;
|
||||||
}
|
}
|
||||||
@@ -121,6 +129,17 @@ class TrekOfflineDb extends Dexie {
|
|||||||
tags: 'id',
|
tags: 'id',
|
||||||
categories: 'id',
|
categories: 'id',
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// v3: scope the blob cache by trip so it can be evicted with the trip and
|
||||||
|
// bounded by an LRU budget (see enforceBlobBudget).
|
||||||
|
this.version(3).stores({
|
||||||
|
blobCache: 'url, cachedAt, tripId',
|
||||||
|
}).upgrade(async (tx) => {
|
||||||
|
await tx.table('blobCache').toCollection().modify((row: Partial<BlobCacheEntry>) => {
|
||||||
|
if (row.tripId == null) row.tripId = -1;
|
||||||
|
if (row.bytes == null) row.bytes = row.blob?.size ?? 0;
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -245,6 +264,40 @@ export async function getCachedBlob(url: string): Promise<Blob | null> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Blob-cache budget ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upper bounds for the offline file-blob cache. Kept conservative so trip
|
||||||
|
* documents never starve the map-tile cache (sized at MAX_TILES in
|
||||||
|
* tilePrefetcher.ts) for the origin's storage quota.
|
||||||
|
*/
|
||||||
|
export const BLOB_CACHE_MAX_ENTRIES = 200;
|
||||||
|
export const BLOB_CACHE_MAX_BYTES = 100 * 1024 * 1024; // 100 MB
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Evict oldest-by-cachedAt blobs until the cache is under both the entry-count
|
||||||
|
* and byte budget. Call after inserting new blobs. LRU on insertion time, which
|
||||||
|
* is a reasonable proxy for access for write-once document blobs.
|
||||||
|
*/
|
||||||
|
export async function enforceBlobBudget(
|
||||||
|
maxCount = BLOB_CACHE_MAX_ENTRIES,
|
||||||
|
maxBytes = BLOB_CACHE_MAX_BYTES,
|
||||||
|
): Promise<void> {
|
||||||
|
const entries = await offlineDb.blobCache.orderBy('cachedAt').toArray();
|
||||||
|
let count = entries.length;
|
||||||
|
let totalBytes = entries.reduce((sum, e) => sum + (e.bytes ?? 0), 0);
|
||||||
|
if (count <= maxCount && totalBytes <= maxBytes) return;
|
||||||
|
|
||||||
|
const toDelete: string[] = [];
|
||||||
|
for (const e of entries) {
|
||||||
|
if (count <= maxCount && totalBytes <= maxBytes) break;
|
||||||
|
toDelete.push(e.url);
|
||||||
|
totalBytes -= e.bytes ?? 0;
|
||||||
|
count -= 1;
|
||||||
|
}
|
||||||
|
if (toDelete.length) await offlineDb.blobCache.bulkDelete(toDelete);
|
||||||
|
}
|
||||||
|
|
||||||
// ── Eviction / cleanup ────────────────────────────────────────────────────────
|
// ── Eviction / cleanup ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
/** Delete all cached data for one trip (eviction or explicit clear). */
|
/** Delete all cached data for one trip (eviction or explicit clear). */
|
||||||
@@ -263,6 +316,7 @@ export async function clearTripData(tripId: number): Promise<void> {
|
|||||||
offlineDb.tripMembers,
|
offlineDb.tripMembers,
|
||||||
offlineDb.mutationQueue,
|
offlineDb.mutationQueue,
|
||||||
offlineDb.syncMeta,
|
offlineDb.syncMeta,
|
||||||
|
offlineDb.blobCache,
|
||||||
],
|
],
|
||||||
async () => {
|
async () => {
|
||||||
await offlineDb.days.where('trip_id').equals(tripId).delete();
|
await offlineDb.days.where('trip_id').equals(tripId).delete();
|
||||||
@@ -276,6 +330,7 @@ export async function clearTripData(tripId: number): Promise<void> {
|
|||||||
await offlineDb.tripMembers.where('tripId').equals(tripId).delete();
|
await offlineDb.tripMembers.where('tripId').equals(tripId).delete();
|
||||||
await offlineDb.mutationQueue.where('tripId').equals(tripId).delete();
|
await offlineDb.mutationQueue.where('tripId').equals(tripId).delete();
|
||||||
await offlineDb.syncMeta.where('tripId').equals(tripId).delete();
|
await offlineDb.syncMeta.where('tripId').equals(tripId).delete();
|
||||||
|
await offlineDb.blobCache.where('tripId').equals(tripId).delete();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
// Remove the trip row itself outside the transaction since it's a separate table
|
// Remove the trip row itself outside the transaction since it's a separate table
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import {
|
|||||||
upsertCategories,
|
upsertCategories,
|
||||||
upsertSyncMeta,
|
upsertSyncMeta,
|
||||||
clearTripData,
|
clearTripData,
|
||||||
|
enforceBlobBudget,
|
||||||
} from '../db/offlineDb'
|
} from '../db/offlineDb'
|
||||||
import { prefetchTilesForTrip } from './tilePrefetcher'
|
import { prefetchTilesForTrip } from './tilePrefetcher'
|
||||||
import { isAuthed } from './authGate'
|
import { isAuthed } from './authGate'
|
||||||
@@ -109,13 +110,16 @@ async function cacheFilesForTrip(files: TripFile[]): Promise<void> {
|
|||||||
const resp = await fetch(file.url!, { credentials: 'include' })
|
const resp = await fetch(file.url!, { credentials: 'include' })
|
||||||
if (!resp.ok) continue
|
if (!resp.ok) continue
|
||||||
const blob = await resp.blob()
|
const blob = await resp.blob()
|
||||||
await offlineDb.blobCache.put({ url: file.url!, blob, mime: file.mime_type, cachedAt: Date.now() })
|
await offlineDb.blobCache.put({ url: file.url!, tripId: file.trip_id, blob, bytes: blob.size, mime: file.mime_type, cachedAt: Date.now() })
|
||||||
cached++
|
cached++
|
||||||
} catch {
|
} catch {
|
||||||
// Network failure — skip this file, will retry next sync
|
// Network failure — skip this file, will retry next sync
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Keep the blob cache within its size/count budget after adding new files.
|
||||||
|
if (cached > 0) await enforceBlobBudget().catch(() => {})
|
||||||
|
|
||||||
// Update filesCachedCount in syncMeta
|
// Update filesCachedCount in syncMeta
|
||||||
const tripId = files[0]?.trip_id
|
const tripId = files[0]?.trip_id
|
||||||
if (tripId) {
|
if (tripId) {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import {
|
|||||||
reopenForUser,
|
reopenForUser,
|
||||||
reopenAnonymous,
|
reopenAnonymous,
|
||||||
deleteCurrentUserDb,
|
deleteCurrentUserDb,
|
||||||
|
enforceBlobBudget,
|
||||||
type QueuedMutation,
|
type QueuedMutation,
|
||||||
type SyncMeta,
|
type SyncMeta,
|
||||||
type BlobCacheEntry,
|
type BlobCacheEntry,
|
||||||
@@ -84,6 +85,15 @@ const makePlace = (id: number, tripId = 1): Place => ({
|
|||||||
created_at: '2026-01-01T00:00:00Z',
|
created_at: '2026-01-01T00:00:00Z',
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const makeBlob = (url: string, tripId = 1, bytes = 10, cachedAt = 1): BlobCacheEntry => ({
|
||||||
|
url,
|
||||||
|
tripId,
|
||||||
|
blob: new Blob(['x'.repeat(bytes)], { type: 'application/pdf' }),
|
||||||
|
bytes,
|
||||||
|
mime: 'application/pdf',
|
||||||
|
cachedAt,
|
||||||
|
});
|
||||||
|
|
||||||
// ── Lifecycle ─────────────────────────────────────────────────────────────────
|
// ── Lifecycle ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
@@ -223,7 +233,9 @@ describe('offlineDb — blobCache', () => {
|
|||||||
const blob = new Blob(['%PDF-1.4 test'], { type: 'application/pdf' });
|
const blob = new Blob(['%PDF-1.4 test'], { type: 'application/pdf' });
|
||||||
const entry: BlobCacheEntry = {
|
const entry: BlobCacheEntry = {
|
||||||
url: '/api/files/99/download',
|
url: '/api/files/99/download',
|
||||||
|
tripId: 1,
|
||||||
blob,
|
blob,
|
||||||
|
bytes: blob.size,
|
||||||
mime: 'application/pdf',
|
mime: 'application/pdf',
|
||||||
cachedAt: Date.now(),
|
cachedAt: Date.now(),
|
||||||
};
|
};
|
||||||
@@ -234,6 +246,49 @@ describe('offlineDb — blobCache', () => {
|
|||||||
expect(stored!.mime).toBe('application/pdf');
|
expect(stored!.mime).toBe('application/pdf');
|
||||||
expect(stored!.blob).toBeDefined();
|
expect(stored!.blob).toBeDefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('queries blobs by tripId index', async () => {
|
||||||
|
await offlineDb.blobCache.bulkPut([
|
||||||
|
makeBlob('/api/files/1/download', 1),
|
||||||
|
makeBlob('/api/files/2/download', 1),
|
||||||
|
makeBlob('/api/files/3/download', 2),
|
||||||
|
]);
|
||||||
|
const trip1 = await offlineDb.blobCache.where('tripId').equals(1).toArray();
|
||||||
|
expect(trip1).toHaveLength(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('offlineDb — enforceBlobBudget', () => {
|
||||||
|
it('evicts oldest-by-cachedAt entries past the count budget', async () => {
|
||||||
|
// 5 entries with strictly increasing cachedAt; cap to 3.
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
await offlineDb.blobCache.put(makeBlob(`/api/files/${i}/download`, 1, 10, i + 1));
|
||||||
|
}
|
||||||
|
await enforceBlobBudget(3, Infinity);
|
||||||
|
|
||||||
|
expect(await offlineDb.blobCache.count()).toBe(3);
|
||||||
|
// Oldest two (cachedAt 1 and 2) are gone; newest survive.
|
||||||
|
expect(await offlineDb.blobCache.get('/api/files/0/download')).toBeUndefined();
|
||||||
|
expect(await offlineDb.blobCache.get('/api/files/1/download')).toBeUndefined();
|
||||||
|
expect(await offlineDb.blobCache.get('/api/files/4/download')).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('evicts oldest entries past the byte budget', async () => {
|
||||||
|
// 3 entries of 100 bytes each; cap to 250 bytes → newest two (200) survive.
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
await offlineDb.blobCache.put(makeBlob(`/api/files/${i}/download`, 1, 100, i + 1));
|
||||||
|
}
|
||||||
|
await enforceBlobBudget(Infinity, 250);
|
||||||
|
|
||||||
|
expect(await offlineDb.blobCache.count()).toBe(2);
|
||||||
|
expect(await offlineDb.blobCache.get('/api/files/0/download')).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('is a no-op when already within budget', async () => {
|
||||||
|
await offlineDb.blobCache.put(makeBlob('/api/files/1/download', 1));
|
||||||
|
await enforceBlobBudget(10, Infinity);
|
||||||
|
expect(await offlineDb.blobCache.count()).toBe(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('offlineDb — clearTripData', () => {
|
describe('offlineDb — clearTripData', () => {
|
||||||
@@ -244,9 +299,12 @@ describe('offlineDb — clearTripData', () => {
|
|||||||
const item: PackingItem = { id: 5, trip_id: 1, name: 'Towel', category: null, checked: 0, sort_order: 0, quantity: 1 };
|
const item: PackingItem = { id: 5, trip_id: 1, name: 'Towel', category: null, checked: 0, sort_order: 0, quantity: 1 };
|
||||||
await upsertPackingItems([item]);
|
await upsertPackingItems([item]);
|
||||||
|
|
||||||
|
await offlineDb.blobCache.put(makeBlob('/api/files/1/download', 1));
|
||||||
|
|
||||||
// Also add data for a different trip — should NOT be removed
|
// Also add data for a different trip — should NOT be removed
|
||||||
await upsertTrip(makeTrip(2));
|
await upsertTrip(makeTrip(2));
|
||||||
await upsertDays([makeDay(99, 2)]);
|
await upsertDays([makeDay(99, 2)]);
|
||||||
|
await offlineDb.blobCache.put(makeBlob('/api/files/2/download', 2));
|
||||||
|
|
||||||
await clearTripData(1);
|
await clearTripData(1);
|
||||||
|
|
||||||
@@ -254,10 +312,12 @@ describe('offlineDb — clearTripData', () => {
|
|||||||
expect(await offlineDb.days.where('trip_id').equals(1).count()).toBe(0);
|
expect(await offlineDb.days.where('trip_id').equals(1).count()).toBe(0);
|
||||||
expect(await offlineDb.places.where('trip_id').equals(1).count()).toBe(0);
|
expect(await offlineDb.places.where('trip_id').equals(1).count()).toBe(0);
|
||||||
expect(await offlineDb.packingItems.where('trip_id').equals(1).count()).toBe(0);
|
expect(await offlineDb.packingItems.where('trip_id').equals(1).count()).toBe(0);
|
||||||
|
expect(await offlineDb.blobCache.where('tripId').equals(1).count()).toBe(0);
|
||||||
|
|
||||||
// Trip 2 intact
|
// Trip 2 intact
|
||||||
expect(await offlineDb.trips.get(2)).toBeDefined();
|
expect(await offlineDb.trips.get(2)).toBeDefined();
|
||||||
expect(await offlineDb.days.where('trip_id').equals(2).count()).toBe(1);
|
expect(await offlineDb.days.where('trip_id').equals(2).count()).toBe(1);
|
||||||
|
expect(await offlineDb.blobCache.get('/api/files/2/download')).toBeDefined();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user