mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-06-19 13:21:46 +00:00
feat(photos): add 1h disk cache for remote thumbnails and keep tabs mounted
Closes #686 - Add trekPhotoCache service: SHA1-keyed disk cache under uploads/photos/trek/, 1h TTL, in-flight dedup map to prevent stampedes on concurrent requests - Add migration 108: trek_photo_cache_meta table - Hook cache into streamPhoto for Immich/Synology thumbnail path; originals bypass cache - Add fetchImmichThumbnailBytes / fetchSynologyThumbnailBytes returning Buffer instead of piping, used by the cache layer - Add scheduler entry (every 2h + startup sweep) to evict expired disk files and DB rows via sweepExpired() - Client: convert journey tab conditional-mount to hidden-toggle so img elements stay in DOM across tab switches, preventing redundant thumbnail requests on rapid tab changes - Expose invalidateSize() on JourneyMapHandle; call it on map tab activation to fix Leaflet rendering in previously-hidden container
This commit is contained in:
@@ -1673,6 +1673,15 @@ function runMigrations(db: Database.Database): void {
|
||||
)
|
||||
`);
|
||||
}},
|
||||
// Migration 108: Disk cache metadata for remote-provider photo thumbnails (Immich / Synology)
|
||||
() => db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS trek_photo_cache_meta (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
content_type TEXT NOT NULL DEFAULT 'image/jpeg',
|
||||
fetched_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_trek_photo_cache_meta_fetched_at ON trek_photo_cache_meta (fetched_at);
|
||||
`),
|
||||
];
|
||||
|
||||
if (currentVersion < migrations.length) {
|
||||
|
||||
@@ -49,6 +49,7 @@ const server = app.listen(PORT, () => {
|
||||
scheduler.startVersionCheck();
|
||||
scheduler.startDemoReset();
|
||||
scheduler.startIdempotencyCleanup();
|
||||
scheduler.startTrekPhotoCacheCleanup();
|
||||
const { startTokenCleanup } = require('./services/ephemeralTokens');
|
||||
startTokenCleanup();
|
||||
import('./websocket').then(({ setupWebSocket }) => {
|
||||
|
||||
+25
-1
@@ -248,12 +248,36 @@ function startIdempotencyCleanup(): void {
|
||||
}, { timezone: tz });
|
||||
}
|
||||
|
||||
// Trek photo cache cleanup: every 2 hours — evict disk files and DB rows past their 1h TTL
|
||||
let trekPhotoCacheTask: ScheduledTask | null = null;
|
||||
|
||||
function startTrekPhotoCacheCleanup(): void {
|
||||
if (trekPhotoCacheTask) { trekPhotoCacheTask.stop(); trekPhotoCacheTask = null; }
|
||||
|
||||
// Run once immediately on startup to evict any entries left over from a previous run
|
||||
try {
|
||||
const { sweepExpired } = require('./services/memories/trekPhotoCache');
|
||||
sweepExpired();
|
||||
} catch { /* cache dir may not exist yet — harmless */ }
|
||||
|
||||
trekPhotoCacheTask = cron.schedule('0 */2 * * *', () => {
|
||||
try {
|
||||
const { sweepExpired } = require('./services/memories/trekPhotoCache');
|
||||
sweepExpired();
|
||||
} catch (err: unknown) {
|
||||
const { logError: le } = require('./services/auditLog');
|
||||
le(`Trek photo cache cleanup: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (currentTask) { currentTask.stop(); currentTask = null; }
|
||||
if (demoTask) { demoTask.stop(); demoTask = null; }
|
||||
if (reminderTask) { reminderTask.stop(); reminderTask = null; }
|
||||
if (versionCheckTask) { versionCheckTask.stop(); versionCheckTask = null; }
|
||||
if (idempotencyCleanupTask) { idempotencyCleanupTask.stop(); idempotencyCleanupTask = null; }
|
||||
if (trekPhotoCacheTask) { trekPhotoCacheTask.stop(); trekPhotoCacheTask = null; }
|
||||
}
|
||||
|
||||
export { start, stop, startDemoReset, startTripReminders, startVersionCheck, startIdempotencyCleanup, loadSettings, saveSettings, VALID_INTERVALS };
|
||||
export { start, stop, startDemoReset, startTripReminders, startVersionCheck, startIdempotencyCleanup, startTrekPhotoCacheCleanup, loadSettings, saveSettings, VALID_INTERVALS };
|
||||
|
||||
@@ -230,6 +230,30 @@ export async function getAssetInfo(
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchImmichThumbnailBytes(
|
||||
userId: number,
|
||||
assetId: string,
|
||||
ownerUserId?: number
|
||||
): Promise<{ bytes: Buffer; contentType: string } | { error: string; status: number }> {
|
||||
const effectiveUserId = ownerUserId ?? userId;
|
||||
const creds = getImmichCredentials(effectiveUserId);
|
||||
if (!creds) return { error: 'Not found', status: 404 };
|
||||
|
||||
const url = `${creds.immich_url}/api/assets/${assetId}/thumbnail?size=thumbnail`;
|
||||
try {
|
||||
const resp = await safeFetch(url, {
|
||||
headers: { 'x-api-key': creds.immich_api_key },
|
||||
signal: AbortSignal.timeout(10000) as any,
|
||||
});
|
||||
if (!resp.ok) return { error: 'Upstream error', status: resp.status };
|
||||
const contentType = resp.headers.get('content-type') || 'image/jpeg';
|
||||
const bytes = Buffer.from(await resp.arrayBuffer());
|
||||
return { bytes, contentType };
|
||||
} catch {
|
||||
return { error: 'Proxy error', status: 502 };
|
||||
}
|
||||
}
|
||||
|
||||
export async function streamImmichAsset(
|
||||
response: Response,
|
||||
userId: number,
|
||||
|
||||
@@ -3,11 +3,12 @@ import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { db } from '../../db/database';
|
||||
import type { TrekPhoto } from '../../types';
|
||||
import { streamImmichAsset, getAssetInfo as getImmichAssetInfo } from './immichService';
|
||||
import { streamSynologyAsset, getSynologyAssetInfo } from './synologyService';
|
||||
import { streamImmichAsset, fetchImmichThumbnailBytes, getAssetInfo as getImmichAssetInfo } from './immichService';
|
||||
import { streamSynologyAsset, fetchSynologyThumbnailBytes, getSynologyAssetInfo } from './synologyService';
|
||||
import type { ServiceResult, AssetInfo } from './helpersService';
|
||||
import { fail, success } from './helpersService';
|
||||
import { encrypt_api_key, decrypt_api_key } from '../apiKeyCrypto';
|
||||
import * as photoCache from './trekPhotoCache';
|
||||
|
||||
// ── Lookup / Register ────────────────────────────────────────────────────
|
||||
|
||||
@@ -57,6 +58,36 @@ export function resolveTrekPhoto(photoId: number): TrekPhoto | null {
|
||||
|
||||
// ── Streaming ────────────────────────────────────────────────────────────
|
||||
|
||||
async function streamCachedThumbnail(
|
||||
res: Response,
|
||||
photo: TrekPhoto,
|
||||
fetchBytes: () => Promise<{ bytes: Buffer; contentType: string } | { error: string; status: number }>,
|
||||
fallback: () => Promise<unknown>,
|
||||
): Promise<void> {
|
||||
const key = photoCache.cacheKey(photo.provider!, photo.asset_id!, 'thumbnail', photo.owner_id!);
|
||||
|
||||
if (photoCache.serveFresh(res, key)) return;
|
||||
|
||||
const existing = photoCache.getInFlight(key);
|
||||
if (existing) {
|
||||
const bytes = await existing;
|
||||
if (bytes && photoCache.serveFresh(res, key)) return;
|
||||
await fallback();
|
||||
return;
|
||||
}
|
||||
|
||||
const promise = fetchBytes().then(async result => {
|
||||
if ('error' in result) return null;
|
||||
await photoCache.put(key, result.bytes, result.contentType);
|
||||
return result.bytes;
|
||||
});
|
||||
photoCache.setInFlight(key, promise);
|
||||
|
||||
const bytes = await promise;
|
||||
if (bytes && photoCache.serveFresh(res, key)) return;
|
||||
await fallback();
|
||||
}
|
||||
|
||||
export async function streamPhoto(
|
||||
res: Response,
|
||||
userId: number,
|
||||
@@ -84,11 +115,27 @@ export async function streamPhoto(
|
||||
return;
|
||||
}
|
||||
case 'immich': {
|
||||
if (kind === 'thumbnail') {
|
||||
await streamCachedThumbnail(
|
||||
res, photo,
|
||||
() => fetchImmichThumbnailBytes(userId, photo.asset_id!, photo.owner_id!),
|
||||
() => streamImmichAsset(res, userId, photo.asset_id!, kind, photo.owner_id!),
|
||||
);
|
||||
return;
|
||||
}
|
||||
await streamImmichAsset(res, userId, photo.asset_id!, kind, photo.owner_id!);
|
||||
return;
|
||||
}
|
||||
case 'synologyphotos': {
|
||||
const passphrase = photo.passphrase ? (decrypt_api_key(photo.passphrase) || undefined) : undefined;
|
||||
if (kind === 'thumbnail') {
|
||||
await streamCachedThumbnail(
|
||||
res, photo,
|
||||
() => fetchSynologyThumbnailBytes(userId, photo.owner_id!, photo.asset_id!, passphrase),
|
||||
() => streamSynologyAsset(res, userId, photo.owner_id!, photo.asset_id!, kind, undefined, passphrase),
|
||||
);
|
||||
return;
|
||||
}
|
||||
await streamSynologyAsset(res, userId, photo.owner_id!, photo.asset_id!, kind, undefined, passphrase);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -604,6 +604,47 @@ export async function getSynologyAssetInfo(userId: number, photoId: string, targ
|
||||
return success(normalized);
|
||||
}
|
||||
|
||||
export async function fetchSynologyThumbnailBytes(
|
||||
userId: number,
|
||||
targetUserId: number,
|
||||
photoId: string,
|
||||
passphrase?: string,
|
||||
): Promise<{ bytes: Buffer; contentType: string } | { error: string; status: number }> {
|
||||
const parsedId = _splitPackedSynologyId(photoId);
|
||||
if (!parsedId) return { error: 'Invalid photo ID format', status: 400 };
|
||||
|
||||
const synology_credentials = _getSynologyCredentials(targetUserId);
|
||||
if (!synology_credentials.success) return { error: 'Credentials error', status: 500 };
|
||||
|
||||
const sid = await _getSynologySession(targetUserId);
|
||||
if (!sid.success) return { error: 'Session error', status: 500 };
|
||||
if (!sid.data) return { error: 'Session ID missing', status: 500 };
|
||||
|
||||
const params = new URLSearchParams({
|
||||
api: 'SYNO.Foto.Thumbnail',
|
||||
method: 'get',
|
||||
version: '2',
|
||||
mode: 'download',
|
||||
id: parsedId.id,
|
||||
type: 'unit',
|
||||
size: 'sm',
|
||||
cache_key: parsedId.cacheKey,
|
||||
_sid: sid.data,
|
||||
});
|
||||
if (passphrase) params.append('passphrase', passphrase);
|
||||
|
||||
const url = _buildSynologyEndpoint(synology_credentials.data.synology_url, params.toString());
|
||||
try {
|
||||
const resp = await safeFetch(url);
|
||||
if (!resp.ok) return { error: 'Upstream error', status: resp.status };
|
||||
const contentType = resp.headers.get('content-type') || 'image/jpeg';
|
||||
const bytes = Buffer.from(await resp.arrayBuffer());
|
||||
return { bytes, contentType };
|
||||
} catch {
|
||||
return { error: 'Proxy error', status: 502 };
|
||||
}
|
||||
}
|
||||
|
||||
export async function streamSynologyAsset(
|
||||
response: Response,
|
||||
userId: number,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import fsPromises from 'node:fs/promises';
|
||||
import crypto from 'node:crypto';
|
||||
import { Response } from 'express';
|
||||
import { db } from '../../db/database';
|
||||
|
||||
const TREK_PHOTO_DIR = path.join(__dirname, '../../../uploads/photos/trek');
|
||||
export const CACHE_TTL = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
const inFlight = new Map<string, Promise<Buffer | null>>();
|
||||
|
||||
export function cacheKey(provider: string, assetId: string, kind: string, ownerId: number): string {
|
||||
return crypto.createHash('sha1').update(`${provider}:${assetId}:${kind}:${ownerId}`).digest('hex');
|
||||
}
|
||||
|
||||
function ensureDir(): void {
|
||||
if (!fs.existsSync(TREK_PHOTO_DIR)) {
|
||||
fs.mkdirSync(TREK_PHOTO_DIR, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function cachedFilePath(key: string): string {
|
||||
return path.join(TREK_PHOTO_DIR, `${key}.bin`);
|
||||
}
|
||||
|
||||
export function getFresh(key: string): { filePath: string; contentType: string } | null {
|
||||
const row = db.prepare(
|
||||
'SELECT content_type, fetched_at FROM trek_photo_cache_meta WHERE cache_key = ?'
|
||||
).get(key) as { content_type: string; fetched_at: number } | undefined;
|
||||
|
||||
if (!row) return null;
|
||||
|
||||
if (Date.now() - row.fetched_at >= CACHE_TTL) {
|
||||
db.prepare('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?').run(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
const fp = cachedFilePath(key);
|
||||
if (!fs.existsSync(fp)) {
|
||||
db.prepare('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?').run(key);
|
||||
return null;
|
||||
}
|
||||
|
||||
return { filePath: fp, contentType: row.content_type };
|
||||
}
|
||||
|
||||
export async function put(key: string, bytes: Buffer, contentType: string): Promise<void> {
|
||||
ensureDir();
|
||||
const fp = cachedFilePath(key);
|
||||
const tmp = fp + '.tmp';
|
||||
|
||||
await fsPromises.writeFile(tmp, bytes);
|
||||
await fsPromises.rename(tmp, fp);
|
||||
|
||||
db.prepare(
|
||||
'INSERT OR REPLACE INTO trek_photo_cache_meta (cache_key, content_type, fetched_at) VALUES (?, ?, ?)'
|
||||
).run(key, contentType, Date.now());
|
||||
}
|
||||
|
||||
export function serveFresh(res: Response, key: string): boolean {
|
||||
const entry = getFresh(key);
|
||||
if (!entry) return false;
|
||||
|
||||
res.set('Content-Type', entry.contentType);
|
||||
res.set('Cache-Control', 'public, max-age=3600');
|
||||
res.sendFile(entry.filePath);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function getInFlight(key: string): Promise<Buffer | null> | undefined {
|
||||
return inFlight.get(key);
|
||||
}
|
||||
|
||||
export function setInFlight(key: string, promise: Promise<Buffer | null>): void {
|
||||
inFlight.set(key, promise);
|
||||
promise.finally(() => inFlight.delete(key));
|
||||
}
|
||||
|
||||
export function sweepExpired(): void {
|
||||
const cutoff = Date.now() - CACHE_TTL * 2;
|
||||
const stale = db.prepare(
|
||||
'SELECT cache_key FROM trek_photo_cache_meta WHERE fetched_at < ?'
|
||||
).all(cutoff) as { cache_key: string }[];
|
||||
|
||||
for (const row of stale) {
|
||||
db.prepare('DELETE FROM trek_photo_cache_meta WHERE cache_key = ?').run(row.cache_key);
|
||||
const fp = cachedFilePath(row.cache_key);
|
||||
if (fs.existsSync(fp)) fs.unlinkSync(fp);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user