diff --git a/server/scripts/repository-owners.json b/server/scripts/repository-owners.json index ed87027ab..b8b1db67a 100644 --- a/server/scripts/repository-owners.json +++ b/server/scripts/repository-owners.json @@ -130,6 +130,7 @@ "TourWaypoints": "tours", "Tours": "tours", "TrekPhotoCacheMeta": "memories", + "TrekPhotoSignals": "photos", "TrekPhotos": "photos", "TripAlbumLinks": "memories", "TripDocumentLinks": "doc-sync", diff --git a/server/src/db/entities/TrekPhotoSignals.entity.ts b/server/src/db/entities/TrekPhotoSignals.entity.ts new file mode 100644 index 000000000..120f68e56 --- /dev/null +++ b/server/src/db/entities/TrekPhotoSignals.entity.ts @@ -0,0 +1,38 @@ +import { EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core'; +import { TrekPhotoSignalsRepository } from '../repositories/TrekPhotoSignals.repository'; +import { DbTimestampType } from '../types'; +import { TrekPhotos } from './TrekPhotos.entity'; + +export class TrekPhotoSignals { + [EntityRepositoryType]?: TrekPhotoSignalsRepository; + id!: number & Opt; + photo!: Ref; + photo_id!: number; + algo!: number; + phash?: string | null; + sharpness?: number | null; + luma?: number | null; + clipped?: number | null; + orig_width?: number | null; + orig_height?: number | null; + computed_at!: string & Opt; +} + +export const TrekPhotoSignalsSchema = defineEntity({ + class: TrekPhotoSignals, + repository: () => TrekPhotoSignalsRepository, + uniques: [{ properties: ['photo'] }], + properties: { + id: p.integer().primary(), + photo: () => p.manyToOne(TrekPhotos).ref().deleteRule('cascade').hidden(), + photo_id: p.integer().persist(false), + algo: p.integer(), + phash: p.text().nullable(), + sharpness: p.double().nullable(), + luma: p.double().nullable(), + clipped: p.double().nullable(), + orig_width: p.integer().nullable(), + orig_height: p.integer().nullable(), + computed_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`), + }, +}); diff --git a/server/src/db/entities/TrekPhotos.entity.ts b/server/src/db/entities/TrekPhotos.entity.ts index 6c2c969de..eb78b5b80 100644 --- a/server/src/db/entities/TrekPhotos.entity.ts +++ b/server/src/db/entities/TrekPhotos.entity.ts @@ -2,6 +2,7 @@ import { Collection, EntityRepositoryType, type Opt, type Ref, defineEntity, p } import { TrekPhotosRepository } from '../repositories/TrekPhotos.repository'; import { DbTimestampType } from '../types'; import { JourneyPhotos } from './JourneyPhotos.entity'; +import { TrekPhotoSignals } from './TrekPhotoSignals.entity'; import { TripPhotos } from './TripPhotos.entity'; import { Users } from './Users.entity'; @@ -24,6 +25,7 @@ export class TrekPhotos { lat?: number | null; lng?: number | null; journey_photos_collection = new Collection(this); + trek_photo_signals_collection = new Collection(this); trip_photos_collection = new Collection(this); } @@ -62,6 +64,7 @@ export const TrekPhotosSchema = defineEntity({ lat: p.double().nullable(), lng: p.double().nullable(), journey_photos_collection: () => p.oneToMany(JourneyPhotos).mappedBy('photo').hidden(), + trek_photo_signals_collection: () => p.oneToMany(TrekPhotoSignals).mappedBy('photo').hidden(), trip_photos_collection: () => p.oneToMany(TripPhotos).mappedBy('photo').hidden(), }, }); diff --git a/server/src/db/entities/index.ts b/server/src/db/entities/index.ts index 90bfa807c..417a27e05 100644 --- a/server/src/db/entities/index.ts +++ b/server/src/db/entities/index.ts @@ -129,6 +129,7 @@ import { TourTypesSchema } from './TourTypes.entity'; import { TourWaypointsSchema } from './TourWaypoints.entity'; import { ToursSchema } from './Tours.entity'; import { TrekPhotoCacheMetaSchema } from './TrekPhotoCacheMeta.entity'; +import { TrekPhotoSignalsSchema } from './TrekPhotoSignals.entity'; import { TrekPhotosSchema } from './TrekPhotos.entity'; import { TripAlbumLinksSchema } from './TripAlbumLinks.entity'; import { TripDocumentLinksSchema } from './TripDocumentLinks.entity'; @@ -289,6 +290,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [ TourWaypointsSchema, ToursSchema, TrekPhotoCacheMetaSchema, + TrekPhotoSignalsSchema, TrekPhotosSchema, TripAlbumLinksSchema, TripDocumentLinksSchema, diff --git a/server/src/db/kysely/TrekPhotoSignals.table.ts b/server/src/db/kysely/TrekPhotoSignals.table.ts new file mode 100644 index 000000000..47c53c680 --- /dev/null +++ b/server/src/db/kysely/TrekPhotoSignals.table.ts @@ -0,0 +1,16 @@ +// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand: +// change the migration, then run `npm run gen:db-types`. +import type { InsertOptional } from './columns'; + +export interface TrekPhotoSignalsTable { + id: InsertOptional; + photo_id: number; + algo: number; + phash: InsertOptional; + sharpness: InsertOptional; + luma: InsertOptional; + clipped: InsertOptional; + orig_width: InsertOptional; + orig_height: InsertOptional; + computed_at: InsertOptional; +} diff --git a/server/src/db/kysely/db.ts b/server/src/db/kysely/db.ts index baeaf5068..6e2d401fc 100644 --- a/server/src/db/kysely/db.ts +++ b/server/src/db/kysely/db.ts @@ -135,6 +135,7 @@ import type { TourTypesTable } from './TourTypes.table'; import type { TourWaypointsTable } from './TourWaypoints.table'; import type { ToursTable } from './Tours.table'; import type { TrekPhotoCacheMetaTable } from './TrekPhotoCacheMeta.table'; +import type { TrekPhotoSignalsTable } from './TrekPhotoSignals.table'; import type { TrekPhotosTable } from './TrekPhotos.table'; import type { TripAlbumLinksTable } from './TripAlbumLinks.table'; import type { TripDocumentLinksTable } from './TripDocumentLinks.table'; @@ -301,6 +302,7 @@ export interface DB { tour_waypoints: TourWaypointsTable; tours: ToursTable; trek_photo_cache_meta: TrekPhotoCacheMetaTable; + trek_photo_signals: TrekPhotoSignalsTable; trek_photos: TrekPhotosTable; trip_album_links: TripAlbumLinksTable; trip_document_links: TripDocumentLinksTable; diff --git a/server/src/db/migrations/Migration20200101046240_a_photo_keeps_its_quality_signals.ts b/server/src/db/migrations/Migration20200101046240_a_photo_keeps_its_quality_signals.ts new file mode 100644 index 000000000..c3a1fff2e --- /dev/null +++ b/server/src/db/migrations/Migration20200101046240_a_photo_keeps_its_quality_signals.ts @@ -0,0 +1,40 @@ +import { Migration } from '@mikro-orm/migrations'; + +/** + * What a photo's pixels say, measured once so the assistant can pick photos + * for a TREK Studio book without looking at them: a perceptual hash (`phash`, + * 64 bits as 16 hex digits) to find near duplicates, how sharp it is + * (`sharpness`, the variance of a Laplacian over the grey image), its mean + * brightness (`luma`, 0 to 1) and the share of its pixels that are burnt out + * or crushed black (`clipped`, 0 to 1), and the original's size in pixels + * (`orig_width`, `orig_height`) for the print check. + * + * `algo` is the version of the measurement a row was made with, so a better + * one later measures again instead of mixing the two. A null signal is one + * that could not be measured (a format TREK cannot decode, a photo too large to + * read), which is an answer too: the photo is not measured again. + * + * One row per photo, `photo_id UNIQUE` beside its own id rather than the photo + * id as key: a key that is also a foreign key maps to a one-to-one on both + * entities (see `trip_journal_links`). The row goes with its photo. + */ +export class Migration20200101046240_a_photo_keeps_its_quality_signals extends Migration { + override name = 'Migration20200101046240_a_photo_keeps_its_quality_signals'; + + override up(): void { + this.addSql(` + CREATE TABLE IF NOT EXISTS trek_photo_signals ( + id INTEGER PRIMARY KEY, + photo_id INTEGER NOT NULL UNIQUE REFERENCES trek_photos(id) ON DELETE CASCADE, + algo INTEGER NOT NULL, + phash TEXT, + sharpness REAL, + luma REAL, + clipped REAL, + orig_width INTEGER, + orig_height INTEGER, + computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + } +} diff --git a/server/src/db/repositories/TrekPhotoSignals.repository.ts b/server/src/db/repositories/TrekPhotoSignals.repository.ts new file mode 100644 index 000000000..e2bc471e5 --- /dev/null +++ b/server/src/db/repositories/TrekPhotoSignals.repository.ts @@ -0,0 +1,123 @@ +import type { TrekPhotoSignals } from '../entities/TrekPhotoSignals.entity'; +import type { DB } from '../kysely/db'; +import { dbNow } from '../types'; +import { TrekRepository } from './_shared/trek-repository'; + +type SignalsDB = Pick; + +/** What one measurement of a photo found; a null is a signal that could not be measured. */ +export interface TrekPhotoSignalWrite { + photo_id: number; + algo: number; + phash: string | null; + sharpness: number | null; + luma: number | null; + clipped: number | null; + orig_width: number | null; + orig_height: number | null; +} + +/** A stored measurement, with when it was made. */ +export interface TrekPhotoSignalRow extends TrekPhotoSignalWrite { + computed_at: string; +} + +/** A photo of a journey's gallery as the photo picker reads it: no bytes, no paths. */ +export interface GalleryPhotoFacts { + photo_id: number; + caption: string | null; + provider: string; + media_type: string; + width: number | null; + height: number | null; + taken_at: string | null; + lat: number | null; + lng: number | null; +} + +/** SQLite binds at most so many values per statement; a gallery can hold more photos. */ +const IN_CHUNK = 400; + +/** + * `trek_photo_signals`: what a photo's pixels say, one row per photo, written + * by `PhotoSignalsService` once its worker has measured the photo and read + * again whenever photos are picked for a book. A new measurement of a photo + * replaces the old one (upsert on `photo_id`). + */ +export class TrekPhotoSignalsRepository extends TrekRepository { + private db() { + return this.kysely(); + } + + /** The measurements stored for these photos, whatever version made them; a photo without one is left out. */ + async listForPhotos(photoIds: number[]): Promise { + const out: TrekPhotoSignalRow[] = []; + for (let i = 0; i < photoIds.length; i += IN_CHUNK) { + const rows = await this.db() + .selectFrom('trek_photo_signals') + .select([ + 'photo_id', + 'algo', + 'phash', + 'sharpness', + 'luma', + 'clipped', + 'orig_width', + 'orig_height', + 'computed_at', + ]) + .where('photo_id', 'in', photoIds.slice(i, i + IN_CHUNK)) + .execute(); + out.push(...rows); + } + return out; + } + + /** Stores a measurement, replacing the photo's earlier one. */ + async saveSignals(row: TrekPhotoSignalWrite): Promise { + const computedAt = dbNow(); + await this.db() + .insertInto('trek_photo_signals') + .values({ ...row, computed_at: computedAt }) + .onConflict((oc) => + oc.column('photo_id').doUpdateSet({ + algo: row.algo, + phash: row.phash, + sharpness: row.sharpness, + luma: row.luma, + clipped: row.clipped, + orig_width: row.orig_width, + orig_height: row.orig_height, + computed_at: computedAt, + }), + ) + .execute(); + } + + /** + * The photos of a journey's gallery with what their rows know about them + * (capture time and place, size, kind), in gallery order. The size is a + * thumbnail's for an uploaded photo and none for a provider's: good for the + * shape, never for print. + */ + async listGalleryFacts(journeyId: number): Promise { + return await this.db() + .selectFrom('journey_photos as gp') + .innerJoin('trek_photos as tp', 'tp.id', 'gp.photo_id') + .select([ + 'gp.photo_id', + 'gp.caption', + 'tp.provider', + 'tp.media_type', + 'tp.width', + 'tp.height', + 'tp.taken_at', + 'tp.lat', + 'tp.lng', + ]) + .where('gp.journey_id', '=', journeyId) + .orderBy('gp.sort_order', 'asc') + .orderBy('gp.id', 'asc') + .execute(); + } +} diff --git a/server/src/nest/memories/photo-resolver.service.ts b/server/src/nest/memories/photo-resolver.service.ts index 46c535a48..9405a538c 100644 --- a/server/src/nest/memories/photo-resolver.service.ts +++ b/server/src/nest/memories/photo-resolver.service.ts @@ -10,6 +10,19 @@ import { TrekPhotoCacheService } from './trek-photo-cache.service'; import { Injectable } from '@nestjs/common'; import { Response } from 'express'; +import fsPromises from 'node:fs/promises'; + +/** + * What a photo's quality signals are measured on (photos/photo-signals.service.ts): + * a local original, or a provider's thumbnail with the original's size as the + * provider reports it. `none` is nothing to measure, `unavailable` a provider + * that did not answer now, which a later try may reach. + */ +export type PhotoSignalSource = + | { kind: 'original'; bytes: Buffer } + | { kind: 'thumbnail'; bytes: Buffer; width: number | null; height: number | null } + | { kind: 'none' } + | { kind: 'unavailable' }; /** * Resolves a stored trek_photo to bytes or metadata by asking whichever provider @@ -150,6 +163,35 @@ export class PhotoResolverService { await provider.streamAsset(res, { ...ref, range }, kind); } + /** + * The bytes a photo's quality signals are measured on, read with its + * owner's access rather than a viewer's: the signals belong to the photo, + * whoever asks. A local photo answers its original whole, so the size it + * decodes to is the original's; a provider photo its thumbnail, since an + * original can be hundreds of megabytes, with the size the provider reports. + * `none` for a photo that is gone, a video, a local original over + * `maxBytes` or missing, a provider TREK does not know. + */ + async readForSignals(photoId: number, maxBytes: number): Promise { + const photo = await this.photos.resolve(photoId); + if (!photo || photo.media_type === 'video') return { kind: 'none' }; + if (photo.file_path) { + const name = this.storageName(photo.file_path); + const stat = name ? await this.storage.stat('journey', name).catch(() => null) : null; + if (!name || !stat || stat.size > maxBytes) return { kind: 'none' }; + const bytes = await this.storage.withLocalFile('journey', name, (p) => fsPromises.readFile(p)); + return { kind: 'original', bytes }; + } + const provider = this.providers.get(photo.provider); + if (!provider || photo.owner_id == null || !photo.asset_id) return { kind: 'none' }; + const ref = this.refFor(photo, photo.owner_id); + const thumbnail = await provider.fetchThumbnailBytes(ref); + if ('error' in thumbnail) return { kind: 'unavailable' }; + const info = await provider.getAssetInfo(ref); + const size = info.success ? info.data : null; + return { kind: 'thumbnail', bytes: thumbnail.bytes, width: size?.width ?? null, height: size?.height ?? null }; + } + /** * The stored row as a provider-shaped request. One place decrypts the * Synology share passphrase, and one place decides which of the two user ids diff --git a/server/src/nest/photos/photo-signals-runner.ts b/server/src/nest/photos/photo-signals-runner.ts new file mode 100644 index 000000000..e643949d8 --- /dev/null +++ b/server/src/nest/photos/photo-signals-runner.ts @@ -0,0 +1,161 @@ +import { Jimp } from 'jimp'; + +/** + * The version of the measurement below. A stored row of another version is + * measured again, so a better measurement never mixes with an older one. + * Version 2 measures every photo at the same `SIGNAL_EDGE`, up or down. + */ +export const PHOTO_SIGNALS_ALGO = 2; + +/** + * The long edge every photo is scaled to, up or down, before it is measured. A + * smaller picture has a higher Laplacian variance per pixel, so sharpness only + * compares between photos measured at one size; this one is no larger than the + * thumbnails the providers hand out (Immich about 250 px, Synology about 320). + */ +export const SIGNAL_EDGE = 240; + +/** An image declaring more than this many megapixels is not decoded at all (a 48 MP decode takes about 200 MB). */ +const MAX_DECODE_MP = 48; + +/** Grey levels at or beyond these count as burnt out or crushed black. */ +const CLIP_LOW = 5; +const CLIP_HIGH = 250; + +/** What a photo's pixels say. `width`/`height` are the decoded image's, before it is scaled to measure. */ +export interface PhotoPixels { + /** The perceptual hash, 64 bits as 16 hex digits; null if the hash came out in another shape. */ + phash: string | null; + /** The variance of the grey Laplacian at `SIGNAL_EDGE`: the higher, the sharper. */ + sharpness: number; + /** The mean grey level, 0 (black) to 1 (white). */ + luma: number; + /** The share of pixels burnt out or crushed black, 0 to 1. */ + clipped: number; + width: number; + height: number; +} + +/** + * Measures one photo. CPU work on a decoded bitmap, so the server runs it in + * its worker thread (photo-signals.worker.ts), never on the event loop. + * Answers null for bytes that are no image TREK decodes (WebP, HEIC, TIFF, a + * video, anything past `MAX_DECODE_MP`): such a photo has no pixel signals. + */ +export async function measurePhoto(bytes: Uint8Array): Promise { + if (!withinDecodeCap(bytes)) return null; + try { + const image = await Jimp.read(Buffer.from(bytes), { 'image/jpeg': { maxResolutionInMP: MAX_DECODE_MP } }); + const { width, height } = image.bitmap; + if (width < 3 || height < 3) return null; + image.scaleToFit({ w: SIGNAL_EDGE, h: SIGNAL_EDGE }); + // A strip far longer than it is wide ends up under 3 pixels across, which leaves no inner pixel. + if (image.bitmap.width < 3 || image.bitmap.height < 3) return null; + const grey = greyStats(image.bitmap.data, image.bitmap.width, image.bitmap.height); + return { phash: bitsToHex(image.pHash()), ...grey, width, height }; + } catch { + return null; + } +} + +/** + * Whether these bytes may be decoded: a JPEG, which Jimp holds to + * `MAX_DECODE_MP` itself, or a PNG, GIF or BMP whose header declares at most + * that many pixels. A small file can declare a canvas of gigabytes, and the + * decoder allocates all of it before the first pixel, so the size is read + * first. Anything else (TIFF among the formats Jimp reads) is not decoded. + */ +export function withinDecodeCap(bytes: Uint8Array): boolean { + const data = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) return true; + const size = declaredSize(data); + return size !== null && size.width * size.height <= MAX_DECODE_MP * 1_000_000; +} + +/** The canvas a PNG (IHDR), GIF (logical screen) or BMP (info or core header) declares; null for anything else. */ +function declaredSize(data: Buffer): { width: number; height: number } | null { + if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47) { + return { width: data.readUInt32BE(16), height: data.readUInt32BE(20) }; + } + if (data.length >= 10 && data.toString('latin1', 0, 4) === 'GIF8') { + return { width: data.readUInt16LE(6), height: data.readUInt16LE(8) }; + } + if (data.length >= 26 && data.toString('latin1', 0, 2) === 'BM') { + // The old core header holds two 16-bit sides, every later one two signed 32-bit ones (negative: top-down). + if (data.readUInt32LE(14) === 12) return { width: data.readUInt16LE(18), height: data.readUInt16LE(20) }; + return { width: Math.abs(data.readInt32LE(18)), height: Math.abs(data.readInt32LE(22)) }; + } + return null; +} + +/** + * Brightness, clipping and sharpness from one pass over the grey image. The + * Laplacian is summed by hand rather than through Jimp's `convolute`, which + * clamps every result to 0..255 and so loses the negative half of each edge. + * The caller hands in at least 3 x 3 pixels, so there is an inner pixel. + */ +function greyStats(data: Uint8Array, width: number, height: number) { + const total = width * height; + const grey = new Float32Array(total); + let sum = 0; + let clipped = 0; + for (let i = 0; i < total; i++) { + const o = i * 4; + const y = 0.299 * (data[o] ?? 0) + 0.587 * (data[o + 1] ?? 0) + 0.114 * (data[o + 2] ?? 0); + grey[i] = y; + sum += y; + if (y <= CLIP_LOW || y >= CLIP_HIGH) clipped++; + } + // Welford's running variance over the inner pixels, so no second pass is needed. + let n = 0; + let mean = 0; + let m2 = 0; + const at = (k: number) => grey[k] ?? 0; + for (let row = 1; row < height - 1; row++) { + for (let col = 1; col < width - 1; col++) { + const i = row * width + col; + const lap = at(i - 1) + at(i + 1) + at(i - width) + at(i + width) - 4 * at(i); + n++; + const delta = lap - mean; + mean += delta / n; + m2 += delta * (lap - mean); + } + } + return { + sharpness: round(m2 / n, 2), + luma: round(sum / (total * 255), 4), + clipped: round(clipped / total, 4), + }; +} + +function round(value: number, digits: number): number { + const f = 10 ** digits; + return Math.round(value * f) / f; +} + +/** Jimp's pHash is 64 characters of 0 and 1; stored as 16 hex digits. */ +export function bitsToHex(bits: string): string | null { + if (!/^[01]{64}$/.test(bits)) return null; + let out = ''; + for (let i = 0; i < 64; i += 4) out += parseInt(bits.slice(i, i + 4), 2).toString(16); + return out; +} + +/** Measures photos: in a worker thread in production, on the calling thread in tests. */ +export interface PhotoSignalsRunner { + measure(bytes: Uint8Array): Promise; + close(): Promise; +} + +export const PHOTO_SIGNALS_RUNNER = Symbol('PHOTO_SIGNALS_RUNNER'); + +/** The runner on the calling thread, for the tests (the TypeScript sources ship no compiled worker script). */ +export class InlinePhotoSignalsRunner implements PhotoSignalsRunner { + measure(bytes: Uint8Array): Promise { + return measurePhoto(bytes); + } + + close(): Promise { + return Promise.resolve(); + } +} diff --git a/server/src/nest/photos/photo-signals.module.ts b/server/src/nest/photos/photo-signals.module.ts new file mode 100644 index 000000000..a79f198b4 --- /dev/null +++ b/server/src/nest/photos/photo-signals.module.ts @@ -0,0 +1,29 @@ +import { TrekPhotoSignals } from '../../db/entities/TrekPhotoSignals.entity'; +import { MemoriesModule } from '../memories/memories.module'; +import { InlinePhotoSignalsRunner, PHOTO_SIGNALS_RUNNER } from './photo-signals-runner'; +import { PhotoSignalsService } from './photo-signals.service'; +import { WorkerPhotoSignalsRunner } from './worker-photo-signals-runner'; +import { MikroOrmModule } from '@mikro-orm/nestjs'; +import { Module } from '@nestjs/common'; + +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; + +/** The worker thread where the build ships its script; the calling thread under the TypeScript sources (the tests). */ +export function photoSignalsRunner(entry = join(__dirname, 'photo-signals.worker.js')) { + return existsSync(entry) ? new WorkerPhotoSignalsRunner(entry) : new InlinePhotoSignalsRunner(); +} + +/** + * What a photo's pixels say, measured once per photo off the event loop and + * kept in `trek_photo_signals` (PhotoSignalsService). Its own module rather + * than part of TrekPhotosModule: reading a photo's bytes needs the memories + * resolver, and MemoriesModule imports TrekPhotosModule, so the leaf has to + * stay a leaf. + */ +@Module({ + imports: [MemoriesModule, MikroOrmModule.forFeature([TrekPhotoSignals])], + providers: [PhotoSignalsService, { provide: PHOTO_SIGNALS_RUNNER, useFactory: () => photoSignalsRunner() }], + exports: [PhotoSignalsService], +}) +export class PhotoSignalsModule {} diff --git a/server/src/nest/photos/photo-signals.service.ts b/server/src/nest/photos/photo-signals.service.ts new file mode 100644 index 000000000..184d43d86 --- /dev/null +++ b/server/src/nest/photos/photo-signals.service.ts @@ -0,0 +1,229 @@ +import { TrekPhotoSignals } from '../../db/entities/TrekPhotoSignals.entity'; +import type { + GalleryPhotoFacts, + TrekPhotoSignalRow, + TrekPhotoSignalsRepository, +} from '../../db/repositories/TrekPhotoSignals.repository'; +import { logWarn } from '../audit/audit-log.logger'; +import { withRequestContext } from '../database/request-context'; +import { PhotoResolverService, type PhotoSignalSource } from '../memories/photo-resolver.service'; +import { + PHOTO_SIGNALS_ALGO, + PHOTO_SIGNALS_RUNNER, + type PhotoPixels, + type PhotoSignalsRunner, +} from './photo-signals-runner'; +import { MikroORM } from '@mikro-orm/core'; +import { InjectRepository } from '@mikro-orm/nestjs'; +import { Inject, Injectable, type OnModuleDestroy } from '@nestjs/common'; +import type { PhotoPickSignals } from '@trek/shared'; + +/** How long a caller waits for photos still being measured before it answers on what the rows know. */ +const SIGNALS_WAIT_MS = 8000; +/** Photos in line or being measured at most; more are left for a later call. */ +export const MAX_QUEUED_PHOTOS = 500; +/** Photos fetched at the same time (a provider is asked with its owner's credentials, so gently). */ +export const PARALLEL_FETCHES = 2; +/** A local original larger than this is not read for its signals. */ +export const MAX_ORIGINAL_BYTES = 40 * 1024 * 1024; + +/** Errors of the runner that say nothing about the photo: it is measured again on a later call. */ +const TRANSIENT = new Set(['busy', 'closed', 'unavailable']); + +/** + * What a photo's pixels say (a perceptual hash, sharpness, brightness, clipping, + * the original's size), measured once per photo and kept in + * `trek_photo_signals`, for picking photos for a book without a model looking + * at them (`suggest_book_photos`). + * + * Measuring is CPU work on a decoded image, so it runs in a worker thread + * (`PHOTO_SIGNALS_RUNNER`, one photo at a time) and never on the event loop; + * reading the bytes is I/O, at most `PARALLEL_FETCHES` at once. A photo is in + * line once however often it is asked for, the line is bounded, and a caller + * waits at most `SIGNALS_WAIT_MS` before it answers with what is ready; the + * rest keep being measured in the background, each in a request context of + * its own. A photo that cannot be measured gets a row of nulls, which is an + * answer too; one whose provider did not answer, or whose thread could not + * start, gets none and is tried again on the next call, and so does one that + * timed out once (the host may have been busy), not twice. + */ +@Injectable() +export class PhotoSignalsService implements OnModuleDestroy { + private readonly waiting: number[] = []; + /** Every photo in line or being measured, with what its callers wait on. */ + private readonly inLine = new Map; finish: () => void }>(); + private active = 0; + private closed = false; + /** Photos that timed out once; a second timeout stores them as unmeasurable. */ + private readonly timedOut = new Set(); + private warnedUnavailable = false; + + constructor( + @InjectRepository(TrekPhotoSignals) private readonly signalsRepo: TrekPhotoSignalsRepository, + private readonly resolver: PhotoResolverService, + @Inject(PHOTO_SIGNALS_RUNNER) private readonly runner: PhotoSignalsRunner, + private readonly orm: MikroORM, + ) {} + + async onModuleDestroy(): Promise { + this.closed = true; + this.waiting.length = 0; + await this.runner.close(); + } + + /** + * The photos of a journey's gallery, videos left out, without their signals + * (`signalsFor`). The caller has checked that the user may see the journey. + */ + async galleryFacts(journeyId: number): Promise { + return (await this.signalsRepo.listGalleryFacts(journeyId)).filter((f) => f.media_type !== 'video'); + } + + /** + * The signals of these photos: the stored ones at once, the missing ones + * after they are measured or `waitMs` has passed, whichever comes first. + * `pending` counts the photos still without signals. The photos in `later` + * answer what is stored for them too, and the missing ones join the line + * behind the others, neither waited for nor counted: a caller asking about a + * few photos of a large gallery gets those measured first. + */ + async signalsFor( + photoIds: readonly number[], + { waitMs = SIGNALS_WAIT_MS, later = [] }: { waitMs?: number; later?: readonly number[] } = {}, + ): Promise<{ byPhoto: Map; pending: number }> { + const ids = [...new Set(photoIds)]; + const wanted = new Set(ids); + const rest = [...new Set(later)].filter((id) => !wanted.has(id)); + const byPhoto = await this.stored([...ids, ...rest]); + const missing = ids.filter((id) => !byPhoto.has(id)); + const jobs = missing.map((id) => this.enqueue(id)).filter((job): job is Promise => job !== null); + for (const id of rest) if (!byPhoto.has(id)) void this.enqueue(id); + if (missing.length > 0) { + if (jobs.length > 0 && waitMs > 0) await waitAtMost(Promise.all(jobs), waitMs); + for (const [id, signals] of await this.stored(missing)) byPhoto.set(id, signals); + } + return { byPhoto, pending: ids.filter((id) => !byPhoto.has(id)).length }; + } + + /** The signals stored for these photos by the current measurement; older ones count as missing. */ + private async stored(ids: number[]): Promise> { + if (ids.length === 0) return new Map(); + const rows = await this.signalsRepo.listForPhotos(ids); + return new Map(rows.filter((r) => r.algo === PHOTO_SIGNALS_ALGO).map((r) => [r.photo_id, toSignals(r)])); + } + + /** Puts a photo in line once; null when the line is full or closed, so the photo stays pending. */ + private enqueue(photoId: number): Promise | null { + const known = this.inLine.get(photoId); + if (known) return known.done; + if (this.closed || this.inLine.size >= MAX_QUEUED_PHOTOS) return null; + let finish: () => void = () => undefined; + const done = new Promise((resolve) => { + finish = resolve; + }); + this.inLine.set(photoId, { done, finish }); + this.waiting.push(photoId); + this.pump(); + return done; + } + + private pump(): void { + while (this.active < PARALLEL_FETCHES) { + const photoId = this.waiting.shift(); + if (photoId === undefined) return; + this.active++; + void withRequestContext(this.orm, () => this.measureOne(photoId)) + .catch((err: unknown) => { + logWarn(`[photo-signals] photo ${photoId} not measured: ${err instanceof Error ? err.message : String(err)}`); + }) + .finally(() => { + this.active--; + this.inLine.get(photoId)?.finish(); + this.inLine.delete(photoId); + this.pump(); + }); + } + } + + /** Reads one photo, has it measured and stores what came out; a provider that did not answer stores nothing. */ + private async measureOne(photoId: number): Promise { + const source = await this.resolver.readForSignals(photoId, MAX_ORIGINAL_BYTES); + if (source.kind === 'unavailable') return; + let pixels: PhotoPixels | null = null; + if (source.kind !== 'none') { + try { + pixels = await this.runner.measure(source.bytes); + } catch (err: unknown) { + const reason = err instanceof Error ? err.message : String(err); + if (TRANSIENT.has(reason)) { + this.warnOnceUnavailable(reason, err); + return; + } + // The first timeout may be the host's load rather than the photo, so it is left for a later call. + if (reason === 'timeout' && !this.timedOut.has(photoId)) { + this.timedOut.add(photoId); + return; + } + // A second timeout, or a thread that died over this photo: measured as nothing, so it is not tried forever. + } + } + this.timedOut.delete(photoId); + await this.signalsRepo.saveSignals({ + photo_id: photoId, + algo: PHOTO_SIGNALS_ALGO, + phash: pixels?.phash ?? null, + sharpness: pixels?.sharpness ?? null, + luma: pixels?.luma ?? null, + clipped: pixels?.clipped ?? null, + ...originalSize(source, pixels), + }); + } + + /** A thread that cannot start fails every photo alike; the log says so once, not once per photo. */ + private warnOnceUnavailable(reason: string, err: unknown): void { + if (reason !== 'unavailable' || this.warnedUnavailable) return; + this.warnedUnavailable = true; + const cause = err instanceof Error && err.cause instanceof Error ? `: ${err.cause.message}` : ''; + logWarn(`[photo-signals] the measuring thread is unavailable${cause}; photos stay pending`); + } +} + +/** + * The original's size: what the decoded original measured, or what the + * provider says of it. A provider may report the sensor's sides, before the + * camera's rotation; the thumbnail is drawn upright, so when the two disagree + * on which side is the long one, the provider's sides are turned to match. + */ +function originalSize(source: PhotoSignalSource, pixels: PhotoPixels | null) { + if (source.kind !== 'thumbnail') return { orig_width: pixels?.width ?? null, orig_height: pixels?.height ?? null }; + const { width, height } = source; + const turned = + width !== null && + height !== null && + pixels !== null && + Math.sign(width - height) * Math.sign(pixels.width - pixels.height) === -1; + return turned ? { orig_width: height, orig_height: width } : { orig_width: width, orig_height: height }; +} + +function toSignals(row: TrekPhotoSignalRow): PhotoPickSignals { + return { + phash: row.phash, + sharpness: row.sharpness, + clipped: row.clipped, + origWidth: row.orig_width, + origHeight: row.orig_height, + }; +} + +/** Resolves when `work` settles or after `ms`, whichever comes first; never rejects. */ +function waitAtMost(work: Promise, ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref(); + const done = () => { + clearTimeout(timer); + resolve(); + }; + work.then(done, done); + }); +} diff --git a/server/src/nest/photos/photo-signals.worker.ts b/server/src/nest/photos/photo-signals.worker.ts new file mode 100644 index 000000000..4e5290120 --- /dev/null +++ b/server/src/nest/photos/photo-signals.worker.ts @@ -0,0 +1,16 @@ +import { measurePhoto } from './photo-signals-runner'; + +import { parentPort } from 'node:worker_threads'; + +/** + * The photo signals' thread: says it is ready once its script has loaded (so + * the runner can tell a thread that never started from a photo that broke + * one), then measures the photos it is sent and answers each by its id. + * `measurePhoto` answers null for anything it cannot read and never throws, so + * every photo gets its answer. + */ +parentPort?.on('message', (message: { id: number; bytes?: Uint8Array }) => { + if (!message.bytes) return; + void measurePhoto(message.bytes).then((pixels) => parentPort?.postMessage({ id: message.id, pixels })); +}); +parentPort?.postMessage({ ready: true }); diff --git a/server/src/nest/photos/worker-photo-signals-runner.ts b/server/src/nest/photos/worker-photo-signals-runner.ts new file mode 100644 index 000000000..e021f67c7 --- /dev/null +++ b/server/src/nest/photos/worker-photo-signals-runner.ts @@ -0,0 +1,141 @@ +import type { PhotoPixels, PhotoSignalsRunner } from './photo-signals-runner'; + +import { join } from 'node:path'; +import { Worker } from 'node:worker_threads'; + +/** A photo is decoded and measured in well under this; one that takes longer counts as unmeasurable. */ +export const MEASURE_TIMEOUT_MS = 20_000; +/** With this many photos already in line for the thread, a new one is turned away as busy. */ +export const MAX_WAITING_PHOTOS = 16; +/** A thread with nothing to do ends after ten minutes; the next photo starts one again. */ +const IDLE_MS = 10 * 60 * 1000; +/** The thread's heap. The decoded pixels sit outside it, which is why the decoder checks a photo's size first. */ +const SIGNALS_HEAP_MB = 256; + +type Reply = { id: number; pixels?: PhotoPixels | null; error?: string }; +/** What the thread sends once its script has loaded, before any photo. */ +type Ready = { ready: true }; + +/** + * The photo signals' own thread, so decoding a photo never holds up the + * server's requests (the piste router's thread is the precedent). Photos go to + * it one at a time, in the order they came, each on its own clock from when + * the thread takes it up: a photo not answered in time fails as `timeout` and + * the thread is replaced, a thread that dies fails the photo it held, and the + * next photo starts a fresh one. Started on the first photo, ended when idle. + * + * Only a thread that has said it is ready can blame a photo. One that cannot be + * started, or dies before its script has loaded (a broken build, a bad flag), + * fails its photos as `unavailable`, and one ended by `close()` as `closed`: + * both say nothing about the photo, so the caller tries it again later. + */ +export class WorkerPhotoSignalsRunner implements PhotoSignalsRunner { + private thread: Worker | null = null; + private line: Promise = Promise.resolve(); + private inLine = 0; + private lastId = 0; + private closed = false; + private idleTimer: NodeJS.Timeout | null = null; + /** The threads that have said they are ready. */ + private readonly started = new WeakSet(); + + constructor(private readonly entry = join(__dirname, 'photo-signals.worker.js')) {} + + measure(bytes: Uint8Array): Promise { + if (this.inLine >= MAX_WAITING_PHOTOS) return Promise.reject(new Error('busy')); + this.inLine++; + const turn = this.line.then(() => this.ask(bytes)); + this.line = turn.catch(() => undefined); + return turn.finally(() => { + this.inLine--; + }); + } + + async close(): Promise { + this.closed = true; + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = null; + const worker = this.thread; + this.thread = null; + await worker?.terminate(); + } + + /** One photo to the thread and its answer back. */ + private ask(bytes: Uint8Array): Promise { + if (this.closed) return Promise.reject(new Error('closed')); + let worker: Worker; + try { + worker = this.ensureThread(); + } catch (err: unknown) { + return Promise.reject(new Error('unavailable', { cause: err })); + } + const id = ++this.lastId; + return new Promise((resolve, reject) => { + const settle = (outcome: () => void) => { + clearTimeout(timer); + worker.off('message', onMessage); + worker.off('error', onError); + worker.off('exit', onExit); + outcome(); + }; + const onMessage = (reply: Reply | Ready) => { + if (!('id' in reply) || reply.id !== id) return; + const { pixels, error } = reply; + if (pixels === undefined) settle(() => reject(new Error(error ?? 'measurement failed'))); + else settle(() => resolve(pixels)); + }; + // Why the photo failed: the thread's own fault once it was ready, else nothing to do with the photo. + const blame = (err: Error) => { + if (this.closed) return new Error('closed'); + return this.started.has(worker) ? err : new Error('unavailable', { cause: err }); + }; + const onError = (err: Error) => settle(() => reject(blame(err))); + const onExit = () => settle(() => reject(blame(new Error('signals thread ended')))); + const timer = setTimeout(() => { + // The thread is stuck on this photo; the next photo gets a fresh one. + this.thread = null; + void worker.terminate(); + settle(() => reject(new Error('timeout'))); + }, MEASURE_TIMEOUT_MS); + worker.on('message', onMessage); + worker.on('error', onError); + worker.on('exit', onExit); + worker.postMessage({ id, bytes }); + }); + } + + private ensureThread(): Worker { + if (this.idleTimer) clearTimeout(this.idleTimer); + this.idleTimer = setTimeout(() => void this.endIdle(), IDLE_MS); + this.idleTimer.unref(); + if (this.thread) return this.thread; + // The compiled thread needs the same --require hooks (path aliases) the server was started with. + const hooks = process.execArgv.filter( + (arg, i, all) => ['--require', '-r'].includes(arg) || ['--require', '-r'].includes(all[i - 1] ?? ''), + ); + const worker = new Worker(this.entry, { + execArgv: hooks, + resourceLimits: { maxOldGenerationSizeMb: SIGNALS_HEAP_MB }, + }); + // Kept for the thread's whole life: a thread error with no listener would take the server down. + const forget = () => { + if (this.thread === worker) this.thread = null; + }; + // Any message, its first being `ready`, says the script has loaded. + worker.on('message', () => { + this.started.add(worker); + }); + worker.on('error', forget); + worker.on('exit', forget); + worker.unref(); + this.thread = worker; + return worker; + } + + /** An idle thread ends; the runner stays open for the next photo. */ + private async endIdle(): Promise { + const worker = this.thread; + this.thread = null; + await worker?.terminate(); + } +} diff --git a/server/tests/unit/db/entities-index.test.ts b/server/tests/unit/db/entities-index.test.ts index 207b9a3c8..40467e761 100644 --- a/server/tests/unit/db/entities-index.test.ts +++ b/server/tests/unit/db/entities-index.test.ts @@ -53,7 +53,8 @@ describe('ALL_ENTITIES', () => { * + tour_drafts, the Tours planner's unsaved draft per user and trip * + ai_connections, the AI assistant's connections to a language model * + assistant_threads, assistant_messages, assistant_tool_calls and assistant_usage, - * the trip assistant's conversations, the changes it proposed and what it used) + * the trip assistant's conversations, the changes it proposed and what it used + * + trek_photo_signals, what a photo's pixels say, for picking photos for a book) * is registered, and no two share a class name (a silent duplicate would * shadow one of them in `orm.getMetadata()`, exactly the kind of drift * `entity-schema-parity.test.ts` can't see — it iterates the metadata map, diff --git a/server/tests/unit/db/entity-class-fields.test.ts b/server/tests/unit/db/entity-class-fields.test.ts index 3c1946d3b..b1592fae0 100644 --- a/server/tests/unit/db/entity-class-fields.test.ts +++ b/server/tests/unit/db/entity-class-fields.test.ts @@ -53,7 +53,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; // assistant_messages: id, thread_id, role, parts; assistant_tool_calls: id, thread_id, message_id, // provider_call_id, tool_name, args, preview; assistant_usage: user_id, day, paid_by (each row is // written whole by the assistant, which always knows all of them). -const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 566; +// + trek_photo_signals: id, photo_id, algo (a measurement always names its photo and how it was made). +const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 569; describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => { it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => { diff --git a/server/tests/unit/db/legacy-baseline.test.ts b/server/tests/unit/db/legacy-baseline.test.ts index 29f9e2332..4c3585400 100644 --- a/server/tests/unit/db/legacy-baseline.test.ts +++ b/server/tests/unit/db/legacy-baseline.test.ts @@ -91,6 +91,7 @@ const UNNUMBERED = [ 'Migration20200101046210_the_trip_assistant_remembers_its_conversations', 'Migration20200101046220_a_chat_message_can_say_it_came_from_the_assistant', 'Migration20200101046230_an_admin_can_share_a_chatgpt_plan', + 'Migration20200101046240_a_photo_keeps_its_quality_signals', ]; // The last step the positional runner ever had; planLegacyBaseline refuses a // schema_version past it. diff --git a/server/tests/unit/db/repositories/TrekPhotoSignals.repository.test.ts b/server/tests/unit/db/repositories/TrekPhotoSignals.repository.test.ts new file mode 100644 index 000000000..6a9d04127 --- /dev/null +++ b/server/tests/unit/db/repositories/TrekPhotoSignals.repository.test.ts @@ -0,0 +1,147 @@ +/** + * `TrekPhotoSignalsRepository` (PSIGREPO-*): one measurement per photo, + * replaced by the next, gone with its photo; read back for many photos at once + * past SQLite's bound-value limit; and the gallery facts the book photo picker + * starts from, in gallery order and for one journey only. + */ +import { JourneyPhotos } from '../../../../src/db/entities/JourneyPhotos.entity'; +import { TrekPhotoSignals } from '../../../../src/db/entities/TrekPhotoSignals.entity'; +import { TrekPhotos } from '../../../../src/db/entities/TrekPhotos.entity'; +import type { + TrekPhotoSignalsRepository, + TrekPhotoSignalWrite, +} from '../../../../src/db/repositories/TrekPhotoSignals.repository'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { makeJourney } from '../../../helpers/factories/journeys'; +import { countRows, deleteRows, insertRow } from '../../../helpers/factories/rows'; +import { makeUser } from '../../../helpers/factories/users'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; + +const testDb = createSnapshotTestDb(); +let t: TestOrm; +let signals: TrekPhotoSignalsRepository; + +beforeAll(async () => { + t = await createTestOrm(testDb); + signals = t.repo(TrekPhotoSignals); +}); +beforeEach(() => { + resetTestDb(testDb); + t.clear(); +}); +afterAll(async () => { + await t.close(); + testDb.close(); +}); + +const measured = (photoId: number, over: Partial = {}): TrekPhotoSignalWrite => ({ + photo_id: photoId, + algo: 1, + phash: '0123456789abcdef', + sharpness: 412.5, + luma: 0.48, + clipped: 0.01, + orig_width: 4032, + orig_height: 3024, + ...over, +}); + +async function trekPhoto(ownerId: number, over: Partial<{ taken_at: string; media_type: string }> = {}) { + return insertRow(t, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg', ...over }); +} + +describe('TrekPhotoSignalsRepository', () => { + it('PSIGREPO-001: a measurement reads back as written, and the next one of the same photo replaces it', async () => { + const { user } = await makeUser(t); + const photoId = await trekPhoto(user.id); + expect(await signals.listForPhotos([photoId])).toEqual([]); + + await signals.saveSignals(measured(photoId)); + const [first] = await signals.listForPhotos([photoId]); + expect(first).toMatchObject(measured(photoId)); + expect(first?.computed_at).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/); + + const unmeasurable = measured(photoId, { + algo: 2, + phash: null, + sharpness: null, + luma: null, + clipped: null, + orig_width: null, + orig_height: null, + }); + await signals.saveSignals(unmeasurable); + expect(await signals.listForPhotos([photoId])).toEqual([expect.objectContaining(unmeasurable)]); + expect(await countRows(t, TrekPhotoSignals)).toBe(1); + }); + + it('PSIGREPO-002: reads the measurements of more photos than one statement may bind', async () => { + const { user } = await makeUser(t); + const first = await trekPhoto(user.id); + const last = await trekPhoto(user.id); + await signals.saveSignals(measured(first)); + await signals.saveSignals(measured(last, { sharpness: 12 })); + const ids = [first, ...Array.from({ length: 450 }, (_, i) => 100_000 + i), last]; + const rows = await signals.listForPhotos(ids); + expect(rows.map((r) => [r.photo_id, r.sharpness]).sort((a, b) => Number(a[0]) - Number(b[0]))).toEqual([ + [first, 412.5], + [last, 12], + ]); + }); + + it('PSIGREPO-003: a measurement goes with its photo', async () => { + const { user } = await makeUser(t); + const photoId = await trekPhoto(user.id); + await signals.saveSignals(measured(photoId)); + await deleteRows(t, TrekPhotos, { id: photoId }); + expect(await countRows(t, TrekPhotoSignals)).toBe(0); + }); + + it("PSIGREPO-004: the gallery facts are one journey's photos in gallery order, with when, where and what they are", async () => { + const { user } = await makeUser(t); + const journey = await makeJourney(t, user.id); + const other = await makeJourney(t, user.id); + const late = await trekPhoto(user.id, { taken_at: '2026-07-03T10:00:00' }); + const early = await insertRow(t, TrekPhotos, { + provider: 'immich', + asset_id: 'a1', + owner: user.id, + width: 1440, + height: 1080, + lat: 38.71, + lng: -9.13, + }); + const clip = await trekPhoto(user.id, { media_type: 'video' }); + const elsewhere = await trekPhoto(user.id); + const now = Date.now(); + await insertRow(t, JourneyPhotos, { journey: journey.id, photo: late, sort_order: 2, created_at: now }); + await insertRow(t, JourneyPhotos, { + journey: journey.id, + photo: early, + caption: 'Tram 28', + sort_order: 1, + created_at: now, + }); + await insertRow(t, JourneyPhotos, { journey: journey.id, photo: clip, sort_order: 2, created_at: now }); + await insertRow(t, JourneyPhotos, { journey: other.id, photo: elsewhere, sort_order: 0, created_at: now }); + + expect(await signals.listGalleryFacts(journey.id)).toEqual([ + { + photo_id: early, + caption: 'Tram 28', + provider: 'immich', + media_type: 'image', + width: 1440, + height: 1080, + taken_at: null, + lat: 38.71, + lng: -9.13, + }, + expect.objectContaining({ photo_id: late, caption: null, taken_at: '2026-07-03T10:00:00', media_type: 'image' }), + expect.objectContaining({ photo_id: clip, media_type: 'video' }), + ]); + }); +}); diff --git a/server/tests/unit/nest/photo-resolver.service.test.ts b/server/tests/unit/nest/photo-resolver.service.test.ts index a458eaa31..ad6d4707f 100644 --- a/server/tests/unit/nest/photo-resolver.service.test.ts +++ b/server/tests/unit/nest/photo-resolver.service.test.ts @@ -25,6 +25,9 @@ import { TrekPhotoRegistrationService } from '../../../src/nest/photos/trek-phot import { deleteRows, findRow, insertRow, insertRowIgnoringConflict } from '../../helpers/factories/rows'; import { createTestOrm, type TestOrm } from '../../helpers/test-orm'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest'; vi.mock('../../../src/db/database', async () => { @@ -67,7 +70,7 @@ const cache = { }; // Local bytes go through the storage facade (slice 3); the resolver only ever // addresses the 'journey' category with the 'journey/' prefix stripped. -const storage = { exists: vi.fn(), sendToResponse: vi.fn() }; +const storage = { exists: vi.fn(), sendToResponse: vi.fn(), stat: vi.fn(), withLocalFile: vi.fn() }; // Real adapters over the stubbed services, and a real registry: the cases below // keep asserting on immich.streamImmichAsset/synology.streamSynologyAsset, so @@ -361,3 +364,61 @@ describe('getPhotoInfo — dispatch', () => { expect(result.success).toBe(false); }); }); + +describe('readForSignals: what the quality signals are measured on', () => { + let dir: string; + let original: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-resolver-signals-')); + original = join(dir, 'p.jpg'); + await writeFile(original, Buffer.from('jpeg bytes')); + }); + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('RESOLVE-020: a local original is read whole through storage, unless it is over the cap, missing or a video', async () => { + const id = await insertPhoto({ provider: 'local', file_path: 'journey/p.jpg' }); + storage.stat.mockResolvedValue({ key: 'p.jpg', size: 10, mtimeMs: 0 }); + storage.withLocalFile.mockImplementation((_category: string, _name: string, fn: (path: string) => unknown) => + fn(original), + ); + + expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'original', bytes: Buffer.from('jpeg bytes') }); + expect(storage.withLocalFile).toHaveBeenCalledWith('journey', 'p.jpg', expect.any(Function)); + expect(await svc.readForSignals(id, 5)).toEqual({ kind: 'none' }); + storage.stat.mockRejectedValue(new Error('invalid key')); + expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'none' }); + + const outside = await insertPhoto({ provider: 'local', file_path: 'elsewhere/p.jpg' }); + const clip = await insertPhoto({ provider: 'local', file_path: 'journey/c.mp4', media_type: 'video' }); + const bare = await insertPhoto({ provider: 'local' }); + for (const photoId of [outside, clip, bare, 999999]) { + expect(await svc.readForSignals(photoId, 100)).toEqual({ kind: 'none' }); + } + expect(storage.withLocalFile).toHaveBeenCalledTimes(1); + }); + + it("RESOLVE-021: a provider photo answers its thumbnail and the original's size, asked with its owner's credentials", async () => { + const id = await insertPhoto({ provider: 'immich', asset_id: 'a1', owner_id: 5 }); + const bytes = Buffer.from('thumb'); + immich.fetchImmichThumbnailBytes.mockResolvedValue({ bytes, contentType: 'image/webp' }); + immich.getAssetInfo.mockResolvedValue({ success: true, data: { id: 'a1', width: 6000, height: 4000 } }); + + expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'thumbnail', bytes, width: 6000, height: 4000 }); + expect(immich.fetchImmichThumbnailBytes).toHaveBeenCalledWith(5, 'a1', 5); + expect(immich.getAssetInfo).toHaveBeenCalledWith(5, 'a1', 5); + + immich.getAssetInfo.mockResolvedValue({ error: 'offline', status: 502 }); + expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'thumbnail', bytes, width: null, height: null }); + + immich.fetchImmichThumbnailBytes.mockResolvedValue({ error: 'offline', status: 502 }); + expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'unavailable' }); + + const unknown = await insertPhoto({ provider: 'picasa', asset_id: 'p1', owner_id: 2 }); + const ownerless = await insertPhoto({ provider: 'immich', asset_id: 'a2' }); + expect(await svc.readForSignals(unknown, 100)).toEqual({ kind: 'none' }); + expect(await svc.readForSignals(ownerless, 100)).toEqual({ kind: 'none' }); + }); +}); diff --git a/server/tests/unit/nest/photos/photo-signals-runner.test.ts b/server/tests/unit/nest/photos/photo-signals-runner.test.ts new file mode 100644 index 000000000..43aafec95 --- /dev/null +++ b/server/tests/unit/nest/photos/photo-signals-runner.test.ts @@ -0,0 +1,346 @@ +/** + * Measuring a photo's pixels (PHSIG-*): on photos Jimp draws here, a sharp + * scene against the same scene blurred, a copy shifted by a few pixels against + * the scene inverted, plain white and black, one scene drawn at a thumbnail's + * size and at an original's, bytes that are no image, and headers declaring a + * canvas too large to decode. Then the thread script, the runner that talks to + * a thread (with small threads the test writes that answer, fail, hang, die or + * never start) and the factory that picks the thread or the calling thread. + */ +import { + bitsToHex, + InlinePhotoSignalsRunner, + measurePhoto, + SIGNAL_EDGE, + withinDecodeCap, +} from '../../../../src/nest/photos/photo-signals-runner'; +import { photoSignalsRunner } from '../../../../src/nest/photos/photo-signals.module'; +import { + MAX_WAITING_PHOTOS, + MEASURE_TIMEOUT_MS, + WorkerPhotoSignalsRunner, +} from '../../../../src/nest/photos/worker-photo-signals-runner'; +import { hashDistance } from '@trek/shared'; + +import { Jimp } from 'jimp'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; + +/** The thread's port, as its script sees it: messages in, messages out. */ +const port = vi.hoisted(() => { + const listeners: Array<(message: unknown) => void> = []; + return { + on: (event: string, listener: (message: unknown) => void) => { + if (event === 'message') listeners.push(listener); + }, + emit: (_event: 'message', message: unknown) => listeners.forEach((listener) => listener(message)), + postMessage: vi.fn(), + }; +}); +vi.mock('node:worker_threads', async (importOriginal) => ({ + ...(await importOriginal()), + parentPort: port, +})); + +/** A grey photo drawn pixel by pixel, as JPEG bytes. */ +async function drawn(shade: (x: number, y: number) => number, width = 256, height = 192, blur = 0): Promise { + const image = new Jimp({ width, height, color: 0xffffffff }); + const { data } = image.bitmap; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const v = Math.max(0, Math.min(255, Math.round(shade(x, y)))); + const i = (y * width + x) * 4; + data[i] = v; + data[i + 1] = v; + data[i + 2] = v; + } + } + if (blur > 0) image.blur(blur); + return Buffer.from(await image.getBuffer('image/jpeg')); +} + +/** Blocks of 32 px, light and dark, over a soft gradient, moved right by `dx`: edges to be sharp or blurred. */ +function scene(dx = 0) { + return (x: number, y: number) => { + const light = (Math.floor((x + dx) / 32) + Math.floor(y / 32)) % 2 === 0; + return light ? 200 + x / 16 : 50 + y / 16; + }; +} + +/** Scenery without edges on the pixel grid, drawn from 0..1 coordinates so it looks the same at any size. */ +function landscape(width: number, height: number) { + return (x: number, y: number) => { + const u = x / width; + const v = y / height; + const hill = (u - 0.43) ** 2 + (v - 0.51) ** 2 < 0.05 ? 30 : -30; + return 128 + 50 * Math.sin(u * 23.1 + v * 7.3) + 40 * Math.sin(v * 31.7 - u * 5.9) + hill; + }; +} + +/** The first bytes of an image file declaring `width` x `height`, with nothing behind them. */ +function header(kind: 'png' | 'gif' | 'bmp' | 'bmp-core', width: number, height: number): Buffer { + const data = Buffer.alloc(40); + if (kind === 'png') { + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(data, 0); + data.writeUInt32BE(13, 8); + data.write('IHDR', 12, 'latin1'); + data.writeUInt32BE(width, 16); + data.writeUInt32BE(height, 20); + } else if (kind === 'gif') { + data.write('GIF89a', 0, 'latin1'); + data.writeUInt16LE(width, 6); + data.writeUInt16LE(height, 8); + } else if (kind === 'bmp') { + data.write('BM', 0, 'latin1'); + data.writeUInt32LE(40, 14); + data.writeInt32LE(width, 18); + data.writeInt32LE(-height, 22); + } else { + data.write('BM', 0, 'latin1'); + data.writeUInt32LE(12, 14); + data.writeUInt16LE(width, 18); + data.writeUInt16LE(height, 20); + } + return data; +} + +let dir: string; + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'trek-photo-signals-')); +}); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('measurePhoto', () => { + it('PHSIG-001: a blurred copy measures far less sharp than the sharp scene', async () => { + const sharp = await measurePhoto(await drawn(scene())); + const blurred = await measurePhoto(await drawn(scene(), 256, 192, 4)); + expect(sharp?.sharpness).toBeGreaterThan(4 * (blurred?.sharpness ?? Infinity)); + expect(sharp).toMatchObject({ width: 256, height: 192 }); + }); + + it('PHSIG-002: a copy shifted by a few pixels hashes alike, the inverted scene does not', async () => { + const base = await measurePhoto(await drawn(scene())); + const shifted = await measurePhoto(await drawn(scene(3))); + const inverted = await measurePhoto(await drawn((x, y) => 255 - scene()(x, y))); + expect(base?.phash).toMatch(/^[0-9a-f]{16}$/); + expect(hashDistance(base?.phash ?? '', shifted?.phash ?? '')).toBeLessThanOrEqual(10); + expect(hashDistance(base?.phash ?? '', inverted?.phash ?? '')).toBeGreaterThan(10); + }); + + it('PHSIG-003: brightness and clipping, and the size of a large photo as it was before it was measured smaller', async () => { + const white = await measurePhoto(await drawn(() => 255)); + expect(white).toMatchObject({ luma: 1, clipped: 1 }); + const black = await measurePhoto(await drawn(() => 0)); + expect(black).toMatchObject({ luma: 0, clipped: 1, sharpness: 0 }); + const grey = await measurePhoto(await drawn(() => 128)); + expect(grey?.clipped).toBe(0); + expect(grey?.luma).toBeCloseTo(128 / 255, 2); + const large = await measurePhoto(await drawn(scene(), 1600, 1200)); + expect(large).toMatchObject({ width: 1600, height: 1200 }); + expect(SIGNAL_EDGE).toBe(240); + }); + + it('PHSIG-004: bytes that are no image, or an image too small or too thin to measure, have no signals', async () => { + expect(await measurePhoto(Buffer.from('not a photo at all'))).toBeNull(); + expect(await measurePhoto(await drawn(() => 128, 2, 2))).toBeNull(); + // 1000 x 4 scaled to a 240 px long edge is under one pixel high. + expect(await measurePhoto(await drawn(() => 128, 1000, 4))).toBeNull(); + }); + + it('PHSIG-005: the 64 bits of a hash become 16 hex digits; anything else is no hash', () => { + expect(bitsToHex(`${'0'.repeat(60)}1111`)).toBe('000000000000000f'); + expect(bitsToHex('1'.repeat(64))).toBe('ffffffffffffffff'); + expect(bitsToHex('0101')).toBeNull(); + expect(bitsToHex('2'.repeat(64))).toBeNull(); + }); + + it("PHSIG-007: one scene measures about as sharp at a thumbnail's 320 px as at an original's 2000 px", async () => { + const thumbnail = await measurePhoto(await drawn(landscape(320, 240), 320, 240)); + const original = await measurePhoto(await drawn(landscape(2000, 1500), 2000, 1500)); + const ratio = (original?.sharpness ?? 0) / (thumbnail?.sharpness ?? Infinity); + expect(ratio).toBeGreaterThan(0.75); + expect(ratio).toBeLessThan(1.33); + expect(original).toMatchObject({ width: 2000, height: 1500 }); + }); + + it('PHSIG-008: a canvas declared past 48 megapixels is refused before it is decoded; PNG and BMP within it are measured', async () => { + const read = vi.spyOn(Jimp, 'read'); + try { + expect(withinDecodeCap(header('png', 30000, 30000))).toBe(false); + expect(await measurePhoto(header('png', 30000, 30000))).toBeNull(); + expect(withinDecodeCap(header('gif', 65535, 65535))).toBe(false); + expect(withinDecodeCap(header('bmp', 10000, 10000))).toBe(false); + expect(withinDecodeCap(header('bmp-core', 60000, 60000))).toBe(false); + expect(withinDecodeCap(Buffer.from('II*\0 a TIFF Jimp could read'))).toBe(false); + expect(read).not.toHaveBeenCalled(); + expect(withinDecodeCap(header('gif', 640, 480))).toBe(true); + expect(withinDecodeCap(header('bmp', 6000, 4000))).toBe(true); + expect(withinDecodeCap(header('bmp-core', 640, 480))).toBe(true); + } finally { + read.mockRestore(); + } + const grey = new Jimp({ width: 40, height: 30, color: 0x808080ff }); + const png = Buffer.from(await grey.getBuffer('image/png')); + const bmp = Buffer.from(await grey.getBuffer('image/bmp')); + expect(await measurePhoto(png)).toMatchObject({ width: 40, height: 30, clipped: 0 }); + expect(await measurePhoto(bmp)).toMatchObject({ width: 40, height: 30, clipped: 0 }); + expect(withinDecodeCap(await drawn(scene()))).toBe(true); + }); + + it('PHSIG-006: the runner on the calling thread measures the same, and closes', async () => { + const bytes = await drawn(scene()); + const runner = new InlinePhotoSignalsRunner(); + expect(await runner.measure(bytes)).toEqual(await measurePhoto(bytes)); + await expect(runner.close()).resolves.toBeUndefined(); + }); +}); + +describe('the thread script', () => { + it('PHSIG-010: says it is ready, answers each photo by its id, an unreadable one with no signals, and ignores a message without bytes', async () => { + await import('../../../../src/nest/photos/photo-signals.worker'); + expect(port.postMessage).toHaveBeenCalledWith({ ready: true }); + port.emit('message', { id: 1, bytes: new Uint8Array(await drawn(scene())) }); + await vi.waitFor(() => + expect(port.postMessage).toHaveBeenCalledWith({ id: 1, pixels: expect.objectContaining({ width: 256 }) }), + ); + port.emit('message', { id: 2 }); + port.emit('message', { id: 3, bytes: new Uint8Array([1, 2, 3]) }); + await vi.waitFor(() => expect(port.postMessage).toHaveBeenCalledWith({ id: 3, pixels: null })); + expect(port.postMessage).toHaveBeenCalledTimes(3); + }); +}); + +/** + * A thread script the runner can start: plain JS, so it runs without the + * server's loader. Like the real one it says it is ready once loaded, unless + * the case wants a thread that never gets that far. + */ +async function script(name: string, body: string, ready = true): Promise { + const path = join(dir, `${name}.cjs`); + const hello = ready ? `parentPort.postMessage({ ready: true });\n` : ''; + await writeFile(path, `const { parentPort } = require('node:worker_threads');\n${body}\n${hello}`); + return path; +} + +const PIXELS = { phash: null, sharpness: 1, luma: 0.5, clipped: 0, width: 4, height: 3 }; + +describe('WorkerPhotoSignalsRunner', () => { + it('PHSIG-020: hands each photo to its thread in turn and answers what it says, by its id; a failure it reports rejects', async () => { + const entry = await script( + 'answers', + `parentPort.on('message', (m) => { + if (m.bytes[0] === 1) parentPort.postMessage({ id: m.id, pixels: ${JSON.stringify(PIXELS)} }); + else if (m.bytes[0] === 2) parentPort.postMessage({ id: m.id, pixels: null }); + else if (m.bytes[0] === 3) parentPort.postMessage({ id: m.id, error: 'cannot decode' }); + else { + parentPort.postMessage({ id: -1, pixels: null }); + parentPort.postMessage({ id: m.id }); + } + });`, + ); + const runner = new WorkerPhotoSignalsRunner(entry); + const answers = await Promise.allSettled([1, 2, 3, 4].map((b) => runner.measure(new Uint8Array([b])))); + expect(answers.map((a) => (a.status === 'fulfilled' ? a.value : (a.reason as Error).message))).toEqual([ + PIXELS, + null, + 'cannot decode', + 'measurement failed', + ]); + await runner.close(); + await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('closed'); + }); + + it('PHSIG-021: a thread that dies or throws fails the photo it held, and the next photo gets a fresh thread', async () => { + const entry = await script( + 'fragile', + `parentPort.on('message', (m) => { + if (m.bytes[0] === 9) process.exit(3); + if (m.bytes[0] === 8) setTimeout(() => { throw new Error('boom'); }); + else parentPort.postMessage({ id: m.id, pixels: null }); + });`, + ); + const runner = new WorkerPhotoSignalsRunner(entry); + await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull(); + await expect(runner.measure(new Uint8Array([9]))).rejects.toThrow('signals thread ended'); + await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull(); + await expect(runner.measure(new Uint8Array([8]))).rejects.toThrow('boom'); + await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull(); + await runner.close(); + }); + + it('PHSIG-022: a full line is busy, and closing fails the photo with the thread and every one in line as closed', async () => { + const entry = await script('silent', `parentPort.on('message', () => undefined);`); + const runner = new WorkerPhotoSignalsRunner(entry); + const waiting = Array.from({ length: MAX_WAITING_PHOTOS }, () => + runner.measure(new Uint8Array([1])).catch((err: Error) => err.message), + ); + await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('busy'); + await runner.close(); + const outcomes = await Promise.all(waiting); + expect(outcomes).toEqual(Array.from({ length: MAX_WAITING_PHOTOS }, () => 'closed')); + }); + + it('PHSIG-023: a photo left unanswered times out on its own clock, the thread is started over for the next', async () => { + const entry = await script( + 'slow', + `parentPort.on('message', (m) => { + if (m.bytes[0] !== 7) parentPort.postMessage({ id: m.id, pixels: null }); + });`, + ); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const runner = new WorkerPhotoSignalsRunner(entry); + const hung = runner.measure(new Uint8Array([7])).catch((err: Error) => err.message); + const next = runner.measure(new Uint8Array([1])); + await vi.advanceTimersByTimeAsync(MEASURE_TIMEOUT_MS); + expect(await hung).toBe('timeout'); + await expect(next).resolves.toBeNull(); + // Ten idle minutes end the thread; the runner starts another for the next photo. + await vi.advanceTimersByTimeAsync(10 * 60 * 1000); + await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull(); + await runner.close(); + } finally { + vi.useRealTimers(); + } + }); + + it('PHSIG-024: a thread that cannot start fails each photo on its own as unavailable', async () => { + // A bare relative path is one node refuses to start a thread from. + const runner = new WorkerPhotoSignalsRunner('no-such-signals.js'); + await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('unavailable'); + await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('unavailable'); + await runner.close(); + }); + + it('PHSIG-026: a thread that fails or ends before it is ready blames no photo: unavailable, with the reason', async () => { + const broken = await script('broken', `throw new Error('broken build');`, false); + const runner = new WorkerPhotoSignalsRunner(broken); + await expect(runner.measure(new Uint8Array([1]))).rejects.toMatchObject({ + message: 'unavailable', + cause: expect.objectContaining({ message: 'broken build' }), + }); + await runner.close(); + const quits = await script('quits', `process.exit(2);`, false); + const other = new WorkerPhotoSignalsRunner(quits); + await expect(other.measure(new Uint8Array([1]))).rejects.toMatchObject({ + message: 'unavailable', + cause: expect.objectContaining({ message: 'signals thread ended' }), + }); + await other.close(); + }); + + it('PHSIG-025: the module takes the thread where the build ships its script, else the calling thread', async () => { + const entry = await script('shipped', `parentPort.on('message', () => undefined);`); + const shipped = photoSignalsRunner(entry); + expect(shipped).toBeInstanceOf(WorkerPhotoSignalsRunner); + await shipped.close(); + expect(photoSignalsRunner(join(dir, 'missing.js'))).toBeInstanceOf(InlinePhotoSignalsRunner); + expect(photoSignalsRunner()).toBeInstanceOf(InlinePhotoSignalsRunner); + }); +}); diff --git a/server/tests/unit/nest/photos/photo-signals.service.test.ts b/server/tests/unit/nest/photos/photo-signals.service.test.ts new file mode 100644 index 000000000..be0ceda1c --- /dev/null +++ b/server/tests/unit/nest/photos/photo-signals.service.test.ts @@ -0,0 +1,303 @@ +/** + * PhotoSignalsService (PHSVC-*): a photo is measured once and its signals + * kept; a caller waits at most so long and is told how many are still + * pending; a photo is in line once however often it is asked for, the line is + * bounded and fetches are at most two at a time, and photos asked for later + * queue behind; a provider that does not answer, a thread that cannot start or + * a first timeout is tried again on the next call, a photo that cannot be + * measured is not; a provider's turned sides follow its upright thumbnail. + * Taken from PhotoSignalsModule over a snapshot test db, the bytes from a + * stubbed resolver and, unless a case swaps it, measured on the calling thread. + */ +import { JourneyPhotos } from '../../../../src/db/entities/JourneyPhotos.entity'; +import { TrekPhotoSignals } from '../../../../src/db/entities/TrekPhotoSignals.entity'; +import { TrekPhotos } from '../../../../src/db/entities/TrekPhotos.entity'; +import { PhotoResolverService, type PhotoSignalSource } from '../../../../src/nest/memories/photo-resolver.service'; +import { + PHOTO_SIGNALS_ALGO, + PHOTO_SIGNALS_RUNNER, + type PhotoSignalsRunner, +} from '../../../../src/nest/photos/photo-signals-runner'; +import { PhotoSignalsModule } from '../../../../src/nest/photos/photo-signals.module'; +import { + MAX_ORIGINAL_BYTES, + MAX_QUEUED_PHOTOS, + PARALLEL_FETCHES, + PhotoSignalsService, +} from '../../../../src/nest/photos/photo-signals.service'; +import { createSnapshotTestDb } from '../../../helpers/db-mock'; +import { makeJourney } from '../../../helpers/factories/journeys'; +import { countRows, insertRow } from '../../../helpers/factories/rows'; +import { makeUser } from '../../../helpers/factories/users'; +import { resetTestDb } from '../../../helpers/test-db'; +import { createTestModule, type TestModule } from '../../../helpers/test-module'; +import { createTestOrm, type TestOrm } from '../../../helpers/test-orm'; + +import { Jimp } from 'jimp'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const db = createSnapshotTestDb(); +const resolver = { readForSignals: vi.fn<(photoId: number, maxBytes: number) => Promise>() }; +let orm: TestOrm; +let photo: Buffer; + +/** The service in a container of its own, with the resolver stubbed and, when given, the runner too. */ +async function boot(runner?: PhotoSignalsRunner): Promise<{ t: TestModule; signals: PhotoSignalsService }> { + const t = await createTestModule({ + db, + imports: [PhotoSignalsModule], + overrides: [ + { provide: PhotoResolverService, useValue: resolver }, + ...(runner ? [{ provide: PHOTO_SIGNALS_RUNNER, useValue: runner }] : []), + ], + }); + return { t, signals: t.get(PhotoSignalsService) }; +} + +let main: { t: TestModule; signals: PhotoSignalsService }; + +beforeAll(async () => { + orm = await createTestOrm(db); + main = await boot(); + photo = Buffer.from(await new Jimp({ width: 300, height: 200, color: 0x808080ff }).getBuffer('image/jpeg')); +}); +beforeEach(() => { + resetTestDb(db); + orm.clear(); + resolver.readForSignals.mockReset(); +}); +afterAll(async () => { + await main.t.close(); + await orm.close(); + db.close(); +}); + +/** A journey with these photos in its gallery, in this order; answers their trek_photos ids. */ +async function gallery(kinds: Array<'image' | 'video'>) { + const { user } = await makeUser(orm); + const journey = await makeJourney(orm, user.id); + const ids: number[] = []; + for (const [i, kind] of kinds.entries()) { + const id = await insertRow(orm, TrekPhotos, { + provider: 'local', + owner: user.id, + file_path: 'journey/p.jpg', + media_type: kind, + }); + await insertRow(orm, JourneyPhotos, { journey: journey.id, photo: id, sort_order: i, created_at: Date.now() }); + ids.push(id); + } + return { journeyId: journey.id, ids }; +} + +/** A promise the case settles when it likes. */ +function deferred() { + let resolve: (value: T) => void = () => undefined; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; +} + +describe('PhotoSignalsService', () => { + it('PHSVC-001: measures a gallery photo once, keeps it, and leaves the videos out', async () => { + const empty = await gallery([]); + expect(await main.signals.galleryFacts(empty.journeyId)).toEqual([]); + expect(await main.signals.signalsFor([])).toEqual({ byPhoto: new Map(), pending: 0 }); + const { journeyId, ids } = await gallery(['image', 'video']); + const id = ids[0] ?? 0; + resolver.readForSignals.mockResolvedValue({ kind: 'original', bytes: photo }); + + const facts = await main.signals.galleryFacts(journeyId); + expect(facts.map((f) => f.photo_id)).toEqual([id]); + const first = await main.signals.signalsFor(facts.map((f) => f.photo_id)); + expect(first.pending).toBe(0); + expect(first.byPhoto.get(id)).toMatchObject({ origWidth: 300, origHeight: 200, clipped: 0 }); + expect(first.byPhoto.get(id)?.phash).toMatch(/^[0-9a-f]{16}$/); + expect(resolver.readForSignals).toHaveBeenCalledWith(id, MAX_ORIGINAL_BYTES); + + const again = await main.signals.signalsFor([id]); + expect(again.byPhoto.get(id)).toEqual(first.byPhoto.get(id)); + expect(resolver.readForSignals).toHaveBeenCalledTimes(1); + expect(await countRows(orm, TrekPhotoSignals, { photo: ids[0], algo: PHOTO_SIGNALS_ALGO })).toBe(1); + expect(await countRows(orm, TrekPhotoSignals)).toBe(1); + }); + + it("PHSVC-002: a provider's thumbnail keeps the original's size it reports; nothing to measure is an answer, no answer is not", async () => { + const { ids } = await gallery(['image', 'image', 'image']); + const [thumb, none, offline] = ids as [number, number, number]; + const sources = new Map([ + [thumb, { kind: 'thumbnail', bytes: photo, width: 6000, height: 4000 }], + [none, { kind: 'none' }], + ]); + resolver.readForSignals.mockImplementation((photoId) => + Promise.resolve(sources.get(photoId) ?? { kind: 'unavailable' }), + ); + + const first = await main.signals.signalsFor(ids); + expect(first.pending).toBe(1); + expect(first.byPhoto.get(thumb)).toMatchObject({ origWidth: 6000, origHeight: 4000 }); + expect(first.byPhoto.get(thumb)?.sharpness).toEqual(expect.any(Number)); + expect(first.byPhoto.get(none)).toEqual({ + phash: null, + sharpness: null, + clipped: null, + origWidth: null, + origHeight: null, + }); + expect(first.byPhoto.has(offline)).toBe(false); + + await main.signals.signalsFor(ids); + expect(resolver.readForSignals.mock.calls.map(([id]) => id)).toEqual([thumb, none, offline, offline]); + }); + + it('PHSVC-003: a measurement of an older version counts as missing and is made again', async () => { + const { ids } = await gallery(['image']); + const id = ids[0] ?? 0; + await insertRow(orm, TrekPhotoSignals, { photo: id, algo: PHOTO_SIGNALS_ALGO - 1, sharpness: 1 }); + resolver.readForSignals.mockResolvedValue({ kind: 'original', bytes: photo }); + + const { byPhoto } = await main.signals.signalsFor([id]); + expect(byPhoto.get(id)?.origWidth).toBe(300); + expect(await countRows(orm, TrekPhotoSignals)).toBe(1); + }); + + it('PHSVC-004: waits at most as long as told, and a photo asked for twice is in line once', async () => { + const { ids } = await gallery(['image']); + const id = ids[0] ?? 0; + const bytes = deferred(); + resolver.readForSignals.mockReturnValue(bytes.promise); + + const [a, b] = await Promise.all([ + main.signals.signalsFor([id], { waitMs: 20 }), + main.signals.signalsFor([id], { waitMs: 20 }), + ]); + expect([a.pending, b.pending]).toEqual([1, 1]); + expect(resolver.readForSignals).toHaveBeenCalledTimes(1); + + bytes.resolve({ kind: 'none' }); + await vi.waitFor(async () => expect(await countRows(orm, TrekPhotoSignals)).toBe(1)); + expect((await main.signals.signalsFor([id], { waitMs: 0 })).pending).toBe(0); + expect(resolver.readForSignals).toHaveBeenCalledTimes(1); + }); +}); + +describe('PhotoSignalsService, the line', () => { + it('PHSVC-005: fetches two photos at a time and keeps no more than the line holds', async () => { + const bytes = deferred(); + resolver.readForSignals.mockReturnValue(bytes.promise); + const ids = Array.from({ length: MAX_QUEUED_PHOTOS + 5 }, (_, i) => 900_000 + i); + + const { pending } = await main.signals.signalsFor(ids, { waitMs: 0 }); + expect(pending).toBe(ids.length); + expect(resolver.readForSignals).toHaveBeenCalledTimes(PARALLEL_FETCHES); + + // Nothing answers: the line drains without writing, the photos past its end were never in it. + bytes.resolve({ kind: 'unavailable' }); + await vi.waitFor(() => expect(resolver.readForSignals).toHaveBeenCalledTimes(MAX_QUEUED_PHOTOS)); + expect(await countRows(orm, TrekPhotoSignals)).toBe(0); + }); + + it('PHSVC-006: a photo that times out twice is measured as nothing, once is tried again; busy or unavailable is left for later', async () => { + const runner = { measure: vi.fn(), close: vi.fn(() => Promise.resolve()) }; + const own = await boot(runner); + try { + const { ids } = await gallery(['image', 'image', 'image']); + const [slow, busy, unavailable] = ids as [number, number, number]; + const reasons = new Map([ + [slow, 'timeout'], + [busy, 'busy'], + [unavailable, 'unavailable'], + ]); + // Each photo's bytes carry its place in `ids`, so the runner knows which one it holds. + resolver.readForSignals.mockImplementation((photoId) => + Promise.resolve({ kind: 'original', bytes: Buffer.from([ids.indexOf(photoId)]) }), + ); + runner.measure.mockImplementation((bytes) => + Promise.reject(new Error(reasons.get(ids[bytes[0] ?? 0] ?? 0), { cause: new Error('no thread script') })), + ); + + const first = await own.signals.signalsFor(ids); + expect(first.pending).toBe(3); + expect(await countRows(orm, TrekPhotoSignals)).toBe(0); + const second = await own.signals.signalsFor(ids); + expect(second.byPhoto.get(slow)).toMatchObject({ sharpness: null, origWidth: null }); + expect([second.byPhoto.has(busy), second.byPhoto.has(unavailable)]).toEqual([false, false]); + expect(second.pending).toBe(2); + expect(await countRows(orm, TrekPhotoSignals)).toBe(1); + expect(runner.measure).toHaveBeenCalledTimes(6); + } finally { + await own.t.close(); + } + expect(runner.close).toHaveBeenCalledTimes(1); + }); + + it('PHSVC-008: a photo whose bytes cannot be read stays pending, and the line goes on', async () => { + const { ids } = await gallery(['image', 'image']); + const [broken, fine] = ids as [number, number]; + resolver.readForSignals.mockImplementation((photoId) => + photoId === broken + ? Promise.reject(new Error('disk gone')) + : Promise.resolve({ kind: 'none' }), + ); + + const { byPhoto, pending } = await main.signals.signalsFor([broken, fine]); + expect(pending).toBe(1); + expect(byPhoto.has(fine)).toBe(true); + expect(await countRows(orm, TrekPhotoSignals)).toBe(1); + }); + + it('PHSVC-007: once the module is shutting down a photo is no longer put in line', async () => { + const own = await boot(); + try { + const { ids } = await gallery(['image']); + await own.signals.onModuleDestroy(); + const { pending } = await own.signals.signalsFor(ids, { waitMs: 0 }); + expect(pending).toBe(1); + expect(resolver.readForSignals).not.toHaveBeenCalled(); + } finally { + await own.t.close(); + } + }); + + it('PHSVC-009: photos asked for later answer what is stored and queue behind, neither waited for nor counted', async () => { + const { ids } = await gallery(['image', 'image', 'image']); + const [asked, stored, behind] = ids as [number, number, number]; + await insertRow(orm, TrekPhotoSignals, { photo: stored, algo: PHOTO_SIGNALS_ALGO, sharpness: 42 }); + const slow = deferred(); + resolver.readForSignals.mockImplementation((photoId) => + photoId === behind ? slow.promise : Promise.resolve({ kind: 'none' }), + ); + + const { byPhoto, pending } = await main.signals.signalsFor([asked], { later: [stored, behind, asked] }); + expect(pending).toBe(0); + expect(byPhoto.get(stored)?.sharpness).toBe(42); + expect(byPhoto.has(behind)).toBe(false); + expect(resolver.readForSignals.mock.calls.map(([id]) => id)).toEqual([asked, behind]); + + slow.resolve({ kind: 'none' }); + await vi.waitFor(async () => expect(await countRows(orm, TrekPhotoSignals)).toBe(3)); + }); + + it("PHSVC-010: a provider's sides that disagree with its upright thumbnail on the long side are turned to match", async () => { + const { ids } = await gallery(['image', 'image', 'image', 'image']); + const [turned, square, undecoded, unsized] = ids as [number, number, number, number]; + // An undecoded thumbnail (Immich's WebP) cannot say which side is long, so the provider's sides stay. + const sources = new Map([ + [turned, { kind: 'thumbnail', bytes: photo, width: 4000, height: 6000 }], + [square, { kind: 'thumbnail', bytes: photo, width: 5000, height: 5000 }], + [undecoded, { kind: 'thumbnail', bytes: Buffer.from('RIFF....WEBP'), width: 4000, height: 6000 }], + [unsized, { kind: 'thumbnail', bytes: photo, width: null, height: null }], + ]); + resolver.readForSignals.mockImplementation((photoId) => + Promise.resolve(sources.get(photoId) ?? { kind: 'none' }), + ); + + const { byPhoto } = await main.signals.signalsFor(ids); + expect(byPhoto.get(turned)).toMatchObject({ origWidth: 6000, origHeight: 4000 }); + expect(byPhoto.get(square)).toMatchObject({ origWidth: 5000, origHeight: 5000 }); + expect(byPhoto.get(undecoded)).toMatchObject({ sharpness: null, origWidth: 4000, origHeight: 6000 }); + expect(byPhoto.get(unsized)).toMatchObject({ origWidth: null, origHeight: null }); + expect(byPhoto.get(unsized)?.sharpness).toEqual(expect.any(Number)); + }); +}); diff --git a/shared/src/book/photo-pick.spec.ts b/shared/src/book/photo-pick.spec.ts new file mode 100644 index 000000000..45a9153a0 --- /dev/null +++ b/shared/src/book/photo-pick.spec.ts @@ -0,0 +1,370 @@ +import { + bookPhotoSuggestionsSchema, + hashDistance, + PHOTO_PICK, + photoMoment, + photosInScope, + pickBookPhotos, + type PhotoPickCandidate, + type PhotoPickEntry, + type PhotoPickSignals, +} from './photo-pick'; + +import { describe, expect, it } from 'vitest'; + +/** + * Picking photos for a book without looking at them: days, places, bursts, + * look-alikes by hash, the doubtful ones set aside, a cover's shape, the limit + * shared out over the groups. Lisbon's Alfama and Belem are 6 km apart; two + * points 0.001 degrees of latitude apart are 111 m apart. + */ + +const ALFAMA = { lat: 38.7118, lng: -9.13 }; +const BELEM = { lat: 38.6916, lng: -9.216 }; + +const signals = (over: Partial = {}): PhotoPickSignals => ({ + phash: null, + sharpness: null, + clipped: null, + origWidth: null, + origHeight: null, + ...over, +}); + +const photo = (photoId: number, over: Partial = {}): PhotoPickCandidate => ({ + photoId, + takenAt: null, + lat: null, + lng: null, + width: null, + height: null, + caption: null, + entryIds: [], + inBook: false, + signals: null, + ...over, +}); + +const entry = (id: number, over: Partial = {}): PhotoPickEntry => ({ + id, + date: '2026-07-03', + title: null, + location: null, + lat: null, + lng: null, + ...over, +}); + +/** A photo in the Alfama on 3 July at hh:mm:ss local time (no zone). */ +const at = (photoId: number, time: string, over: Partial = {}) => + photo(photoId, { takenAt: `2026-07-03T${time}`, ...ALFAMA, ...over }); + +const ids = (result: ReturnType) => result.groups.map((g) => g.picks.map((p) => p.photoId)); + +describe('photoMoment', () => { + it('PHOTOPICK-001: a time without a zone is the wall clock, a UTC time moves by the longitude, an offset is taken as written', () => { + const wall = Date.UTC(2026, 6, 3, 23, 30) / 1000; + expect(photoMoment('2026-07-03T23:30:00', 139.7)).toEqual({ day: '2026-07-03', at: wall }); + expect(photoMoment('2026-07-03T20:00:00.000Z', 139.7)?.day).toBe('2026-07-04'); + expect(photoMoment('2026-07-03T20:00:00Z', null)?.day).toBe('2026-07-03'); + expect(photoMoment('2026-07-03T01:00:00Z', -122.4)?.day).toBe('2026-07-02'); + expect(photoMoment('2026-07-03 23:30:00+02:00', 139.7)).toEqual({ + day: '2026-07-03', + at: Date.UTC(2026, 6, 3, 21, 30) / 1000, + }); + expect(photoMoment('2026-07-03', 10)).toEqual({ day: '2026-07-03', at: null }); + expect(photoMoment('yesterday', 10)).toBeNull(); + expect(photoMoment(null, 10)).toBeNull(); + }); +}); + +describe('hashDistance', () => { + it('PHOTOPICK-002: counts the differing bits of two 64-bit hex hashes and refuses anything else', () => { + expect(hashDistance('0123456789abcdef', '0123456789abcdef')).toBe(0); + expect(hashDistance('0000000000000000', '000000000000000f')).toBe(4); + expect(hashDistance('ffffffffffffffff', '0000000000000000')).toBe(64); + expect(hashDistance('0000', '0000000000000000')).toBeNull(); + expect(hashDistance('zzzzzzzzzzzzzzzz', '0000000000000000')).toBeNull(); + }); +}); + +describe('pickBookPhotos', () => { + it('PHOTOPICK-003: groups by the linked entry day first, then the capture day, undated last, and answers the shared shape', () => { + const entries = [entry(7, { date: '2026-07-01', location: 'Porto' })]; + const late = '2026-07-05T10:00:00'; + const photos = [photo(1), photo(2, { takenAt: late }), photo(3, { takenAt: late, entryIds: [7] })]; + const result = pickBookPhotos(photos, entries, { signalsPending: 2 }); + expect(bookPhotoSuggestionsSchema.parse(result)).toEqual(result); + expect(result.groups.map((g) => [g.day, g.entryId, g.place])).toEqual([ + ['2026-07-01', 7, 'Porto'], + ['2026-07-05', null, null], + ['undated', null, null], + ]); + expect(ids(result)).toEqual([[3], [2], [1]]); + expect(result.signals_pending).toBe(2); + expect(result.groups_more).toBe(0); + }); + + it('PHOTOPICK-004: splits a day into places 250 m apart, named after the nearest entry within 2 km, with its entry only on that day', () => { + const entries = [ + entry(1, { location: 'Alfama', ...ALFAMA }), + entry(2, { date: '2026-07-04', title: 'Belem tower', ...BELEM }), + ]; + const result = pickBookPhotos( + [ + at(10, '09:00:00'), + at(11, '09:10:00', { lat: ALFAMA.lat + 0.001 }), + at(12, '15:00:00', BELEM), + at(13, '16:00:00', { lat: ALFAMA.lat + 0.01 }), + ], + entries, + ); + expect(result.groups.map((g) => [g.place, g.entryId, g.picks.map((p) => p.photoId)])).toEqual([ + ['Alfama', 1, [10, 11]], + ['Belem tower', null, [12]], + ['Alfama', 1, [13]], + ]); + }); + + it('PHOTOPICK-005: a burst keeps its sharpest photo and names the others near duplicates of it', () => { + const result = pickBookPhotos( + [ + at(1, '10:00:00', { signals: signals({ sharpness: 200 }) }), + at(2, '10:00:04', { signals: signals({ sharpness: 400 }) }), + at(3, '10:00:09', { signals: signals({ sharpness: 300 }) }), + at(4, '10:05:00', { signals: signals({ sharpness: 250 }) }), + ], + [], + ); + const [group] = result.groups; + expect(group?.picks.map((p) => p.photoId)).toEqual([2, 4]); + expect(group?.picks[0]?.why[0]).toBe('sharpest of 3 similar'); + expect(group?.dropped).toEqual([ + { photoId: 3, reason: 'near_duplicate_of', of: 2 }, + { photoId: 1, reason: 'near_duplicate_of', of: 2 }, + ]); + }); + + it('PHOTOPICK-006: photos far apart in time look alike when their hashes differ in at most 10 bits', () => { + const result = pickBookPhotos( + [ + at(1, '10:00:00', { signals: signals({ phash: '0000000000000000', sharpness: 100 }) }), + at(2, '12:00:00', { signals: signals({ phash: '00000000000003ff', sharpness: 150 }) }), + at(3, '14:00:00', { signals: signals({ phash: '0000000000000fff', sharpness: 120 }) }), + ], + [], + ); + // 1 and 2 differ in 10 bits, 2 and 3 in 2, so all three are one set even though 1 and 3 differ in 12. + expect(ids(result)).toEqual([[2]]); + expect(result.groups[0]?.dropped.map((d) => d.photoId).sort()).toEqual([1, 3]); + const apart = pickBookPhotos( + [ + at(1, '10:00:00', { signals: signals({ phash: '0000000000000000' }) }), + at(2, '12:00:00', { signals: signals({ phash: '0000000000000fff' }) }), + ], + [], + ); + expect(ids(apart)).toEqual([[1, 2]]); + }); + + it('PHOTOPICK-007: a possibly blurry photo is set aside and said so, unless it is all its group has', () => { + const sharp = (id: number, time: string, sharpness: number) => at(id, time, { signals: signals({ sharpness }) }); + const result = pickBookPhotos( + [ + sharp(1, '09:00:00', 300), + sharp(2, '10:00:00', 280), + sharp(3, '11:00:00', 20), + photo(4, { takenAt: '2026-07-04T10:00:00', signals: signals({ sharpness: 15 }) }), + ], + [], + ); + expect(result.groups[0]?.picks.map((p) => p.photoId)).toEqual([1, 2]); + expect(result.groups[0]?.dropped).toEqual([{ photoId: 3, reason: 'possibly_blurry' }]); + expect(result.groups[1]?.picks).toEqual([{ photoId: 4, why: ['possibly blurry', 'not in the book yet'] }]); + // Under 35 % of the typical photo but above the floor: not blurry. + const soft = pickBookPhotos([sharp(1, '09:00:00', 900), sharp(2, '10:00:00', 1000), sharp(3, '11:00:00', 100)], []); + expect(soft.groups[0]?.dropped).toEqual([]); + // Fewer than three measured photos say nothing about what is typical. + const two = pickBookPhotos([sharp(1, '09:00:00', 300), sharp(2, '10:00:00', 5)], []); + expect(two.groups[0]?.dropped).toEqual([]); + expect(PHOTO_PICK.minForMedian).toBe(3); + }); + + it('PHOTOPICK-008: an original too small for print is set aside, a cover asks for more pixels, an unknown size counts as fine', () => { + const sized = (id: number, time: string, w: number | null) => + at(id, time, { signals: signals({ origWidth: w, origHeight: w === null ? null : Math.round(w * 0.75) }) }); + const photos = [ + sized(1, '09:00:00', 4000), + sized(2, '10:00:00', 1000), + sized(3, '11:00:00', 2000), + sized(4, '12:00:00', null), + ]; + const page = pickBookPhotos(photos, []); + expect(page.groups[0]?.dropped).toEqual([{ photoId: 2, reason: 'low_resolution' }]); + const cover = pickBookPhotos(photos, [], { purpose: 'cover' }); + expect(cover.groups[0]?.dropped.map((d) => d.photoId).sort()).toEqual([2, 3]); + expect(cover.groups[0]?.picks.map((p) => p.photoId)).toEqual([1, 4]); + }); + + it('PHOTOPICK-009: a cover puts a wide or square photo before a sharper upright one, a page goes by sharpness', () => { + const photos = [ + at(1, '09:00:00', { width: 600, height: 800, signals: signals({ sharpness: 900 }) }), + at(2, '10:00:00', { width: 800, height: 600, signals: signals({ sharpness: 300 }) }), + at(3, '11:00:00', { width: 800, height: 800, signals: signals({ sharpness: 200 }) }), + ]; + const cover = pickBookPhotos(photos, [], { purpose: 'cover', limit: 2 }); + expect(cover.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([ + [2, 'landscape'], + [3, 'square'], + ]); + expect(cover.groups[0]?.more).toBe(1); + const page = pickBookPhotos(photos, [], { limit: 1 }); + expect(page.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([[1, 'portrait']]); + }); + + it('PHOTOPICK-010: without signals a caption wins, then a photo the book does not have yet, best first', () => { + const result = pickBookPhotos( + [ + at(1, '09:00:00', { inBook: true }), + at(2, '10:00:00'), + at(3, '11:00:00', { caption: 'Tram 28', inBook: true }), + ], + [], + { limit: 2 }, + ); + expect(result.groups[0]?.picks).toEqual([ + { photoId: 3, why: ['in the book already', 'has a caption'] }, + { photoId: 2, why: ['not in the book yet'] }, + ]); + expect(result.groups[0]?.more).toBe(1); + }); + + it('PHOTOPICK-011: the limit goes round the groups, so every group gets its best before any gets a second', () => { + const day = (d: string, n: number) => + Array.from({ length: 3 }, (_, i) => photo(n + i, { takenAt: `2026-07-0${d}T1${i}:00:00` })); + const photos = [...day('1', 10), ...day('2', 20), ...day('3', 30)]; + const four = pickBookPhotos(photos, [], { limit: 4 }); + expect(ids(four)).toEqual([[10, 11], [20], [30]]); + expect(four.groups.map((g) => g.more)).toEqual([1, 2, 2]); + const two = pickBookPhotos(photos, [], { limit: 2 }); + expect(ids(two)).toEqual([[10], [20]]); + expect(two.groups_more).toBe(1); + expect(pickBookPhotos(photos, [], { limit: 500 }).groups.flatMap((g) => g.picks)).toHaveLength(9); + }); + + it('PHOTOPICK-012: an entry narrows to its linked photos, or to its day when none is linked; a date narrows to that day', () => { + const entries = [entry(1, { date: '2026-07-01' }), entry(2, { date: '2026-07-02' })]; + const photos = [ + photo(1, { entryIds: [1] }), + photo(2, { takenAt: '2026-07-01T10:00:00' }), + photo(3, { takenAt: '2026-07-02T10:00:00' }), + ]; + expect(ids(pickBookPhotos(photos, entries, { entryId: 1 }))).toEqual([[1]]); + expect(ids(pickBookPhotos(photos, entries, { entryId: 2 }))).toEqual([[3]]); + expect(ids(pickBookPhotos(photos, entries, { entryId: 99 }))).toEqual([]); + // Photo 1 has no capture time of its own, so it comes after the one that does. + expect(ids(pickBookPhotos(photos, entries, { date: '2026-07-01' }))).toEqual([[2, 1]]); + // The same narrowing, for the server to measure those photos first. + expect([...photosInScope(photos, entries, { entryId: 2 })]).toEqual([3]); + expect([...photosInScope(photos, entries, { date: '2026-07-01' })]).toEqual([1, 2]); + expect([...photosInScope(photos, entries)]).toEqual([1, 2, 3]); + }); + + it('PHOTOPICK-013: a photo without a place takes its entry place, and a placeless group takes the name its one entry has', () => { + const entries = [entry(5, { location: 'Sintra', lat: 38.7976, lng: -9.3906 }), entry(6, { location: 'Cascais' })]; + const result = pickBookPhotos( + [photo(1, { entryIds: [5] }), photo(2, { entryIds: [6] }), photo(3, { entryIds: [6] })], + entries, + ); + expect(result.groups.map((g) => [g.place, g.entryId, g.picks.map((p) => p.photoId)])).toEqual([ + ['Sintra', 5, [1]], + ['Cascais', 6, [2, 3]], + ]); + }); + + it('PHOTOPICK-014: photos about as sharp compare by what comes next: clipping, then a caption, then not in the book', () => { + const shot = ( + id: number, + time: string, + sharpness: number, + clipped: number, + over: Partial = {}, + ) => at(id, time, { inBook: true, signals: signals({ sharpness, clipped }), ...over }); + const result = pickBookPhotos( + [ + shot(1, '09:00:00', 310, 0), + shot(2, '10:00:00', 300, 0.012, { caption: 'Tram 28' }), + shot(3, '11:00:00', 305, 0.005, { inBook: false }), + shot(4, '12:00:00', 600, 0.05), + shot(5, '13:00:00', 305, 0.1, { caption: 'Miradouro', inBook: false }), + ], + [], + ); + // 4 is clearly the sharpest; 1, 2, 3 and 5 are within 15 % of each other, and 5 burns out a tenth of its pixels. + expect(ids(result)).toEqual([[4, 2, 3, 1, 5]]); + expect(PHOTO_PICK.bands).toEqual({ sharpness: 1.15, clipped: 0.02, pixels: 1.5 }); + }); + + it('PHOTOPICK-015: shots seconds apart whose hashes clearly differ stay apart; a burst within 20 bits is one set', () => { + const hashed = (id: number, time: string, phash: string) => at(id, time, { signals: signals({ phash }) }); + const walk = pickBookPhotos( + [ + hashed(1, '10:00:00', '0000000000000000'), + hashed(2, '10:00:08', 'ffffffff00000000'), + hashed(3, '10:00:16', 'ffffffffffffffff'), + ], + [], + ); + expect(ids(walk)).toEqual([[1, 2, 3]]); + expect(walk.groups[0]?.dropped).toEqual([]); + const burst = pickBookPhotos( + [hashed(1, '10:00:00', '0000000000000000'), hashed(2, '10:00:08', '00000000000fffff')], + [], + ); + expect(ids(burst)).toEqual([[1]]); + expect(burst.groups[0]?.picks[0]?.why[0]).toBe('best of 2 similar'); + expect(burst.groups[0]?.dropped).toEqual([{ photoId: 2, reason: 'near_duplicate_of', of: 1 }]); + // The same 20 bits an hour apart are no longer alike. + const later = pickBookPhotos( + [hashed(1, '10:00:00', '0000000000000000'), hashed(2, '11:00:00', '00000000000fffff')], + [], + ); + expect(ids(later)).toEqual([[1, 2]]); + expect(PHOTO_PICK.burstHashDistance).toBe(20); + }); + + it('PHOTOPICK-016: dropped names at most 30 per group and 120 in all, and dropped_more counts the rest', () => { + const burst = (day: number, n: number) => + Array.from({ length: n }, (_, i) => + photo(day * 100 + i + 1, { takenAt: `2026-07-0${day}T10:00:${String(i).padStart(2, '0')}` }), + ); + const one = pickBookPhotos(burst(1, 36), []); + expect(ids(one)).toEqual([[101]]); + expect(one.groups[0]?.dropped).toHaveLength(30); + expect(one.groups[0]?.dropped_more).toBe(5); + const five = pickBookPhotos([1, 2, 3, 4, 5].flatMap((day) => burst(day, 31)), []); + expect(bookPhotoSuggestionsSchema.parse(five)).toEqual(five); + expect(five.groups.map((g) => [g.dropped.length, g.dropped_more])).toEqual([ + [30, 0], + [30, 0], + [30, 0], + [30, 0], + [0, 30], + ]); + }); + + it("PHOTOPICK-017: a photo's shape comes from its row before the original's size, which a provider may report turned", () => { + const result = pickBookPhotos( + [ + at(1, '09:00:00', { width: 600, height: 800, signals: signals({ origWidth: 4000, origHeight: 3000 }) }), + at(2, '10:00:00', { signals: signals({ origWidth: 3000, origHeight: 2000 }) }), + ], + [], + { purpose: 'cover' }, + ); + expect(result.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([ + [2, 'landscape'], + [1, 'portrait'], + ]); + }); +}); diff --git a/shared/src/book/photo-pick.ts b/shared/src/book/photo-pick.ts new file mode 100644 index 000000000..0743beef7 --- /dev/null +++ b/shared/src/book/photo-pick.ts @@ -0,0 +1,579 @@ +import { haversineMetres } from '../geo/haversine'; + +import { z } from 'zod'; + +/** + * Photos for a book page, picked without a model looking at them: what the + * rows know (day, place, capture time, size, caption, whether the book has the + * photo already) and two cheap pixel signals measured once per photo on the + * server (a perceptual hash and how sharp it is, photo-signals.service.ts). + * + * Pure: the server's `suggest_book_photos` tool calls it with the journey's + * gallery, and a Studio button can call it later with the same data. + * + * What it cannot do, and the tool says so: judge beauty, find faces or closed + * eyes, tell who is in a photo or what it shows, or know two photos show the + * same scene from another angle (the hash only catches near-identical framing). + */ + +export const PHOTO_PICK = { + /** Photos within this many metres of a place's centre belong to it. */ + placeRadiusM: 250, + /** A place is named after the nearest entry location within this many metres. */ + labelRadiusM: 2000, + /** Two photos taken this close together are one burst... */ + burstSeconds: 10, + /** ...unless both hashes are measured and differ in more than this many bits (a walk shot every few seconds). */ + burstHashDistance: 20, + /** Two photos whose hashes differ in at most this many of 64 bits look the same. */ + hashDistance: 10, + /** Possibly blurry: sharper than less than this share of the journey's typical photo... */ + blurRatio: 0.35, + /** ...and under this absolute floor (the variance of the grey Laplacian at a 240 px long edge). */ + blurFloor: 60, + /** + * Ranking compares in bands, so a later criterion still counts between two + * photos that are about as good: sharpness within this factor of each other, + * clipping within this share of the pixels, size within this factor. + */ + bands: { sharpness: 1.15, clipped: 0.02, pixels: 1.5 }, + /** The typical photo needs this many measured photos before anything counts as blurry. */ + minForMedian: 3, + /** The long edge in pixels a photo needs for print: a part of a page, or a whole cover. */ + minLongEdge: { page: 1200, cover: 2400 }, + defaultLimit: 24, + maxLimit: 60, + /** What one group lists as left out, at most... */ + droppedPerGroup: 30, + /** ...and all groups together; the rest is counted in each group's `dropped_more`. */ + droppedTotal: 120, + /** Reasons one pick carries, at most. */ + whyPerPick: 6, +} as const; + +/** What the server measured on a photo's pixels; a null is a signal that could not be measured. */ +export interface PhotoPickSignals { + phash: string | null; + sharpness: number | null; + clipped: number | null; + /** The original's size in pixels, never a thumbnail's. */ + origWidth: number | null; + origHeight: number | null; +} + +/** A photo of the journey's gallery as the picker reads it. */ +export interface PhotoPickCandidate { + photoId: number; + /** As the row has it: ISO with or without a zone, or a bare date. */ + takenAt: string | null; + lat: number | null; + lng: number | null; + /** Any size with the photo's shape (a thumbnail's will do); null when unknown. */ + width: number | null; + height: number | null; + caption: string | null; + /** The journey entries the photo is linked to. */ + entryIds: readonly number[]; + inBook: boolean; + signals: PhotoPickSignals | null; +} + +/** A journey entry: its day and where it was. */ +export interface PhotoPickEntry { + id: number; + date: string; + title: string | null; + location: string | null; + lat: number | null; + lng: number | null; +} + +export interface PhotoPickOptions { + /** Only the photos linked to this entry (or, when it has none, taken on its day). */ + entryId?: number; + /** Only the photos of this day, `YYYY-MM-DD`. */ + date?: string; + /** A cover wants a wide or square photo and more pixels. */ + purpose?: 'page' | 'cover'; + limit?: number; + /** Photos whose signals were still being measured; passed through to the answer. */ + signalsPending?: number; +} + +const photoIdSchema = z.number().int().positive(); + +export const photoPickDroppedSchema = z.discriminatedUnion('reason', [ + z.object({ photoId: photoIdSchema, reason: z.literal('near_duplicate_of'), of: photoIdSchema }), + z.object({ photoId: photoIdSchema, reason: z.literal('possibly_blurry') }), + z.object({ photoId: photoIdSchema, reason: z.literal('low_resolution') }), +]); +export type PhotoPickDropped = z.infer; + +/** What `suggest_book_photos` answers: groups by day and place, each with its picks and what it left out. */ +export const bookPhotoSuggestionsSchema = z.object({ + groups: z.array( + z.object({ + /** `YYYY-MM-DD`, or `undated` for photos with neither an entry nor a capture time. */ + day: z.string(), + place: z.string().nullable(), + entryId: z.number().int().nullable(), + picks: z.array(z.object({ photoId: photoIdSchema, why: z.array(z.string()).max(PHOTO_PICK.whyPerPick) })), + dropped: z.array(photoPickDroppedSchema).max(PHOTO_PICK.droppedPerGroup), + /** Photos left out of this group that `dropped` has no room to name. */ + dropped_more: z.number().int().nonnegative(), + /** Good photos of this group the limit left out. */ + more: z.number().int().nonnegative(), + }), + ), + /** Groups left out whole because the limit was reached. */ + groups_more: z.number().int().nonnegative(), + /** Photos judged on their rows alone, their pixels not measured yet. */ + signals_pending: z.number().int().nonnegative(), +}); +export type BookPhotoSuggestions = z.infer; + +// ── Time ────────────────────────────────────────────────────────────────── + +const STAMP = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?\s*(Z|[+-]\d{2}:?\d{2})?$/i; +const BARE_DATE = /^(\d{4}-\d{2}-\d{2})$/; + +interface Moment { + /** The local day the photo was taken on. */ + day: string; + /** Seconds, comparable between photos; null for a bare date. */ + at: number | null; +} + +/** + * When a photo was taken. A time without a zone is the camera's wall clock and + * its date is the day. A time in UTC is moved by the photo's longitude (an + * hour per 15 degrees, the sun's time) when it has one, since the day a + * traveller lived it is the local one; an explicit offset is taken as written. + */ +export function photoMoment(takenAt: string | null, lng: number | null): Moment | null { + if (!takenAt) return null; + const bare = BARE_DATE.exec(takenAt.trim()); + if (bare?.[1]) return { day: bare[1], at: null }; + const m = STAMP.exec(takenAt.trim()); + if (!m) return null; + const [, y, mo, d, h, mi, s, zone] = m; + const wall = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(h), Number(mi), Number(s ?? 0)); + if (Number.isNaN(wall)) return null; + const offsetMin = zone ? zoneMinutes(zone) : 0; + const at = (wall - offsetMin * 60_000) / 1000; + if (zone === undefined || offsetMin !== 0) return { day: takenAt.trim().slice(0, 10), at }; + const shift = lng !== null ? Math.round(lng / 15) * 3600 : 0; + return { day: new Date((at + shift) * 1000).toISOString().slice(0, 10), at }; +} + +function zoneMinutes(zone: string): number { + if (zone.toUpperCase() === 'Z') return 0; + const digits = zone.replace(':', ''); + const sign = digits.startsWith('-') ? -1 : 1; + return sign * (Number(digits.slice(1, 3)) * 60 + Number(digits.slice(3, 5))); +} + +// ── Likeness ────────────────────────────────────────────────────────────── + +/** A 64-bit hash as its two 32-bit halves, read once per photo rather than once per pair. */ +type Hash = readonly [number, number]; + +const HASH = /^[0-9a-f]{16}$/i; + +function parseHash(hex: string | null | undefined): Hash | null { + if (!hex || !HASH.test(hex)) return null; + return [parseInt(hex.slice(0, 8), 16), parseInt(hex.slice(8), 16)]; +} + +/** The set bits of a 32-bit number. */ +function bitCount(x: number): number { + let v = x - ((x >>> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >>> 2) & 0x33333333); + return Math.imul((v + (v >>> 4)) & 0x0f0f0f0f, 0x01010101) >>> 24; +} + +function bitsApart(a: Hash, b: Hash): number { + return bitCount(a[0] ^ b[0]) + bitCount(a[1] ^ b[1]); +} + +/** How many of the 64 bits two hashes (16 hex digits each) differ in; null when either is not one. */ +export function hashDistance(a: string, b: string): number | null { + const ha = parseHash(a); + const hb = parseHash(b); + return ha && hb ? bitsApart(ha, hb) : null; +} + +// ── Picking ─────────────────────────────────────────────────────────────── + +/** A candidate with what the picker worked out about it. */ +interface Placed { + photo: PhotoPickCandidate; + day: string; + at: number | null; + pos: { lat: number; lng: number } | null; +} + +interface Group { + day: string; + members: Placed[]; + centre: { lat: number; lng: number } | null; +} + +interface Ranked { + photoId: number; + at: number | null; + why: string[]; +} + +interface Judged { + picks: Ranked[]; + dropped: PhotoPickDropped[]; +} + +/** + * The photos `pickBookPhotos` judges under these options: those of the entry or + * the day asked for, or all of them. The server measures these first. + */ +export function photosInScope( + candidates: readonly PhotoPickCandidate[], + entries: readonly PhotoPickEntry[], + options: Pick = {}, +): Set { + const byEntry = new Map(entries.map((e) => [e.id, e])); + const placed = candidates.map((photo) => place(photo, byEntry)); + return new Set(filterFor(placed, options, byEntry).map((p) => p.photo.photoId)); +} + +/** Photos for a page or the cover, grouped by day and place, best first within each group. */ +export function pickBookPhotos( + candidates: readonly PhotoPickCandidate[], + entries: readonly PhotoPickEntry[], + options: PhotoPickOptions = {}, +): BookPhotoSuggestions { + const purpose = options.purpose ?? 'page'; + const limit = Math.min(PHOTO_PICK.maxLimit, Math.max(1, options.limit ?? PHOTO_PICK.defaultLimit)); + const byEntry = new Map(entries.map((e) => [e.id, e])); + const median = medianSharpness(candidates); + const placed = candidates.map((photo) => place(photo, byEntry)); + const chosen = filterFor(placed, options, byEntry); + const groups = groupByDayAndPlace(chosen); + const judged = groups.map((g) => judge(g, purpose, median)); + const shown = takeRoundRobin(judged.map((j) => j.picks), limit); + const out: BookPhotoSuggestions['groups'] = []; + let groupsMore = 0; + let droppedRoom: number = PHOTO_PICK.droppedTotal; + groups.forEach((group, i) => { + const picks = shown[i] ?? []; + const all = judged[i]; + if (!all || picks.length === 0) { + if (all && all.picks.length > 0) groupsMore++; + return; + } + const label = labelOf(group, entries); + const dropped = all.dropped.slice(0, Math.min(PHOTO_PICK.droppedPerGroup, droppedRoom)); + droppedRoom -= dropped.length; + out.push({ + day: group.day, + place: label.place, + entryId: label.entryId, + picks: picks.map((p) => ({ photoId: p.photoId, why: p.why.slice(0, PHOTO_PICK.whyPerPick) })), + dropped, + dropped_more: all.dropped.length - dropped.length, + more: all.picks.length - picks.length, + }); + }); + return { groups: out, groups_more: groupsMore, signals_pending: options.signalsPending ?? 0 }; +} + +/** The sharpness of the journey's typical photo, over every measured one; null with too few to say. */ +function medianSharpness(candidates: readonly PhotoPickCandidate[]): number | null { + const values = candidates + .map((c) => c.signals?.sharpness) + .filter((v): v is number => typeof v === 'number' && Number.isFinite(v)) + .sort((a, b) => a - b); + if (values.length < PHOTO_PICK.minForMedian) return null; + const mid = Math.floor(values.length / 2); + const upper = values[mid] ?? 0; + return values.length % 2 === 1 ? upper : ((values[mid - 1] ?? upper) + upper) / 2; +} + +/** The photo's day (its entry's first, else when it was taken) and where it was (its own place, else its entry's). */ +function place(photo: PhotoPickCandidate, byEntry: ReadonlyMap): Placed { + const entry = photo.entryIds.map((id) => byEntry.get(id)).find((e) => e !== undefined); + const moment = photoMoment(photo.takenAt, photo.lng); + const day = entry?.date ?? moment?.day ?? 'undated'; + const own = photo.lat !== null && photo.lng !== null ? { lat: photo.lat, lng: photo.lng } : null; + const fromEntry = entry && entry.lat !== null && entry.lng !== null ? { lat: entry.lat, lng: entry.lng } : null; + return { photo, day, at: moment?.at ?? null, pos: own ?? fromEntry }; +} + +function filterFor( + placed: readonly Placed[], + options: PhotoPickOptions, + byEntry: ReadonlyMap, +): Placed[] { + let out = [...placed]; + if (options.entryId !== undefined) { + const entryId = options.entryId; + const linked = out.filter((p) => p.photo.entryIds.includes(entryId)); + const entryDay = byEntry.get(entryId)?.date; + out = linked.length > 0 || !entryDay ? linked : out.filter((p) => p.day === entryDay); + } + if (options.date !== undefined) out = out.filter((p) => p.day === options.date); + return out; +} + +/** Days in order (undated last); within a day, greedy places of `placeRadiusM`, the photos without one together. */ +function groupByDayAndPlace(placed: readonly Placed[]): Group[] { + const days = new Map(); + for (const p of [...placed].sort(byTime)) { + const list = days.get(p.day); + if (list) list.push(p); + else days.set(p.day, [p]); + } + const out: Group[] = []; + for (const day of [...days.keys()].sort(byDay)) { + const located: Group[] = []; + const nowhere: Group = { day, members: [], centre: null }; + for (const p of days.get(day) ?? []) { + if (!p.pos) { + nowhere.members.push(p); + continue; + } + const pos = p.pos; + const near = located.find( + (g) => g.centre && haversineMetres(g.centre.lat, g.centre.lng, pos.lat, pos.lng) <= PHOTO_PICK.placeRadiusM, + ); + if (!near) { + located.push({ day, members: [p], centre: { ...pos } }); + continue; + } + near.members.push(p); + const n = near.members.length; + const c = near.centre ?? pos; + near.centre = { lat: c.lat + (pos.lat - c.lat) / n, lng: c.lng + (pos.lng - c.lng) / n }; + } + out.push(...located); + if (nowhere.members.length > 0) out.push(nowhere); + } + return out; +} + +/** Near duplicates folded onto their best, then the doubtful ones set aside unless nothing else is left. */ +function judge(group: Group, purpose: 'page' | 'cover', median: number | null): Judged { + const sets = similarSets(group.members); + const dropped: PhotoPickDropped[] = []; + const kept: { p: Placed; why: string[] }[] = []; + for (const set of sets) { + const ranked = [...set].sort((a, b) => compare(a, b, purpose, median)); + const best = ranked[0]; + if (!best) continue; + const why: string[] = []; + if (set.length > 1) { + const measured = set.filter((p) => typeof p.photo.signals?.sharpness === 'number').length; + why.push(measured >= 2 ? `sharpest of ${set.length} similar` : `best of ${set.length} similar`); + for (const other of ranked.slice(1)) { + dropped.push({ photoId: other.photo.photoId, reason: 'near_duplicate_of', of: best.photo.photoId }); + } + } + kept.push({ p: best, why }); + } + const minEdge = PHOTO_PICK.minLongEdge[purpose]; + const doubts = kept.map(({ p }) => doubtOf(p.photo, median, minEdge)); + const sound = kept.filter((_, i) => doubts[i] === null); + const keepDoubtful = sound.length === 0; + const picks: Ranked[] = []; + kept.forEach(({ p, why }, i) => { + const doubt = doubts[i] ?? null; + if (doubt && !keepDoubtful) { + dropped.push({ photoId: p.photo.photoId, reason: doubt }); + return; + } + if (doubt === 'possibly_blurry') why.push('possibly blurry'); + if (doubt === 'low_resolution') why.push('low resolution for print'); + picks.push({ photoId: p.photo.photoId, at: p.at, why: [...why, ...plainWhy(p.photo)] }); + }); + const rank = new Map( + kept + .map(({ p }) => p) + .sort((a, b) => compare(a, b, purpose, median)) + .map((p, i) => [p.photo.photoId, i]), + ); + picks.sort((a, b) => (rank.get(a.photoId) ?? 0) - (rank.get(b.photoId) ?? 0)); + return { picks, dropped }; +} + +/** + * Photos that look alike, joined transitively: a burst (taken within + * `burstSeconds`) whose hashes, where both are measured, are within + * `burstHashDistance`, or near-identical hashes at any time. `members` come in + * time order (`groupByDayAndPlace`), so once a photo without a hash is past + * its burst window no later one can match it. Every hash is read once, and a + * pair costs two XORs, so a day of thousands of photos stays quick. + */ +function similarSets(members: readonly Placed[]): Placed[][] { + const parent = members.map((_, i) => i); + const root = (i: number): number => { + let r = i; + while (parent[r] !== r) r = parent[r] ?? r; + // Every index on the way now points straight at the root. + for (let k = i; parent[k] !== r; ) { + const next = parent[k] ?? r; + parent[k] = r; + k = next; + } + return r; + }; + const hashes = members.map((p) => parseHash(p.photo.signals?.phash)); + members.forEach((a, i) => { + const ha = hashes[i] ?? null; + for (let j = i + 1; j < members.length; j++) { + const b = members[j]; + if (!b) continue; + const burst = a.at !== null && b.at !== null && b.at - a.at <= PHOTO_PICK.burstSeconds; + if (!burst && ha === null) break; + if (alike(burst, ha, hashes[j] ?? null)) parent[root(j)] = root(i); + } + }); + const sets = new Map(); + members.forEach((p, i) => sets.set(root(i), [...(sets.get(root(i)) ?? []), p])); + return [...sets.values()]; +} + +function alike(burst: boolean, ha: Hash | null, hb: Hash | null): boolean { + const distance = ha && hb ? bitsApart(ha, hb) : null; + if (burst) return distance === null || distance <= PHOTO_PICK.burstHashDistance; + return distance !== null && distance <= PHOTO_PICK.hashDistance; +} + +/** Which band of `PHOTO_PICK.bands` a ratio falls in, centred on 1; zero or nothing is the lowest band. */ +function band(ratio: number, step: number): number { + return ratio > 0 ? Math.round(Math.log(ratio) / Math.log(step)) : -Infinity; +} + +/** + * Best first: for a cover a wide or square photo, then the sharper relative to + * the journey, then fewer burnt-out or black pixels, then more pixels, then a + * caption, then one the book does not have yet, each compared in bands so the + * next one decides between two photos about as good. An unknown sharpness + * counts as typical, an unknown clipping as none, an unknown size as the + * smallest. + */ +function compare(a: Placed, b: Placed, purpose: 'page' | 'cover', median: number | null): number { + const { bands } = PHOTO_PICK; + const keys: Array<(p: Placed) => number> = [ + (p) => (purpose === 'cover' && shapeOf(p.photo) === 'portrait' ? 1 : 0), + (p) => { + const s = p.photo.signals?.sharpness; + return -band(typeof s === 'number' && median ? s / median : 1, bands.sharpness); + }, + (p) => Math.floor((p.photo.signals?.clipped ?? 0) / bands.clipped), + (p) => -band((p.photo.signals?.origWidth ?? 0) * (p.photo.signals?.origHeight ?? 0), bands.pixels), + (p) => (p.photo.caption?.trim() ? 0 : 1), + (p) => (p.photo.inBook ? 1 : 0), + ]; + for (const key of keys) { + const ka = key(a); + const kb = key(b); + if (ka !== kb) return ka - kb; + } + return byTime(a, b); +} + +function doubtOf( + photo: PhotoPickCandidate, + median: number | null, + minEdge: number, +): 'possibly_blurry' | 'low_resolution' | null { + const s = photo.signals; + if (s && typeof s.sharpness === 'number' && median !== null) { + if (s.sharpness < median * PHOTO_PICK.blurRatio && s.sharpness < PHOTO_PICK.blurFloor) return 'possibly_blurry'; + } + if (s?.origWidth && s.origHeight && Math.max(s.origWidth, s.origHeight) < minEdge) return 'low_resolution'; + return null; +} + +/** The photo's shape: from its row's size (drawn upright), else from the original's size the server measured. */ +function shapeOf(photo: PhotoPickCandidate): 'landscape' | 'portrait' | 'square' | null { + const own = photo.width && photo.height; + const w = own ? photo.width : photo.signals?.origWidth; + const h = own ? photo.height : photo.signals?.origHeight; + if (!w || !h) return null; + const ratio = w / h; + if (ratio >= 1.1) return 'landscape'; + return ratio <= 0.9 ? 'portrait' : 'square'; +} + +function plainWhy(photo: PhotoPickCandidate): string[] { + const why: string[] = []; + const shape = shapeOf(photo); + if (shape) why.push(shape); + why.push(photo.inBook ? 'in the book already' : 'not in the book yet'); + if (photo.caption?.trim()) why.push('has a caption'); + return why; +} + +/** Up to `limit` picks, one from each group in turn, so every day and place gets its best before any gets a second. */ +function takeRoundRobin(lists: readonly Ranked[][], limit: number): Ranked[][] { + const out = lists.map((): Ranked[] => []); + let taken = 0; + for (let round = 0; taken < limit; round++) { + let tookOne = false; + for (let i = 0; i < lists.length && taken < limit; i++) { + const next = lists[i]?.[round]; + if (!next) continue; + out[i]?.push(next); + taken++; + tookOne = true; + } + if (!tookOne) break; + } + return out; +} + +/** + * A group's place and entry: the entry most of its photos are linked to, else + * the nearest entry location within `labelRadiusM` (its entry only when it is + * of the same day). A group without a place is named after its one entry, if + * all its photos share one. + */ +function labelOf(group: Group, entries: readonly PhotoPickEntry[]): { place: string | null; entryId: number | null } { + const counts = new Map(); + for (const m of group.members) for (const id of m.photo.entryIds) counts.set(id, (counts.get(id) ?? 0) + 1); + const linked = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0]?.[0] ?? null; + const linkedEntry = linked === null ? undefined : entries.find((e) => e.id === linked); + const centre = group.centre; + if (!centre) { + const all = linked !== null && group.members.every((m) => m.photo.entryIds.includes(linked)); + return { place: all ? (linkedEntry?.location ?? null) : null, entryId: linked }; + } + let nearest: { entry: PhotoPickEntry; d: number } | null = null; + for (const e of entries) { + if (e.lat === null || e.lng === null) continue; + const d = haversineMetres(centre.lat, centre.lng, e.lat, e.lng); + if (d <= PHOTO_PICK.labelRadiusM && (!nearest || d < nearest.d)) nearest = { entry: e, d }; + } + const name = nearest ? (nearest.entry.location ?? nearest.entry.title) : null; + const sameDay = nearest && nearest.entry.date === group.day ? nearest.entry.id : null; + return { place: name ?? linkedEntry?.location ?? null, entryId: linked ?? sameDay }; +} + +/** Days in calendar order, `undated` last. */ +function byDay(a: string, b: string): number { + if (a === 'undated' || b === 'undated') return Number(a === 'undated') - Number(b === 'undated'); + return a < b ? -1 : 1; +} + +/** Something taken at a time: a pick (its own id) or a placed photo (its photo's). */ +interface Timed { + at: number | null; + photoId?: number; + photo?: { photoId: number }; +} + +function byTime(a: Timed, b: Timed): number { + if (a.at !== null && b.at !== null && a.at !== b.at) return a.at - b.at; + if (a.at === null && b.at !== null) return 1; + if (a.at !== null && b.at === null) return -1; + return (a.photoId ?? a.photo?.photoId ?? 0) - (b.photoId ?? b.photo?.photoId ?? 0); +}