mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-23 07:26:30 +00:00
feat(photos): a photo keeps its quality signals, measured once off the event loop
This commit is contained in:
Maurice
@@ -130,6 +130,7 @@
|
||||
"TourWaypoints": "tours",
|
||||
"Tours": "tours",
|
||||
"TrekPhotoCacheMeta": "memories",
|
||||
"TrekPhotoSignals": "photos",
|
||||
"TrekPhotos": "photos",
|
||||
"TripAlbumLinks": "memories",
|
||||
"TripDocumentLinks": "doc-sync",
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { EntityRepositoryType, type Opt, type Ref, defineEntity, p } from '@mikro-orm/core';
|
||||
import { TrekPhotoSignalsRepository } from '../repositories/TrekPhotoSignals.repository';
|
||||
import { DbTimestampType } from '../types';
|
||||
import { TrekPhotos } from './TrekPhotos.entity';
|
||||
|
||||
export class TrekPhotoSignals {
|
||||
[EntityRepositoryType]?: TrekPhotoSignalsRepository;
|
||||
id!: number & Opt;
|
||||
photo!: Ref<TrekPhotos>;
|
||||
photo_id!: number;
|
||||
algo!: number;
|
||||
phash?: string | null;
|
||||
sharpness?: number | null;
|
||||
luma?: number | null;
|
||||
clipped?: number | null;
|
||||
orig_width?: number | null;
|
||||
orig_height?: number | null;
|
||||
computed_at!: string & Opt;
|
||||
}
|
||||
|
||||
export const TrekPhotoSignalsSchema = defineEntity({
|
||||
class: TrekPhotoSignals,
|
||||
repository: () => TrekPhotoSignalsRepository,
|
||||
uniques: [{ properties: ['photo'] }],
|
||||
properties: {
|
||||
id: p.integer().primary(),
|
||||
photo: () => p.manyToOne(TrekPhotos).ref().deleteRule('cascade').hidden(),
|
||||
photo_id: p.integer().persist(false),
|
||||
algo: p.integer(),
|
||||
phash: p.text().nullable(),
|
||||
sharpness: p.double().nullable(),
|
||||
luma: p.double().nullable(),
|
||||
clipped: p.double().nullable(),
|
||||
orig_width: p.integer().nullable(),
|
||||
orig_height: p.integer().nullable(),
|
||||
computed_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`),
|
||||
},
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { Collection, EntityRepositoryType, type Opt, type Ref, defineEntity, p }
|
||||
import { TrekPhotosRepository } from '../repositories/TrekPhotos.repository';
|
||||
import { DbTimestampType } from '../types';
|
||||
import { JourneyPhotos } from './JourneyPhotos.entity';
|
||||
import { TrekPhotoSignals } from './TrekPhotoSignals.entity';
|
||||
import { TripPhotos } from './TripPhotos.entity';
|
||||
import { Users } from './Users.entity';
|
||||
|
||||
@@ -24,6 +25,7 @@ export class TrekPhotos {
|
||||
lat?: number | null;
|
||||
lng?: number | null;
|
||||
journey_photos_collection = new Collection<JourneyPhotos>(this);
|
||||
trek_photo_signals_collection = new Collection<TrekPhotoSignals>(this);
|
||||
trip_photos_collection = new Collection<TripPhotos>(this);
|
||||
}
|
||||
|
||||
@@ -62,6 +64,7 @@ export const TrekPhotosSchema = defineEntity({
|
||||
lat: p.double().nullable(),
|
||||
lng: p.double().nullable(),
|
||||
journey_photos_collection: () => p.oneToMany(JourneyPhotos).mappedBy('photo').hidden(),
|
||||
trek_photo_signals_collection: () => p.oneToMany(TrekPhotoSignals).mappedBy('photo').hidden(),
|
||||
trip_photos_collection: () => p.oneToMany(TripPhotos).mappedBy('photo').hidden(),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -129,6 +129,7 @@ import { TourTypesSchema } from './TourTypes.entity';
|
||||
import { TourWaypointsSchema } from './TourWaypoints.entity';
|
||||
import { ToursSchema } from './Tours.entity';
|
||||
import { TrekPhotoCacheMetaSchema } from './TrekPhotoCacheMeta.entity';
|
||||
import { TrekPhotoSignalsSchema } from './TrekPhotoSignals.entity';
|
||||
import { TrekPhotosSchema } from './TrekPhotos.entity';
|
||||
import { TripAlbumLinksSchema } from './TripAlbumLinks.entity';
|
||||
import { TripDocumentLinksSchema } from './TripDocumentLinks.entity';
|
||||
@@ -289,6 +290,7 @@ export const ALL_ENTITIES: readonly EntitySchema[] = [
|
||||
TourWaypointsSchema,
|
||||
ToursSchema,
|
||||
TrekPhotoCacheMetaSchema,
|
||||
TrekPhotoSignalsSchema,
|
||||
TrekPhotosSchema,
|
||||
TripAlbumLinksSchema,
|
||||
TripDocumentLinksSchema,
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:
|
||||
// change the migration, then run `npm run gen:db-types`.
|
||||
import type { InsertOptional } from './columns';
|
||||
|
||||
export interface TrekPhotoSignalsTable {
|
||||
id: InsertOptional<number>;
|
||||
photo_id: number;
|
||||
algo: number;
|
||||
phash: InsertOptional<string | null>;
|
||||
sharpness: InsertOptional<number | null>;
|
||||
luma: InsertOptional<number | null>;
|
||||
clipped: InsertOptional<number | null>;
|
||||
orig_width: InsertOptional<number | null>;
|
||||
orig_height: InsertOptional<number | null>;
|
||||
computed_at: InsertOptional<string>;
|
||||
}
|
||||
@@ -135,6 +135,7 @@ import type { TourTypesTable } from './TourTypes.table';
|
||||
import type { TourWaypointsTable } from './TourWaypoints.table';
|
||||
import type { ToursTable } from './Tours.table';
|
||||
import type { TrekPhotoCacheMetaTable } from './TrekPhotoCacheMeta.table';
|
||||
import type { TrekPhotoSignalsTable } from './TrekPhotoSignals.table';
|
||||
import type { TrekPhotosTable } from './TrekPhotos.table';
|
||||
import type { TripAlbumLinksTable } from './TripAlbumLinks.table';
|
||||
import type { TripDocumentLinksTable } from './TripDocumentLinks.table';
|
||||
@@ -301,6 +302,7 @@ export interface DB {
|
||||
tour_waypoints: TourWaypointsTable;
|
||||
tours: ToursTable;
|
||||
trek_photo_cache_meta: TrekPhotoCacheMetaTable;
|
||||
trek_photo_signals: TrekPhotoSignalsTable;
|
||||
trek_photos: TrekPhotosTable;
|
||||
trip_album_links: TripAlbumLinksTable;
|
||||
trip_document_links: TripDocumentLinksTable;
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Migration } from '@mikro-orm/migrations';
|
||||
|
||||
/**
|
||||
* What a photo's pixels say, measured once so the assistant can pick photos
|
||||
* for a TREK Studio book without looking at them: a perceptual hash (`phash`,
|
||||
* 64 bits as 16 hex digits) to find near duplicates, how sharp it is
|
||||
* (`sharpness`, the variance of a Laplacian over the grey image), its mean
|
||||
* brightness (`luma`, 0 to 1) and the share of its pixels that are burnt out
|
||||
* or crushed black (`clipped`, 0 to 1), and the original's size in pixels
|
||||
* (`orig_width`, `orig_height`) for the print check.
|
||||
*
|
||||
* `algo` is the version of the measurement a row was made with, so a better
|
||||
* one later measures again instead of mixing the two. A null signal is one
|
||||
* that could not be measured (a format TREK cannot decode, a photo too large to
|
||||
* read), which is an answer too: the photo is not measured again.
|
||||
*
|
||||
* One row per photo, `photo_id UNIQUE` beside its own id rather than the photo
|
||||
* id as key: a key that is also a foreign key maps to a one-to-one on both
|
||||
* entities (see `trip_journal_links`). The row goes with its photo.
|
||||
*/
|
||||
export class Migration20200101046240_a_photo_keeps_its_quality_signals extends Migration {
|
||||
override name = 'Migration20200101046240_a_photo_keeps_its_quality_signals';
|
||||
|
||||
override up(): void {
|
||||
this.addSql(`
|
||||
CREATE TABLE IF NOT EXISTS trek_photo_signals (
|
||||
id INTEGER PRIMARY KEY,
|
||||
photo_id INTEGER NOT NULL UNIQUE REFERENCES trek_photos(id) ON DELETE CASCADE,
|
||||
algo INTEGER NOT NULL,
|
||||
phash TEXT,
|
||||
sharpness REAL,
|
||||
luma REAL,
|
||||
clipped REAL,
|
||||
orig_width INTEGER,
|
||||
orig_height INTEGER,
|
||||
computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
import type { TrekPhotoSignals } from '../entities/TrekPhotoSignals.entity';
|
||||
import type { DB } from '../kysely/db';
|
||||
import { dbNow } from '../types';
|
||||
import { TrekRepository } from './_shared/trek-repository';
|
||||
|
||||
type SignalsDB = Pick<DB, 'trek_photo_signals' | 'trek_photos' | 'journey_photos'>;
|
||||
|
||||
/** What one measurement of a photo found; a null is a signal that could not be measured. */
|
||||
export interface TrekPhotoSignalWrite {
|
||||
photo_id: number;
|
||||
algo: number;
|
||||
phash: string | null;
|
||||
sharpness: number | null;
|
||||
luma: number | null;
|
||||
clipped: number | null;
|
||||
orig_width: number | null;
|
||||
orig_height: number | null;
|
||||
}
|
||||
|
||||
/** A stored measurement, with when it was made. */
|
||||
export interface TrekPhotoSignalRow extends TrekPhotoSignalWrite {
|
||||
computed_at: string;
|
||||
}
|
||||
|
||||
/** A photo of a journey's gallery as the photo picker reads it: no bytes, no paths. */
|
||||
export interface GalleryPhotoFacts {
|
||||
photo_id: number;
|
||||
caption: string | null;
|
||||
provider: string;
|
||||
media_type: string;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
taken_at: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
}
|
||||
|
||||
/** SQLite binds at most so many values per statement; a gallery can hold more photos. */
|
||||
const IN_CHUNK = 400;
|
||||
|
||||
/**
|
||||
* `trek_photo_signals`: what a photo's pixels say, one row per photo, written
|
||||
* by `PhotoSignalsService` once its worker has measured the photo and read
|
||||
* again whenever photos are picked for a book. A new measurement of a photo
|
||||
* replaces the old one (upsert on `photo_id`).
|
||||
*/
|
||||
export class TrekPhotoSignalsRepository extends TrekRepository<TrekPhotoSignals> {
|
||||
private db() {
|
||||
return this.kysely<SignalsDB>();
|
||||
}
|
||||
|
||||
/** The measurements stored for these photos, whatever version made them; a photo without one is left out. */
|
||||
async listForPhotos(photoIds: number[]): Promise<TrekPhotoSignalRow[]> {
|
||||
const out: TrekPhotoSignalRow[] = [];
|
||||
for (let i = 0; i < photoIds.length; i += IN_CHUNK) {
|
||||
const rows = await this.db()
|
||||
.selectFrom('trek_photo_signals')
|
||||
.select([
|
||||
'photo_id',
|
||||
'algo',
|
||||
'phash',
|
||||
'sharpness',
|
||||
'luma',
|
||||
'clipped',
|
||||
'orig_width',
|
||||
'orig_height',
|
||||
'computed_at',
|
||||
])
|
||||
.where('photo_id', 'in', photoIds.slice(i, i + IN_CHUNK))
|
||||
.execute();
|
||||
out.push(...rows);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Stores a measurement, replacing the photo's earlier one. */
|
||||
async saveSignals(row: TrekPhotoSignalWrite): Promise<void> {
|
||||
const computedAt = dbNow();
|
||||
await this.db()
|
||||
.insertInto('trek_photo_signals')
|
||||
.values({ ...row, computed_at: computedAt })
|
||||
.onConflict((oc) =>
|
||||
oc.column('photo_id').doUpdateSet({
|
||||
algo: row.algo,
|
||||
phash: row.phash,
|
||||
sharpness: row.sharpness,
|
||||
luma: row.luma,
|
||||
clipped: row.clipped,
|
||||
orig_width: row.orig_width,
|
||||
orig_height: row.orig_height,
|
||||
computed_at: computedAt,
|
||||
}),
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* The photos of a journey's gallery with what their rows know about them
|
||||
* (capture time and place, size, kind), in gallery order. The size is a
|
||||
* thumbnail's for an uploaded photo and none for a provider's: good for the
|
||||
* shape, never for print.
|
||||
*/
|
||||
async listGalleryFacts(journeyId: number): Promise<GalleryPhotoFacts[]> {
|
||||
return await this.db()
|
||||
.selectFrom('journey_photos as gp')
|
||||
.innerJoin('trek_photos as tp', 'tp.id', 'gp.photo_id')
|
||||
.select([
|
||||
'gp.photo_id',
|
||||
'gp.caption',
|
||||
'tp.provider',
|
||||
'tp.media_type',
|
||||
'tp.width',
|
||||
'tp.height',
|
||||
'tp.taken_at',
|
||||
'tp.lat',
|
||||
'tp.lng',
|
||||
])
|
||||
.where('gp.journey_id', '=', journeyId)
|
||||
.orderBy('gp.sort_order', 'asc')
|
||||
.orderBy('gp.id', 'asc')
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,19 @@ import { TrekPhotoCacheService } from './trek-photo-cache.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { Response } from 'express';
|
||||
import fsPromises from 'node:fs/promises';
|
||||
|
||||
/**
|
||||
* What a photo's quality signals are measured on (photos/photo-signals.service.ts):
|
||||
* a local original, or a provider's thumbnail with the original's size as the
|
||||
* provider reports it. `none` is nothing to measure, `unavailable` a provider
|
||||
* that did not answer now, which a later try may reach.
|
||||
*/
|
||||
export type PhotoSignalSource =
|
||||
| { kind: 'original'; bytes: Buffer }
|
||||
| { kind: 'thumbnail'; bytes: Buffer; width: number | null; height: number | null }
|
||||
| { kind: 'none' }
|
||||
| { kind: 'unavailable' };
|
||||
|
||||
/**
|
||||
* Resolves a stored trek_photo to bytes or metadata by asking whichever provider
|
||||
@@ -150,6 +163,35 @@ export class PhotoResolverService {
|
||||
await provider.streamAsset(res, { ...ref, range }, kind);
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes a photo's quality signals are measured on, read with its
|
||||
* owner's access rather than a viewer's: the signals belong to the photo,
|
||||
* whoever asks. A local photo answers its original whole, so the size it
|
||||
* decodes to is the original's; a provider photo its thumbnail, since an
|
||||
* original can be hundreds of megabytes, with the size the provider reports.
|
||||
* `none` for a photo that is gone, a video, a local original over
|
||||
* `maxBytes` or missing, a provider TREK does not know.
|
||||
*/
|
||||
async readForSignals(photoId: number, maxBytes: number): Promise<PhotoSignalSource> {
|
||||
const photo = await this.photos.resolve(photoId);
|
||||
if (!photo || photo.media_type === 'video') return { kind: 'none' };
|
||||
if (photo.file_path) {
|
||||
const name = this.storageName(photo.file_path);
|
||||
const stat = name ? await this.storage.stat('journey', name).catch(() => null) : null;
|
||||
if (!name || !stat || stat.size > maxBytes) return { kind: 'none' };
|
||||
const bytes = await this.storage.withLocalFile('journey', name, (p) => fsPromises.readFile(p));
|
||||
return { kind: 'original', bytes };
|
||||
}
|
||||
const provider = this.providers.get(photo.provider);
|
||||
if (!provider || photo.owner_id == null || !photo.asset_id) return { kind: 'none' };
|
||||
const ref = this.refFor(photo, photo.owner_id);
|
||||
const thumbnail = await provider.fetchThumbnailBytes(ref);
|
||||
if ('error' in thumbnail) return { kind: 'unavailable' };
|
||||
const info = await provider.getAssetInfo(ref);
|
||||
const size = info.success ? info.data : null;
|
||||
return { kind: 'thumbnail', bytes: thumbnail.bytes, width: size?.width ?? null, height: size?.height ?? null };
|
||||
}
|
||||
|
||||
/**
|
||||
* The stored row as a provider-shaped request. One place decrypts the
|
||||
* Synology share passphrase, and one place decides which of the two user ids
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
import { Jimp } from 'jimp';
|
||||
|
||||
/**
|
||||
* The version of the measurement below. A stored row of another version is
|
||||
* measured again, so a better measurement never mixes with an older one.
|
||||
* Version 2 measures every photo at the same `SIGNAL_EDGE`, up or down.
|
||||
*/
|
||||
export const PHOTO_SIGNALS_ALGO = 2;
|
||||
|
||||
/**
|
||||
* The long edge every photo is scaled to, up or down, before it is measured. A
|
||||
* smaller picture has a higher Laplacian variance per pixel, so sharpness only
|
||||
* compares between photos measured at one size; this one is no larger than the
|
||||
* thumbnails the providers hand out (Immich about 250 px, Synology about 320).
|
||||
*/
|
||||
export const SIGNAL_EDGE = 240;
|
||||
|
||||
/** An image declaring more than this many megapixels is not decoded at all (a 48 MP decode takes about 200 MB). */
|
||||
const MAX_DECODE_MP = 48;
|
||||
|
||||
/** Grey levels at or beyond these count as burnt out or crushed black. */
|
||||
const CLIP_LOW = 5;
|
||||
const CLIP_HIGH = 250;
|
||||
|
||||
/** What a photo's pixels say. `width`/`height` are the decoded image's, before it is scaled to measure. */
|
||||
export interface PhotoPixels {
|
||||
/** The perceptual hash, 64 bits as 16 hex digits; null if the hash came out in another shape. */
|
||||
phash: string | null;
|
||||
/** The variance of the grey Laplacian at `SIGNAL_EDGE`: the higher, the sharper. */
|
||||
sharpness: number;
|
||||
/** The mean grey level, 0 (black) to 1 (white). */
|
||||
luma: number;
|
||||
/** The share of pixels burnt out or crushed black, 0 to 1. */
|
||||
clipped: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Measures one photo. CPU work on a decoded bitmap, so the server runs it in
|
||||
* its worker thread (photo-signals.worker.ts), never on the event loop.
|
||||
* Answers null for bytes that are no image TREK decodes (WebP, HEIC, TIFF, a
|
||||
* video, anything past `MAX_DECODE_MP`): such a photo has no pixel signals.
|
||||
*/
|
||||
export async function measurePhoto(bytes: Uint8Array): Promise<PhotoPixels | null> {
|
||||
if (!withinDecodeCap(bytes)) return null;
|
||||
try {
|
||||
const image = await Jimp.read(Buffer.from(bytes), { 'image/jpeg': { maxResolutionInMP: MAX_DECODE_MP } });
|
||||
const { width, height } = image.bitmap;
|
||||
if (width < 3 || height < 3) return null;
|
||||
image.scaleToFit({ w: SIGNAL_EDGE, h: SIGNAL_EDGE });
|
||||
// A strip far longer than it is wide ends up under 3 pixels across, which leaves no inner pixel.
|
||||
if (image.bitmap.width < 3 || image.bitmap.height < 3) return null;
|
||||
const grey = greyStats(image.bitmap.data, image.bitmap.width, image.bitmap.height);
|
||||
return { phash: bitsToHex(image.pHash()), ...grey, width, height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether these bytes may be decoded: a JPEG, which Jimp holds to
|
||||
* `MAX_DECODE_MP` itself, or a PNG, GIF or BMP whose header declares at most
|
||||
* that many pixels. A small file can declare a canvas of gigabytes, and the
|
||||
* decoder allocates all of it before the first pixel, so the size is read
|
||||
* first. Anything else (TIFF among the formats Jimp reads) is not decoded.
|
||||
*/
|
||||
export function withinDecodeCap(bytes: Uint8Array): boolean {
|
||||
const data = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) return true;
|
||||
const size = declaredSize(data);
|
||||
return size !== null && size.width * size.height <= MAX_DECODE_MP * 1_000_000;
|
||||
}
|
||||
|
||||
/** The canvas a PNG (IHDR), GIF (logical screen) or BMP (info or core header) declares; null for anything else. */
|
||||
function declaredSize(data: Buffer): { width: number; height: number } | null {
|
||||
if (data.length >= 24 && data.readUInt32BE(0) === 0x89504e47) {
|
||||
return { width: data.readUInt32BE(16), height: data.readUInt32BE(20) };
|
||||
}
|
||||
if (data.length >= 10 && data.toString('latin1', 0, 4) === 'GIF8') {
|
||||
return { width: data.readUInt16LE(6), height: data.readUInt16LE(8) };
|
||||
}
|
||||
if (data.length >= 26 && data.toString('latin1', 0, 2) === 'BM') {
|
||||
// The old core header holds two 16-bit sides, every later one two signed 32-bit ones (negative: top-down).
|
||||
if (data.readUInt32LE(14) === 12) return { width: data.readUInt16LE(18), height: data.readUInt16LE(20) };
|
||||
return { width: Math.abs(data.readInt32LE(18)), height: Math.abs(data.readInt32LE(22)) };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Brightness, clipping and sharpness from one pass over the grey image. The
|
||||
* Laplacian is summed by hand rather than through Jimp's `convolute`, which
|
||||
* clamps every result to 0..255 and so loses the negative half of each edge.
|
||||
* The caller hands in at least 3 x 3 pixels, so there is an inner pixel.
|
||||
*/
|
||||
function greyStats(data: Uint8Array, width: number, height: number) {
|
||||
const total = width * height;
|
||||
const grey = new Float32Array(total);
|
||||
let sum = 0;
|
||||
let clipped = 0;
|
||||
for (let i = 0; i < total; i++) {
|
||||
const o = i * 4;
|
||||
const y = 0.299 * (data[o] ?? 0) + 0.587 * (data[o + 1] ?? 0) + 0.114 * (data[o + 2] ?? 0);
|
||||
grey[i] = y;
|
||||
sum += y;
|
||||
if (y <= CLIP_LOW || y >= CLIP_HIGH) clipped++;
|
||||
}
|
||||
// Welford's running variance over the inner pixels, so no second pass is needed.
|
||||
let n = 0;
|
||||
let mean = 0;
|
||||
let m2 = 0;
|
||||
const at = (k: number) => grey[k] ?? 0;
|
||||
for (let row = 1; row < height - 1; row++) {
|
||||
for (let col = 1; col < width - 1; col++) {
|
||||
const i = row * width + col;
|
||||
const lap = at(i - 1) + at(i + 1) + at(i - width) + at(i + width) - 4 * at(i);
|
||||
n++;
|
||||
const delta = lap - mean;
|
||||
mean += delta / n;
|
||||
m2 += delta * (lap - mean);
|
||||
}
|
||||
}
|
||||
return {
|
||||
sharpness: round(m2 / n, 2),
|
||||
luma: round(sum / (total * 255), 4),
|
||||
clipped: round(clipped / total, 4),
|
||||
};
|
||||
}
|
||||
|
||||
function round(value: number, digits: number): number {
|
||||
const f = 10 ** digits;
|
||||
return Math.round(value * f) / f;
|
||||
}
|
||||
|
||||
/** Jimp's pHash is 64 characters of 0 and 1; stored as 16 hex digits. */
|
||||
export function bitsToHex(bits: string): string | null {
|
||||
if (!/^[01]{64}$/.test(bits)) return null;
|
||||
let out = '';
|
||||
for (let i = 0; i < 64; i += 4) out += parseInt(bits.slice(i, i + 4), 2).toString(16);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Measures photos: in a worker thread in production, on the calling thread in tests. */
|
||||
export interface PhotoSignalsRunner {
|
||||
measure(bytes: Uint8Array): Promise<PhotoPixels | null>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export const PHOTO_SIGNALS_RUNNER = Symbol('PHOTO_SIGNALS_RUNNER');
|
||||
|
||||
/** The runner on the calling thread, for the tests (the TypeScript sources ship no compiled worker script). */
|
||||
export class InlinePhotoSignalsRunner implements PhotoSignalsRunner {
|
||||
measure(bytes: Uint8Array): Promise<PhotoPixels | null> {
|
||||
return measurePhoto(bytes);
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { TrekPhotoSignals } from '../../db/entities/TrekPhotoSignals.entity';
|
||||
import { MemoriesModule } from '../memories/memories.module';
|
||||
import { InlinePhotoSignalsRunner, PHOTO_SIGNALS_RUNNER } from './photo-signals-runner';
|
||||
import { PhotoSignalsService } from './photo-signals.service';
|
||||
import { WorkerPhotoSignalsRunner } from './worker-photo-signals-runner';
|
||||
import { MikroOrmModule } from '@mikro-orm/nestjs';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
/** The worker thread where the build ships its script; the calling thread under the TypeScript sources (the tests). */
|
||||
export function photoSignalsRunner(entry = join(__dirname, 'photo-signals.worker.js')) {
|
||||
return existsSync(entry) ? new WorkerPhotoSignalsRunner(entry) : new InlinePhotoSignalsRunner();
|
||||
}
|
||||
|
||||
/**
|
||||
* What a photo's pixels say, measured once per photo off the event loop and
|
||||
* kept in `trek_photo_signals` (PhotoSignalsService). Its own module rather
|
||||
* than part of TrekPhotosModule: reading a photo's bytes needs the memories
|
||||
* resolver, and MemoriesModule imports TrekPhotosModule, so the leaf has to
|
||||
* stay a leaf.
|
||||
*/
|
||||
@Module({
|
||||
imports: [MemoriesModule, MikroOrmModule.forFeature([TrekPhotoSignals])],
|
||||
providers: [PhotoSignalsService, { provide: PHOTO_SIGNALS_RUNNER, useFactory: () => photoSignalsRunner() }],
|
||||
exports: [PhotoSignalsService],
|
||||
})
|
||||
export class PhotoSignalsModule {}
|
||||
@@ -0,0 +1,229 @@
|
||||
import { TrekPhotoSignals } from '../../db/entities/TrekPhotoSignals.entity';
|
||||
import type {
|
||||
GalleryPhotoFacts,
|
||||
TrekPhotoSignalRow,
|
||||
TrekPhotoSignalsRepository,
|
||||
} from '../../db/repositories/TrekPhotoSignals.repository';
|
||||
import { logWarn } from '../audit/audit-log.logger';
|
||||
import { withRequestContext } from '../database/request-context';
|
||||
import { PhotoResolverService, type PhotoSignalSource } from '../memories/photo-resolver.service';
|
||||
import {
|
||||
PHOTO_SIGNALS_ALGO,
|
||||
PHOTO_SIGNALS_RUNNER,
|
||||
type PhotoPixels,
|
||||
type PhotoSignalsRunner,
|
||||
} from './photo-signals-runner';
|
||||
import { MikroORM } from '@mikro-orm/core';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { Inject, Injectable, type OnModuleDestroy } from '@nestjs/common';
|
||||
import type { PhotoPickSignals } from '@trek/shared';
|
||||
|
||||
/** How long a caller waits for photos still being measured before it answers on what the rows know. */
|
||||
const SIGNALS_WAIT_MS = 8000;
|
||||
/** Photos in line or being measured at most; more are left for a later call. */
|
||||
export const MAX_QUEUED_PHOTOS = 500;
|
||||
/** Photos fetched at the same time (a provider is asked with its owner's credentials, so gently). */
|
||||
export const PARALLEL_FETCHES = 2;
|
||||
/** A local original larger than this is not read for its signals. */
|
||||
export const MAX_ORIGINAL_BYTES = 40 * 1024 * 1024;
|
||||
|
||||
/** Errors of the runner that say nothing about the photo: it is measured again on a later call. */
|
||||
const TRANSIENT = new Set(['busy', 'closed', 'unavailable']);
|
||||
|
||||
/**
|
||||
* What a photo's pixels say (a perceptual hash, sharpness, brightness, clipping,
|
||||
* the original's size), measured once per photo and kept in
|
||||
* `trek_photo_signals`, for picking photos for a book without a model looking
|
||||
* at them (`suggest_book_photos`).
|
||||
*
|
||||
* Measuring is CPU work on a decoded image, so it runs in a worker thread
|
||||
* (`PHOTO_SIGNALS_RUNNER`, one photo at a time) and never on the event loop;
|
||||
* reading the bytes is I/O, at most `PARALLEL_FETCHES` at once. A photo is in
|
||||
* line once however often it is asked for, the line is bounded, and a caller
|
||||
* waits at most `SIGNALS_WAIT_MS` before it answers with what is ready; the
|
||||
* rest keep being measured in the background, each in a request context of
|
||||
* its own. A photo that cannot be measured gets a row of nulls, which is an
|
||||
* answer too; one whose provider did not answer, or whose thread could not
|
||||
* start, gets none and is tried again on the next call, and so does one that
|
||||
* timed out once (the host may have been busy), not twice.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PhotoSignalsService implements OnModuleDestroy {
|
||||
private readonly waiting: number[] = [];
|
||||
/** Every photo in line or being measured, with what its callers wait on. */
|
||||
private readonly inLine = new Map<number, { done: Promise<void>; finish: () => void }>();
|
||||
private active = 0;
|
||||
private closed = false;
|
||||
/** Photos that timed out once; a second timeout stores them as unmeasurable. */
|
||||
private readonly timedOut = new Set<number>();
|
||||
private warnedUnavailable = false;
|
||||
|
||||
constructor(
|
||||
@InjectRepository(TrekPhotoSignals) private readonly signalsRepo: TrekPhotoSignalsRepository,
|
||||
private readonly resolver: PhotoResolverService,
|
||||
@Inject(PHOTO_SIGNALS_RUNNER) private readonly runner: PhotoSignalsRunner,
|
||||
private readonly orm: MikroORM,
|
||||
) {}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
this.closed = true;
|
||||
this.waiting.length = 0;
|
||||
await this.runner.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* The photos of a journey's gallery, videos left out, without their signals
|
||||
* (`signalsFor`). The caller has checked that the user may see the journey.
|
||||
*/
|
||||
async galleryFacts(journeyId: number): Promise<GalleryPhotoFacts[]> {
|
||||
return (await this.signalsRepo.listGalleryFacts(journeyId)).filter((f) => f.media_type !== 'video');
|
||||
}
|
||||
|
||||
/**
|
||||
* The signals of these photos: the stored ones at once, the missing ones
|
||||
* after they are measured or `waitMs` has passed, whichever comes first.
|
||||
* `pending` counts the photos still without signals. The photos in `later`
|
||||
* answer what is stored for them too, and the missing ones join the line
|
||||
* behind the others, neither waited for nor counted: a caller asking about a
|
||||
* few photos of a large gallery gets those measured first.
|
||||
*/
|
||||
async signalsFor(
|
||||
photoIds: readonly number[],
|
||||
{ waitMs = SIGNALS_WAIT_MS, later = [] }: { waitMs?: number; later?: readonly number[] } = {},
|
||||
): Promise<{ byPhoto: Map<number, PhotoPickSignals>; pending: number }> {
|
||||
const ids = [...new Set(photoIds)];
|
||||
const wanted = new Set(ids);
|
||||
const rest = [...new Set(later)].filter((id) => !wanted.has(id));
|
||||
const byPhoto = await this.stored([...ids, ...rest]);
|
||||
const missing = ids.filter((id) => !byPhoto.has(id));
|
||||
const jobs = missing.map((id) => this.enqueue(id)).filter((job): job is Promise<void> => job !== null);
|
||||
for (const id of rest) if (!byPhoto.has(id)) void this.enqueue(id);
|
||||
if (missing.length > 0) {
|
||||
if (jobs.length > 0 && waitMs > 0) await waitAtMost(Promise.all(jobs), waitMs);
|
||||
for (const [id, signals] of await this.stored(missing)) byPhoto.set(id, signals);
|
||||
}
|
||||
return { byPhoto, pending: ids.filter((id) => !byPhoto.has(id)).length };
|
||||
}
|
||||
|
||||
/** The signals stored for these photos by the current measurement; older ones count as missing. */
|
||||
private async stored(ids: number[]): Promise<Map<number, PhotoPickSignals>> {
|
||||
if (ids.length === 0) return new Map();
|
||||
const rows = await this.signalsRepo.listForPhotos(ids);
|
||||
return new Map(rows.filter((r) => r.algo === PHOTO_SIGNALS_ALGO).map((r) => [r.photo_id, toSignals(r)]));
|
||||
}
|
||||
|
||||
/** Puts a photo in line once; null when the line is full or closed, so the photo stays pending. */
|
||||
private enqueue(photoId: number): Promise<void> | null {
|
||||
const known = this.inLine.get(photoId);
|
||||
if (known) return known.done;
|
||||
if (this.closed || this.inLine.size >= MAX_QUEUED_PHOTOS) return null;
|
||||
let finish: () => void = () => undefined;
|
||||
const done = new Promise<void>((resolve) => {
|
||||
finish = resolve;
|
||||
});
|
||||
this.inLine.set(photoId, { done, finish });
|
||||
this.waiting.push(photoId);
|
||||
this.pump();
|
||||
return done;
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
while (this.active < PARALLEL_FETCHES) {
|
||||
const photoId = this.waiting.shift();
|
||||
if (photoId === undefined) return;
|
||||
this.active++;
|
||||
void withRequestContext(this.orm, () => this.measureOne(photoId))
|
||||
.catch((err: unknown) => {
|
||||
logWarn(`[photo-signals] photo ${photoId} not measured: ${err instanceof Error ? err.message : String(err)}`);
|
||||
})
|
||||
.finally(() => {
|
||||
this.active--;
|
||||
this.inLine.get(photoId)?.finish();
|
||||
this.inLine.delete(photoId);
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads one photo, has it measured and stores what came out; a provider that did not answer stores nothing. */
|
||||
private async measureOne(photoId: number): Promise<void> {
|
||||
const source = await this.resolver.readForSignals(photoId, MAX_ORIGINAL_BYTES);
|
||||
if (source.kind === 'unavailable') return;
|
||||
let pixels: PhotoPixels | null = null;
|
||||
if (source.kind !== 'none') {
|
||||
try {
|
||||
pixels = await this.runner.measure(source.bytes);
|
||||
} catch (err: unknown) {
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
if (TRANSIENT.has(reason)) {
|
||||
this.warnOnceUnavailable(reason, err);
|
||||
return;
|
||||
}
|
||||
// The first timeout may be the host's load rather than the photo, so it is left for a later call.
|
||||
if (reason === 'timeout' && !this.timedOut.has(photoId)) {
|
||||
this.timedOut.add(photoId);
|
||||
return;
|
||||
}
|
||||
// A second timeout, or a thread that died over this photo: measured as nothing, so it is not tried forever.
|
||||
}
|
||||
}
|
||||
this.timedOut.delete(photoId);
|
||||
await this.signalsRepo.saveSignals({
|
||||
photo_id: photoId,
|
||||
algo: PHOTO_SIGNALS_ALGO,
|
||||
phash: pixels?.phash ?? null,
|
||||
sharpness: pixels?.sharpness ?? null,
|
||||
luma: pixels?.luma ?? null,
|
||||
clipped: pixels?.clipped ?? null,
|
||||
...originalSize(source, pixels),
|
||||
});
|
||||
}
|
||||
|
||||
/** A thread that cannot start fails every photo alike; the log says so once, not once per photo. */
|
||||
private warnOnceUnavailable(reason: string, err: unknown): void {
|
||||
if (reason !== 'unavailable' || this.warnedUnavailable) return;
|
||||
this.warnedUnavailable = true;
|
||||
const cause = err instanceof Error && err.cause instanceof Error ? `: ${err.cause.message}` : '';
|
||||
logWarn(`[photo-signals] the measuring thread is unavailable${cause}; photos stay pending`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The original's size: what the decoded original measured, or what the
|
||||
* provider says of it. A provider may report the sensor's sides, before the
|
||||
* camera's rotation; the thumbnail is drawn upright, so when the two disagree
|
||||
* on which side is the long one, the provider's sides are turned to match.
|
||||
*/
|
||||
function originalSize(source: PhotoSignalSource, pixels: PhotoPixels | null) {
|
||||
if (source.kind !== 'thumbnail') return { orig_width: pixels?.width ?? null, orig_height: pixels?.height ?? null };
|
||||
const { width, height } = source;
|
||||
const turned =
|
||||
width !== null &&
|
||||
height !== null &&
|
||||
pixels !== null &&
|
||||
Math.sign(width - height) * Math.sign(pixels.width - pixels.height) === -1;
|
||||
return turned ? { orig_width: height, orig_height: width } : { orig_width: width, orig_height: height };
|
||||
}
|
||||
|
||||
function toSignals(row: TrekPhotoSignalRow): PhotoPickSignals {
|
||||
return {
|
||||
phash: row.phash,
|
||||
sharpness: row.sharpness,
|
||||
clipped: row.clipped,
|
||||
origWidth: row.orig_width,
|
||||
origHeight: row.orig_height,
|
||||
};
|
||||
}
|
||||
|
||||
/** Resolves when `work` settles or after `ms`, whichever comes first; never rejects. */
|
||||
function waitAtMost(work: Promise<unknown>, ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
timer.unref();
|
||||
const done = () => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
};
|
||||
work.then(done, done);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { measurePhoto } from './photo-signals-runner';
|
||||
|
||||
import { parentPort } from 'node:worker_threads';
|
||||
|
||||
/**
|
||||
* The photo signals' thread: says it is ready once its script has loaded (so
|
||||
* the runner can tell a thread that never started from a photo that broke
|
||||
* one), then measures the photos it is sent and answers each by its id.
|
||||
* `measurePhoto` answers null for anything it cannot read and never throws, so
|
||||
* every photo gets its answer.
|
||||
*/
|
||||
parentPort?.on('message', (message: { id: number; bytes?: Uint8Array }) => {
|
||||
if (!message.bytes) return;
|
||||
void measurePhoto(message.bytes).then((pixels) => parentPort?.postMessage({ id: message.id, pixels }));
|
||||
});
|
||||
parentPort?.postMessage({ ready: true });
|
||||
@@ -0,0 +1,141 @@
|
||||
import type { PhotoPixels, PhotoSignalsRunner } from './photo-signals-runner';
|
||||
|
||||
import { join } from 'node:path';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
|
||||
/** A photo is decoded and measured in well under this; one that takes longer counts as unmeasurable. */
|
||||
export const MEASURE_TIMEOUT_MS = 20_000;
|
||||
/** With this many photos already in line for the thread, a new one is turned away as busy. */
|
||||
export const MAX_WAITING_PHOTOS = 16;
|
||||
/** A thread with nothing to do ends after ten minutes; the next photo starts one again. */
|
||||
const IDLE_MS = 10 * 60 * 1000;
|
||||
/** The thread's heap. The decoded pixels sit outside it, which is why the decoder checks a photo's size first. */
|
||||
const SIGNALS_HEAP_MB = 256;
|
||||
|
||||
type Reply = { id: number; pixels?: PhotoPixels | null; error?: string };
|
||||
/** What the thread sends once its script has loaded, before any photo. */
|
||||
type Ready = { ready: true };
|
||||
|
||||
/**
|
||||
* The photo signals' own thread, so decoding a photo never holds up the
|
||||
* server's requests (the piste router's thread is the precedent). Photos go to
|
||||
* it one at a time, in the order they came, each on its own clock from when
|
||||
* the thread takes it up: a photo not answered in time fails as `timeout` and
|
||||
* the thread is replaced, a thread that dies fails the photo it held, and the
|
||||
* next photo starts a fresh one. Started on the first photo, ended when idle.
|
||||
*
|
||||
* Only a thread that has said it is ready can blame a photo. One that cannot be
|
||||
* started, or dies before its script has loaded (a broken build, a bad flag),
|
||||
* fails its photos as `unavailable`, and one ended by `close()` as `closed`:
|
||||
* both say nothing about the photo, so the caller tries it again later.
|
||||
*/
|
||||
export class WorkerPhotoSignalsRunner implements PhotoSignalsRunner {
|
||||
private thread: Worker | null = null;
|
||||
private line: Promise<unknown> = Promise.resolve();
|
||||
private inLine = 0;
|
||||
private lastId = 0;
|
||||
private closed = false;
|
||||
private idleTimer: NodeJS.Timeout | null = null;
|
||||
/** The threads that have said they are ready. */
|
||||
private readonly started = new WeakSet<Worker>();
|
||||
|
||||
constructor(private readonly entry = join(__dirname, 'photo-signals.worker.js')) {}
|
||||
|
||||
measure(bytes: Uint8Array): Promise<PhotoPixels | null> {
|
||||
if (this.inLine >= MAX_WAITING_PHOTOS) return Promise.reject(new Error('busy'));
|
||||
this.inLine++;
|
||||
const turn = this.line.then(() => this.ask(bytes));
|
||||
this.line = turn.catch(() => undefined);
|
||||
return turn.finally(() => {
|
||||
this.inLine--;
|
||||
});
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.closed = true;
|
||||
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||
this.idleTimer = null;
|
||||
const worker = this.thread;
|
||||
this.thread = null;
|
||||
await worker?.terminate();
|
||||
}
|
||||
|
||||
/** One photo to the thread and its answer back. */
|
||||
private ask(bytes: Uint8Array): Promise<PhotoPixels | null> {
|
||||
if (this.closed) return Promise.reject(new Error('closed'));
|
||||
let worker: Worker;
|
||||
try {
|
||||
worker = this.ensureThread();
|
||||
} catch (err: unknown) {
|
||||
return Promise.reject(new Error('unavailable', { cause: err }));
|
||||
}
|
||||
const id = ++this.lastId;
|
||||
return new Promise<PhotoPixels | null>((resolve, reject) => {
|
||||
const settle = (outcome: () => void) => {
|
||||
clearTimeout(timer);
|
||||
worker.off('message', onMessage);
|
||||
worker.off('error', onError);
|
||||
worker.off('exit', onExit);
|
||||
outcome();
|
||||
};
|
||||
const onMessage = (reply: Reply | Ready) => {
|
||||
if (!('id' in reply) || reply.id !== id) return;
|
||||
const { pixels, error } = reply;
|
||||
if (pixels === undefined) settle(() => reject(new Error(error ?? 'measurement failed')));
|
||||
else settle(() => resolve(pixels));
|
||||
};
|
||||
// Why the photo failed: the thread's own fault once it was ready, else nothing to do with the photo.
|
||||
const blame = (err: Error) => {
|
||||
if (this.closed) return new Error('closed');
|
||||
return this.started.has(worker) ? err : new Error('unavailable', { cause: err });
|
||||
};
|
||||
const onError = (err: Error) => settle(() => reject(blame(err)));
|
||||
const onExit = () => settle(() => reject(blame(new Error('signals thread ended'))));
|
||||
const timer = setTimeout(() => {
|
||||
// The thread is stuck on this photo; the next photo gets a fresh one.
|
||||
this.thread = null;
|
||||
void worker.terminate();
|
||||
settle(() => reject(new Error('timeout')));
|
||||
}, MEASURE_TIMEOUT_MS);
|
||||
worker.on('message', onMessage);
|
||||
worker.on('error', onError);
|
||||
worker.on('exit', onExit);
|
||||
worker.postMessage({ id, bytes });
|
||||
});
|
||||
}
|
||||
|
||||
private ensureThread(): Worker {
|
||||
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||
this.idleTimer = setTimeout(() => void this.endIdle(), IDLE_MS);
|
||||
this.idleTimer.unref();
|
||||
if (this.thread) return this.thread;
|
||||
// The compiled thread needs the same --require hooks (path aliases) the server was started with.
|
||||
const hooks = process.execArgv.filter(
|
||||
(arg, i, all) => ['--require', '-r'].includes(arg) || ['--require', '-r'].includes(all[i - 1] ?? ''),
|
||||
);
|
||||
const worker = new Worker(this.entry, {
|
||||
execArgv: hooks,
|
||||
resourceLimits: { maxOldGenerationSizeMb: SIGNALS_HEAP_MB },
|
||||
});
|
||||
// Kept for the thread's whole life: a thread error with no listener would take the server down.
|
||||
const forget = () => {
|
||||
if (this.thread === worker) this.thread = null;
|
||||
};
|
||||
// Any message, its first being `ready`, says the script has loaded.
|
||||
worker.on('message', () => {
|
||||
this.started.add(worker);
|
||||
});
|
||||
worker.on('error', forget);
|
||||
worker.on('exit', forget);
|
||||
worker.unref();
|
||||
this.thread = worker;
|
||||
return worker;
|
||||
}
|
||||
|
||||
/** An idle thread ends; the runner stays open for the next photo. */
|
||||
private async endIdle(): Promise<void> {
|
||||
const worker = this.thread;
|
||||
this.thread = null;
|
||||
await worker?.terminate();
|
||||
}
|
||||
}
|
||||
@@ -53,7 +53,8 @@ describe('ALL_ENTITIES', () => {
|
||||
* + tour_drafts, the Tours planner's unsaved draft per user and trip
|
||||
* + ai_connections, the AI assistant's connections to a language model
|
||||
* + assistant_threads, assistant_messages, assistant_tool_calls and assistant_usage,
|
||||
* the trip assistant's conversations, the changes it proposed and what it used)
|
||||
* the trip assistant's conversations, the changes it proposed and what it used
|
||||
* + trek_photo_signals, what a photo's pixels say, for picking photos for a book)
|
||||
* is registered, and no two share a class name (a silent duplicate would
|
||||
* shadow one of them in `orm.getMetadata()`, exactly the kind of drift
|
||||
* `entity-schema-parity.test.ts` can't see — it iterates the metadata map,
|
||||
|
||||
@@ -53,7 +53,8 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
// assistant_messages: id, thread_id, role, parts; assistant_tool_calls: id, thread_id, message_id,
|
||||
// provider_call_id, tool_name, args, preview; assistant_usage: user_id, day, paid_by (each row is
|
||||
// written whole by the assistant, which always knows all of them).
|
||||
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 566;
|
||||
// + trek_photo_signals: id, photo_id, algo (a measurement always names its photo and how it was made).
|
||||
const KNOWN_UNINITIALISED_NOT_NULL_SCALARS = 569;
|
||||
|
||||
describe('entity class fields never carry a NaN initialiser (task-4-review-shape.md, Important 1)', () => {
|
||||
it('CLASSFIELD-001: new X() has no own property whose value is NaN, across all 133 entities', () => {
|
||||
|
||||
@@ -91,6 +91,7 @@ const UNNUMBERED = [
|
||||
'Migration20200101046210_the_trip_assistant_remembers_its_conversations',
|
||||
'Migration20200101046220_a_chat_message_can_say_it_came_from_the_assistant',
|
||||
'Migration20200101046230_an_admin_can_share_a_chatgpt_plan',
|
||||
'Migration20200101046240_a_photo_keeps_its_quality_signals',
|
||||
];
|
||||
// The last step the positional runner ever had; planLegacyBaseline refuses a
|
||||
// schema_version past it.
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* `TrekPhotoSignalsRepository` (PSIGREPO-*): one measurement per photo,
|
||||
* replaced by the next, gone with its photo; read back for many photos at once
|
||||
* past SQLite's bound-value limit; and the gallery facts the book photo picker
|
||||
* starts from, in gallery order and for one journey only.
|
||||
*/
|
||||
import { JourneyPhotos } from '../../../../src/db/entities/JourneyPhotos.entity';
|
||||
import { TrekPhotoSignals } from '../../../../src/db/entities/TrekPhotoSignals.entity';
|
||||
import { TrekPhotos } from '../../../../src/db/entities/TrekPhotos.entity';
|
||||
import type {
|
||||
TrekPhotoSignalsRepository,
|
||||
TrekPhotoSignalWrite,
|
||||
} from '../../../../src/db/repositories/TrekPhotoSignals.repository';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { makeJourney } from '../../../helpers/factories/journeys';
|
||||
import { countRows, deleteRows, insertRow } from '../../../helpers/factories/rows';
|
||||
import { makeUser } from '../../../helpers/factories/users';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
const testDb = createSnapshotTestDb();
|
||||
let t: TestOrm;
|
||||
let signals: TrekPhotoSignalsRepository;
|
||||
|
||||
beforeAll(async () => {
|
||||
t = await createTestOrm(testDb);
|
||||
signals = t.repo(TrekPhotoSignals);
|
||||
});
|
||||
beforeEach(() => {
|
||||
resetTestDb(testDb);
|
||||
t.clear();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
testDb.close();
|
||||
});
|
||||
|
||||
const measured = (photoId: number, over: Partial<TrekPhotoSignalWrite> = {}): TrekPhotoSignalWrite => ({
|
||||
photo_id: photoId,
|
||||
algo: 1,
|
||||
phash: '0123456789abcdef',
|
||||
sharpness: 412.5,
|
||||
luma: 0.48,
|
||||
clipped: 0.01,
|
||||
orig_width: 4032,
|
||||
orig_height: 3024,
|
||||
...over,
|
||||
});
|
||||
|
||||
async function trekPhoto(ownerId: number, over: Partial<{ taken_at: string; media_type: string }> = {}) {
|
||||
return insertRow(t, TrekPhotos, { provider: 'local', owner: ownerId, file_path: 'journey/p.jpg', ...over });
|
||||
}
|
||||
|
||||
describe('TrekPhotoSignalsRepository', () => {
|
||||
it('PSIGREPO-001: a measurement reads back as written, and the next one of the same photo replaces it', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const photoId = await trekPhoto(user.id);
|
||||
expect(await signals.listForPhotos([photoId])).toEqual([]);
|
||||
|
||||
await signals.saveSignals(measured(photoId));
|
||||
const [first] = await signals.listForPhotos([photoId]);
|
||||
expect(first).toMatchObject(measured(photoId));
|
||||
expect(first?.computed_at).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/);
|
||||
|
||||
const unmeasurable = measured(photoId, {
|
||||
algo: 2,
|
||||
phash: null,
|
||||
sharpness: null,
|
||||
luma: null,
|
||||
clipped: null,
|
||||
orig_width: null,
|
||||
orig_height: null,
|
||||
});
|
||||
await signals.saveSignals(unmeasurable);
|
||||
expect(await signals.listForPhotos([photoId])).toEqual([expect.objectContaining(unmeasurable)]);
|
||||
expect(await countRows(t, TrekPhotoSignals)).toBe(1);
|
||||
});
|
||||
|
||||
it('PSIGREPO-002: reads the measurements of more photos than one statement may bind', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const first = await trekPhoto(user.id);
|
||||
const last = await trekPhoto(user.id);
|
||||
await signals.saveSignals(measured(first));
|
||||
await signals.saveSignals(measured(last, { sharpness: 12 }));
|
||||
const ids = [first, ...Array.from({ length: 450 }, (_, i) => 100_000 + i), last];
|
||||
const rows = await signals.listForPhotos(ids);
|
||||
expect(rows.map((r) => [r.photo_id, r.sharpness]).sort((a, b) => Number(a[0]) - Number(b[0]))).toEqual([
|
||||
[first, 412.5],
|
||||
[last, 12],
|
||||
]);
|
||||
});
|
||||
|
||||
it('PSIGREPO-003: a measurement goes with its photo', async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const photoId = await trekPhoto(user.id);
|
||||
await signals.saveSignals(measured(photoId));
|
||||
await deleteRows(t, TrekPhotos, { id: photoId });
|
||||
expect(await countRows(t, TrekPhotoSignals)).toBe(0);
|
||||
});
|
||||
|
||||
it("PSIGREPO-004: the gallery facts are one journey's photos in gallery order, with when, where and what they are", async () => {
|
||||
const { user } = await makeUser(t);
|
||||
const journey = await makeJourney(t, user.id);
|
||||
const other = await makeJourney(t, user.id);
|
||||
const late = await trekPhoto(user.id, { taken_at: '2026-07-03T10:00:00' });
|
||||
const early = await insertRow(t, TrekPhotos, {
|
||||
provider: 'immich',
|
||||
asset_id: 'a1',
|
||||
owner: user.id,
|
||||
width: 1440,
|
||||
height: 1080,
|
||||
lat: 38.71,
|
||||
lng: -9.13,
|
||||
});
|
||||
const clip = await trekPhoto(user.id, { media_type: 'video' });
|
||||
const elsewhere = await trekPhoto(user.id);
|
||||
const now = Date.now();
|
||||
await insertRow(t, JourneyPhotos, { journey: journey.id, photo: late, sort_order: 2, created_at: now });
|
||||
await insertRow(t, JourneyPhotos, {
|
||||
journey: journey.id,
|
||||
photo: early,
|
||||
caption: 'Tram 28',
|
||||
sort_order: 1,
|
||||
created_at: now,
|
||||
});
|
||||
await insertRow(t, JourneyPhotos, { journey: journey.id, photo: clip, sort_order: 2, created_at: now });
|
||||
await insertRow(t, JourneyPhotos, { journey: other.id, photo: elsewhere, sort_order: 0, created_at: now });
|
||||
|
||||
expect(await signals.listGalleryFacts(journey.id)).toEqual([
|
||||
{
|
||||
photo_id: early,
|
||||
caption: 'Tram 28',
|
||||
provider: 'immich',
|
||||
media_type: 'image',
|
||||
width: 1440,
|
||||
height: 1080,
|
||||
taken_at: null,
|
||||
lat: 38.71,
|
||||
lng: -9.13,
|
||||
},
|
||||
expect.objectContaining({ photo_id: late, caption: null, taken_at: '2026-07-03T10:00:00', media_type: 'image' }),
|
||||
expect.objectContaining({ photo_id: clip, media_type: 'video' }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,9 @@ import { TrekPhotoRegistrationService } from '../../../src/nest/photos/trek-phot
|
||||
import { deleteRows, findRow, insertRow, insertRowIgnoringConflict } from '../../helpers/factories/rows';
|
||||
import { createTestOrm, type TestOrm } from '../../helpers/test-orm';
|
||||
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { describe, it, expect, vi, beforeAll, beforeEach, afterAll } from 'vitest';
|
||||
|
||||
vi.mock('../../../src/db/database', async () => {
|
||||
@@ -67,7 +70,7 @@ const cache = {
|
||||
};
|
||||
// Local bytes go through the storage facade (slice 3); the resolver only ever
|
||||
// addresses the 'journey' category with the 'journey/' prefix stripped.
|
||||
const storage = { exists: vi.fn(), sendToResponse: vi.fn() };
|
||||
const storage = { exists: vi.fn(), sendToResponse: vi.fn(), stat: vi.fn(), withLocalFile: vi.fn() };
|
||||
|
||||
// Real adapters over the stubbed services, and a real registry: the cases below
|
||||
// keep asserting on immich.streamImmichAsset/synology.streamSynologyAsset, so
|
||||
@@ -361,3 +364,61 @@ describe('getPhotoInfo — dispatch', () => {
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('readForSignals: what the quality signals are measured on', () => {
|
||||
let dir: string;
|
||||
let original: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'trek-resolver-signals-'));
|
||||
original = join(dir, 'p.jpg');
|
||||
await writeFile(original, Buffer.from('jpeg bytes'));
|
||||
});
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('RESOLVE-020: a local original is read whole through storage, unless it is over the cap, missing or a video', async () => {
|
||||
const id = await insertPhoto({ provider: 'local', file_path: 'journey/p.jpg' });
|
||||
storage.stat.mockResolvedValue({ key: 'p.jpg', size: 10, mtimeMs: 0 });
|
||||
storage.withLocalFile.mockImplementation((_category: string, _name: string, fn: (path: string) => unknown) =>
|
||||
fn(original),
|
||||
);
|
||||
|
||||
expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'original', bytes: Buffer.from('jpeg bytes') });
|
||||
expect(storage.withLocalFile).toHaveBeenCalledWith('journey', 'p.jpg', expect.any(Function));
|
||||
expect(await svc.readForSignals(id, 5)).toEqual({ kind: 'none' });
|
||||
storage.stat.mockRejectedValue(new Error('invalid key'));
|
||||
expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'none' });
|
||||
|
||||
const outside = await insertPhoto({ provider: 'local', file_path: 'elsewhere/p.jpg' });
|
||||
const clip = await insertPhoto({ provider: 'local', file_path: 'journey/c.mp4', media_type: 'video' });
|
||||
const bare = await insertPhoto({ provider: 'local' });
|
||||
for (const photoId of [outside, clip, bare, 999999]) {
|
||||
expect(await svc.readForSignals(photoId, 100)).toEqual({ kind: 'none' });
|
||||
}
|
||||
expect(storage.withLocalFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("RESOLVE-021: a provider photo answers its thumbnail and the original's size, asked with its owner's credentials", async () => {
|
||||
const id = await insertPhoto({ provider: 'immich', asset_id: 'a1', owner_id: 5 });
|
||||
const bytes = Buffer.from('thumb');
|
||||
immich.fetchImmichThumbnailBytes.mockResolvedValue({ bytes, contentType: 'image/webp' });
|
||||
immich.getAssetInfo.mockResolvedValue({ success: true, data: { id: 'a1', width: 6000, height: 4000 } });
|
||||
|
||||
expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'thumbnail', bytes, width: 6000, height: 4000 });
|
||||
expect(immich.fetchImmichThumbnailBytes).toHaveBeenCalledWith(5, 'a1', 5);
|
||||
expect(immich.getAssetInfo).toHaveBeenCalledWith(5, 'a1', 5);
|
||||
|
||||
immich.getAssetInfo.mockResolvedValue({ error: 'offline', status: 502 });
|
||||
expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'thumbnail', bytes, width: null, height: null });
|
||||
|
||||
immich.fetchImmichThumbnailBytes.mockResolvedValue({ error: 'offline', status: 502 });
|
||||
expect(await svc.readForSignals(id, 100)).toEqual({ kind: 'unavailable' });
|
||||
|
||||
const unknown = await insertPhoto({ provider: 'picasa', asset_id: 'p1', owner_id: 2 });
|
||||
const ownerless = await insertPhoto({ provider: 'immich', asset_id: 'a2' });
|
||||
expect(await svc.readForSignals(unknown, 100)).toEqual({ kind: 'none' });
|
||||
expect(await svc.readForSignals(ownerless, 100)).toEqual({ kind: 'none' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* Measuring a photo's pixels (PHSIG-*): on photos Jimp draws here, a sharp
|
||||
* scene against the same scene blurred, a copy shifted by a few pixels against
|
||||
* the scene inverted, plain white and black, one scene drawn at a thumbnail's
|
||||
* size and at an original's, bytes that are no image, and headers declaring a
|
||||
* canvas too large to decode. Then the thread script, the runner that talks to
|
||||
* a thread (with small threads the test writes that answer, fail, hang, die or
|
||||
* never start) and the factory that picks the thread or the calling thread.
|
||||
*/
|
||||
import {
|
||||
bitsToHex,
|
||||
InlinePhotoSignalsRunner,
|
||||
measurePhoto,
|
||||
SIGNAL_EDGE,
|
||||
withinDecodeCap,
|
||||
} from '../../../../src/nest/photos/photo-signals-runner';
|
||||
import { photoSignalsRunner } from '../../../../src/nest/photos/photo-signals.module';
|
||||
import {
|
||||
MAX_WAITING_PHOTOS,
|
||||
MEASURE_TIMEOUT_MS,
|
||||
WorkerPhotoSignalsRunner,
|
||||
} from '../../../../src/nest/photos/worker-photo-signals-runner';
|
||||
import { hashDistance } from '@trek/shared';
|
||||
|
||||
import { Jimp } from 'jimp';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
/** The thread's port, as its script sees it: messages in, messages out. */
|
||||
const port = vi.hoisted(() => {
|
||||
const listeners: Array<(message: unknown) => void> = [];
|
||||
return {
|
||||
on: (event: string, listener: (message: unknown) => void) => {
|
||||
if (event === 'message') listeners.push(listener);
|
||||
},
|
||||
emit: (_event: 'message', message: unknown) => listeners.forEach((listener) => listener(message)),
|
||||
postMessage: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock('node:worker_threads', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('node:worker_threads')>()),
|
||||
parentPort: port,
|
||||
}));
|
||||
|
||||
/** A grey photo drawn pixel by pixel, as JPEG bytes. */
|
||||
async function drawn(shade: (x: number, y: number) => number, width = 256, height = 192, blur = 0): Promise<Buffer> {
|
||||
const image = new Jimp({ width, height, color: 0xffffffff });
|
||||
const { data } = image.bitmap;
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const v = Math.max(0, Math.min(255, Math.round(shade(x, y))));
|
||||
const i = (y * width + x) * 4;
|
||||
data[i] = v;
|
||||
data[i + 1] = v;
|
||||
data[i + 2] = v;
|
||||
}
|
||||
}
|
||||
if (blur > 0) image.blur(blur);
|
||||
return Buffer.from(await image.getBuffer('image/jpeg'));
|
||||
}
|
||||
|
||||
/** Blocks of 32 px, light and dark, over a soft gradient, moved right by `dx`: edges to be sharp or blurred. */
|
||||
function scene(dx = 0) {
|
||||
return (x: number, y: number) => {
|
||||
const light = (Math.floor((x + dx) / 32) + Math.floor(y / 32)) % 2 === 0;
|
||||
return light ? 200 + x / 16 : 50 + y / 16;
|
||||
};
|
||||
}
|
||||
|
||||
/** Scenery without edges on the pixel grid, drawn from 0..1 coordinates so it looks the same at any size. */
|
||||
function landscape(width: number, height: number) {
|
||||
return (x: number, y: number) => {
|
||||
const u = x / width;
|
||||
const v = y / height;
|
||||
const hill = (u - 0.43) ** 2 + (v - 0.51) ** 2 < 0.05 ? 30 : -30;
|
||||
return 128 + 50 * Math.sin(u * 23.1 + v * 7.3) + 40 * Math.sin(v * 31.7 - u * 5.9) + hill;
|
||||
};
|
||||
}
|
||||
|
||||
/** The first bytes of an image file declaring `width` x `height`, with nothing behind them. */
|
||||
function header(kind: 'png' | 'gif' | 'bmp' | 'bmp-core', width: number, height: number): Buffer {
|
||||
const data = Buffer.alloc(40);
|
||||
if (kind === 'png') {
|
||||
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(data, 0);
|
||||
data.writeUInt32BE(13, 8);
|
||||
data.write('IHDR', 12, 'latin1');
|
||||
data.writeUInt32BE(width, 16);
|
||||
data.writeUInt32BE(height, 20);
|
||||
} else if (kind === 'gif') {
|
||||
data.write('GIF89a', 0, 'latin1');
|
||||
data.writeUInt16LE(width, 6);
|
||||
data.writeUInt16LE(height, 8);
|
||||
} else if (kind === 'bmp') {
|
||||
data.write('BM', 0, 'latin1');
|
||||
data.writeUInt32LE(40, 14);
|
||||
data.writeInt32LE(width, 18);
|
||||
data.writeInt32LE(-height, 22);
|
||||
} else {
|
||||
data.write('BM', 0, 'latin1');
|
||||
data.writeUInt32LE(12, 14);
|
||||
data.writeUInt16LE(width, 18);
|
||||
data.writeUInt16LE(height, 20);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
let dir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'trek-photo-signals-'));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('measurePhoto', () => {
|
||||
it('PHSIG-001: a blurred copy measures far less sharp than the sharp scene', async () => {
|
||||
const sharp = await measurePhoto(await drawn(scene()));
|
||||
const blurred = await measurePhoto(await drawn(scene(), 256, 192, 4));
|
||||
expect(sharp?.sharpness).toBeGreaterThan(4 * (blurred?.sharpness ?? Infinity));
|
||||
expect(sharp).toMatchObject({ width: 256, height: 192 });
|
||||
});
|
||||
|
||||
it('PHSIG-002: a copy shifted by a few pixels hashes alike, the inverted scene does not', async () => {
|
||||
const base = await measurePhoto(await drawn(scene()));
|
||||
const shifted = await measurePhoto(await drawn(scene(3)));
|
||||
const inverted = await measurePhoto(await drawn((x, y) => 255 - scene()(x, y)));
|
||||
expect(base?.phash).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(hashDistance(base?.phash ?? '', shifted?.phash ?? '')).toBeLessThanOrEqual(10);
|
||||
expect(hashDistance(base?.phash ?? '', inverted?.phash ?? '')).toBeGreaterThan(10);
|
||||
});
|
||||
|
||||
it('PHSIG-003: brightness and clipping, and the size of a large photo as it was before it was measured smaller', async () => {
|
||||
const white = await measurePhoto(await drawn(() => 255));
|
||||
expect(white).toMatchObject({ luma: 1, clipped: 1 });
|
||||
const black = await measurePhoto(await drawn(() => 0));
|
||||
expect(black).toMatchObject({ luma: 0, clipped: 1, sharpness: 0 });
|
||||
const grey = await measurePhoto(await drawn(() => 128));
|
||||
expect(grey?.clipped).toBe(0);
|
||||
expect(grey?.luma).toBeCloseTo(128 / 255, 2);
|
||||
const large = await measurePhoto(await drawn(scene(), 1600, 1200));
|
||||
expect(large).toMatchObject({ width: 1600, height: 1200 });
|
||||
expect(SIGNAL_EDGE).toBe(240);
|
||||
});
|
||||
|
||||
it('PHSIG-004: bytes that are no image, or an image too small or too thin to measure, have no signals', async () => {
|
||||
expect(await measurePhoto(Buffer.from('not a photo at all'))).toBeNull();
|
||||
expect(await measurePhoto(await drawn(() => 128, 2, 2))).toBeNull();
|
||||
// 1000 x 4 scaled to a 240 px long edge is under one pixel high.
|
||||
expect(await measurePhoto(await drawn(() => 128, 1000, 4))).toBeNull();
|
||||
});
|
||||
|
||||
it('PHSIG-005: the 64 bits of a hash become 16 hex digits; anything else is no hash', () => {
|
||||
expect(bitsToHex(`${'0'.repeat(60)}1111`)).toBe('000000000000000f');
|
||||
expect(bitsToHex('1'.repeat(64))).toBe('ffffffffffffffff');
|
||||
expect(bitsToHex('0101')).toBeNull();
|
||||
expect(bitsToHex('2'.repeat(64))).toBeNull();
|
||||
});
|
||||
|
||||
it("PHSIG-007: one scene measures about as sharp at a thumbnail's 320 px as at an original's 2000 px", async () => {
|
||||
const thumbnail = await measurePhoto(await drawn(landscape(320, 240), 320, 240));
|
||||
const original = await measurePhoto(await drawn(landscape(2000, 1500), 2000, 1500));
|
||||
const ratio = (original?.sharpness ?? 0) / (thumbnail?.sharpness ?? Infinity);
|
||||
expect(ratio).toBeGreaterThan(0.75);
|
||||
expect(ratio).toBeLessThan(1.33);
|
||||
expect(original).toMatchObject({ width: 2000, height: 1500 });
|
||||
});
|
||||
|
||||
it('PHSIG-008: a canvas declared past 48 megapixels is refused before it is decoded; PNG and BMP within it are measured', async () => {
|
||||
const read = vi.spyOn(Jimp, 'read');
|
||||
try {
|
||||
expect(withinDecodeCap(header('png', 30000, 30000))).toBe(false);
|
||||
expect(await measurePhoto(header('png', 30000, 30000))).toBeNull();
|
||||
expect(withinDecodeCap(header('gif', 65535, 65535))).toBe(false);
|
||||
expect(withinDecodeCap(header('bmp', 10000, 10000))).toBe(false);
|
||||
expect(withinDecodeCap(header('bmp-core', 60000, 60000))).toBe(false);
|
||||
expect(withinDecodeCap(Buffer.from('II*\0 a TIFF Jimp could read'))).toBe(false);
|
||||
expect(read).not.toHaveBeenCalled();
|
||||
expect(withinDecodeCap(header('gif', 640, 480))).toBe(true);
|
||||
expect(withinDecodeCap(header('bmp', 6000, 4000))).toBe(true);
|
||||
expect(withinDecodeCap(header('bmp-core', 640, 480))).toBe(true);
|
||||
} finally {
|
||||
read.mockRestore();
|
||||
}
|
||||
const grey = new Jimp({ width: 40, height: 30, color: 0x808080ff });
|
||||
const png = Buffer.from(await grey.getBuffer('image/png'));
|
||||
const bmp = Buffer.from(await grey.getBuffer('image/bmp'));
|
||||
expect(await measurePhoto(png)).toMatchObject({ width: 40, height: 30, clipped: 0 });
|
||||
expect(await measurePhoto(bmp)).toMatchObject({ width: 40, height: 30, clipped: 0 });
|
||||
expect(withinDecodeCap(await drawn(scene()))).toBe(true);
|
||||
});
|
||||
|
||||
it('PHSIG-006: the runner on the calling thread measures the same, and closes', async () => {
|
||||
const bytes = await drawn(scene());
|
||||
const runner = new InlinePhotoSignalsRunner();
|
||||
expect(await runner.measure(bytes)).toEqual(await measurePhoto(bytes));
|
||||
await expect(runner.close()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('the thread script', () => {
|
||||
it('PHSIG-010: says it is ready, answers each photo by its id, an unreadable one with no signals, and ignores a message without bytes', async () => {
|
||||
await import('../../../../src/nest/photos/photo-signals.worker');
|
||||
expect(port.postMessage).toHaveBeenCalledWith({ ready: true });
|
||||
port.emit('message', { id: 1, bytes: new Uint8Array(await drawn(scene())) });
|
||||
await vi.waitFor(() =>
|
||||
expect(port.postMessage).toHaveBeenCalledWith({ id: 1, pixels: expect.objectContaining({ width: 256 }) }),
|
||||
);
|
||||
port.emit('message', { id: 2 });
|
||||
port.emit('message', { id: 3, bytes: new Uint8Array([1, 2, 3]) });
|
||||
await vi.waitFor(() => expect(port.postMessage).toHaveBeenCalledWith({ id: 3, pixels: null }));
|
||||
expect(port.postMessage).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* A thread script the runner can start: plain JS, so it runs without the
|
||||
* server's loader. Like the real one it says it is ready once loaded, unless
|
||||
* the case wants a thread that never gets that far.
|
||||
*/
|
||||
async function script(name: string, body: string, ready = true): Promise<string> {
|
||||
const path = join(dir, `${name}.cjs`);
|
||||
const hello = ready ? `parentPort.postMessage({ ready: true });\n` : '';
|
||||
await writeFile(path, `const { parentPort } = require('node:worker_threads');\n${body}\n${hello}`);
|
||||
return path;
|
||||
}
|
||||
|
||||
const PIXELS = { phash: null, sharpness: 1, luma: 0.5, clipped: 0, width: 4, height: 3 };
|
||||
|
||||
describe('WorkerPhotoSignalsRunner', () => {
|
||||
it('PHSIG-020: hands each photo to its thread in turn and answers what it says, by its id; a failure it reports rejects', async () => {
|
||||
const entry = await script(
|
||||
'answers',
|
||||
`parentPort.on('message', (m) => {
|
||||
if (m.bytes[0] === 1) parentPort.postMessage({ id: m.id, pixels: ${JSON.stringify(PIXELS)} });
|
||||
else if (m.bytes[0] === 2) parentPort.postMessage({ id: m.id, pixels: null });
|
||||
else if (m.bytes[0] === 3) parentPort.postMessage({ id: m.id, error: 'cannot decode' });
|
||||
else {
|
||||
parentPort.postMessage({ id: -1, pixels: null });
|
||||
parentPort.postMessage({ id: m.id });
|
||||
}
|
||||
});`,
|
||||
);
|
||||
const runner = new WorkerPhotoSignalsRunner(entry);
|
||||
const answers = await Promise.allSettled([1, 2, 3, 4].map((b) => runner.measure(new Uint8Array([b]))));
|
||||
expect(answers.map((a) => (a.status === 'fulfilled' ? a.value : (a.reason as Error).message))).toEqual([
|
||||
PIXELS,
|
||||
null,
|
||||
'cannot decode',
|
||||
'measurement failed',
|
||||
]);
|
||||
await runner.close();
|
||||
await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('closed');
|
||||
});
|
||||
|
||||
it('PHSIG-021: a thread that dies or throws fails the photo it held, and the next photo gets a fresh thread', async () => {
|
||||
const entry = await script(
|
||||
'fragile',
|
||||
`parentPort.on('message', (m) => {
|
||||
if (m.bytes[0] === 9) process.exit(3);
|
||||
if (m.bytes[0] === 8) setTimeout(() => { throw new Error('boom'); });
|
||||
else parentPort.postMessage({ id: m.id, pixels: null });
|
||||
});`,
|
||||
);
|
||||
const runner = new WorkerPhotoSignalsRunner(entry);
|
||||
await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull();
|
||||
await expect(runner.measure(new Uint8Array([9]))).rejects.toThrow('signals thread ended');
|
||||
await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull();
|
||||
await expect(runner.measure(new Uint8Array([8]))).rejects.toThrow('boom');
|
||||
await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull();
|
||||
await runner.close();
|
||||
});
|
||||
|
||||
it('PHSIG-022: a full line is busy, and closing fails the photo with the thread and every one in line as closed', async () => {
|
||||
const entry = await script('silent', `parentPort.on('message', () => undefined);`);
|
||||
const runner = new WorkerPhotoSignalsRunner(entry);
|
||||
const waiting = Array.from({ length: MAX_WAITING_PHOTOS }, () =>
|
||||
runner.measure(new Uint8Array([1])).catch((err: Error) => err.message),
|
||||
);
|
||||
await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('busy');
|
||||
await runner.close();
|
||||
const outcomes = await Promise.all(waiting);
|
||||
expect(outcomes).toEqual(Array.from({ length: MAX_WAITING_PHOTOS }, () => 'closed'));
|
||||
});
|
||||
|
||||
it('PHSIG-023: a photo left unanswered times out on its own clock, the thread is started over for the next', async () => {
|
||||
const entry = await script(
|
||||
'slow',
|
||||
`parentPort.on('message', (m) => {
|
||||
if (m.bytes[0] !== 7) parentPort.postMessage({ id: m.id, pixels: null });
|
||||
});`,
|
||||
);
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
|
||||
try {
|
||||
const runner = new WorkerPhotoSignalsRunner(entry);
|
||||
const hung = runner.measure(new Uint8Array([7])).catch((err: Error) => err.message);
|
||||
const next = runner.measure(new Uint8Array([1]));
|
||||
await vi.advanceTimersByTimeAsync(MEASURE_TIMEOUT_MS);
|
||||
expect(await hung).toBe('timeout');
|
||||
await expect(next).resolves.toBeNull();
|
||||
// Ten idle minutes end the thread; the runner starts another for the next photo.
|
||||
await vi.advanceTimersByTimeAsync(10 * 60 * 1000);
|
||||
await expect(runner.measure(new Uint8Array([1]))).resolves.toBeNull();
|
||||
await runner.close();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('PHSIG-024: a thread that cannot start fails each photo on its own as unavailable', async () => {
|
||||
// A bare relative path is one node refuses to start a thread from.
|
||||
const runner = new WorkerPhotoSignalsRunner('no-such-signals.js');
|
||||
await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('unavailable');
|
||||
await expect(runner.measure(new Uint8Array([1]))).rejects.toThrow('unavailable');
|
||||
await runner.close();
|
||||
});
|
||||
|
||||
it('PHSIG-026: a thread that fails or ends before it is ready blames no photo: unavailable, with the reason', async () => {
|
||||
const broken = await script('broken', `throw new Error('broken build');`, false);
|
||||
const runner = new WorkerPhotoSignalsRunner(broken);
|
||||
await expect(runner.measure(new Uint8Array([1]))).rejects.toMatchObject({
|
||||
message: 'unavailable',
|
||||
cause: expect.objectContaining({ message: 'broken build' }),
|
||||
});
|
||||
await runner.close();
|
||||
const quits = await script('quits', `process.exit(2);`, false);
|
||||
const other = new WorkerPhotoSignalsRunner(quits);
|
||||
await expect(other.measure(new Uint8Array([1]))).rejects.toMatchObject({
|
||||
message: 'unavailable',
|
||||
cause: expect.objectContaining({ message: 'signals thread ended' }),
|
||||
});
|
||||
await other.close();
|
||||
});
|
||||
|
||||
it('PHSIG-025: the module takes the thread where the build ships its script, else the calling thread', async () => {
|
||||
const entry = await script('shipped', `parentPort.on('message', () => undefined);`);
|
||||
const shipped = photoSignalsRunner(entry);
|
||||
expect(shipped).toBeInstanceOf(WorkerPhotoSignalsRunner);
|
||||
await shipped.close();
|
||||
expect(photoSignalsRunner(join(dir, 'missing.js'))).toBeInstanceOf(InlinePhotoSignalsRunner);
|
||||
expect(photoSignalsRunner()).toBeInstanceOf(InlinePhotoSignalsRunner);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,303 @@
|
||||
/**
|
||||
* PhotoSignalsService (PHSVC-*): a photo is measured once and its signals
|
||||
* kept; a caller waits at most so long and is told how many are still
|
||||
* pending; a photo is in line once however often it is asked for, the line is
|
||||
* bounded and fetches are at most two at a time, and photos asked for later
|
||||
* queue behind; a provider that does not answer, a thread that cannot start or
|
||||
* a first timeout is tried again on the next call, a photo that cannot be
|
||||
* measured is not; a provider's turned sides follow its upright thumbnail.
|
||||
* Taken from PhotoSignalsModule over a snapshot test db, the bytes from a
|
||||
* stubbed resolver and, unless a case swaps it, measured on the calling thread.
|
||||
*/
|
||||
import { JourneyPhotos } from '../../../../src/db/entities/JourneyPhotos.entity';
|
||||
import { TrekPhotoSignals } from '../../../../src/db/entities/TrekPhotoSignals.entity';
|
||||
import { TrekPhotos } from '../../../../src/db/entities/TrekPhotos.entity';
|
||||
import { PhotoResolverService, type PhotoSignalSource } from '../../../../src/nest/memories/photo-resolver.service';
|
||||
import {
|
||||
PHOTO_SIGNALS_ALGO,
|
||||
PHOTO_SIGNALS_RUNNER,
|
||||
type PhotoSignalsRunner,
|
||||
} from '../../../../src/nest/photos/photo-signals-runner';
|
||||
import { PhotoSignalsModule } from '../../../../src/nest/photos/photo-signals.module';
|
||||
import {
|
||||
MAX_ORIGINAL_BYTES,
|
||||
MAX_QUEUED_PHOTOS,
|
||||
PARALLEL_FETCHES,
|
||||
PhotoSignalsService,
|
||||
} from '../../../../src/nest/photos/photo-signals.service';
|
||||
import { createSnapshotTestDb } from '../../../helpers/db-mock';
|
||||
import { makeJourney } from '../../../helpers/factories/journeys';
|
||||
import { countRows, insertRow } from '../../../helpers/factories/rows';
|
||||
import { makeUser } from '../../../helpers/factories/users';
|
||||
import { resetTestDb } from '../../../helpers/test-db';
|
||||
import { createTestModule, type TestModule } from '../../../helpers/test-module';
|
||||
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
|
||||
|
||||
import { Jimp } from 'jimp';
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const db = createSnapshotTestDb();
|
||||
const resolver = { readForSignals: vi.fn<(photoId: number, maxBytes: number) => Promise<PhotoSignalSource>>() };
|
||||
let orm: TestOrm;
|
||||
let photo: Buffer;
|
||||
|
||||
/** The service in a container of its own, with the resolver stubbed and, when given, the runner too. */
|
||||
async function boot(runner?: PhotoSignalsRunner): Promise<{ t: TestModule; signals: PhotoSignalsService }> {
|
||||
const t = await createTestModule({
|
||||
db,
|
||||
imports: [PhotoSignalsModule],
|
||||
overrides: [
|
||||
{ provide: PhotoResolverService, useValue: resolver },
|
||||
...(runner ? [{ provide: PHOTO_SIGNALS_RUNNER, useValue: runner }] : []),
|
||||
],
|
||||
});
|
||||
return { t, signals: t.get(PhotoSignalsService) };
|
||||
}
|
||||
|
||||
let main: { t: TestModule; signals: PhotoSignalsService };
|
||||
|
||||
beforeAll(async () => {
|
||||
orm = await createTestOrm(db);
|
||||
main = await boot();
|
||||
photo = Buffer.from(await new Jimp({ width: 300, height: 200, color: 0x808080ff }).getBuffer('image/jpeg'));
|
||||
});
|
||||
beforeEach(() => {
|
||||
resetTestDb(db);
|
||||
orm.clear();
|
||||
resolver.readForSignals.mockReset();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await main.t.close();
|
||||
await orm.close();
|
||||
db.close();
|
||||
});
|
||||
|
||||
/** A journey with these photos in its gallery, in this order; answers their trek_photos ids. */
|
||||
async function gallery(kinds: Array<'image' | 'video'>) {
|
||||
const { user } = await makeUser(orm);
|
||||
const journey = await makeJourney(orm, user.id);
|
||||
const ids: number[] = [];
|
||||
for (const [i, kind] of kinds.entries()) {
|
||||
const id = await insertRow(orm, TrekPhotos, {
|
||||
provider: 'local',
|
||||
owner: user.id,
|
||||
file_path: 'journey/p.jpg',
|
||||
media_type: kind,
|
||||
});
|
||||
await insertRow(orm, JourneyPhotos, { journey: journey.id, photo: id, sort_order: i, created_at: Date.now() });
|
||||
ids.push(id);
|
||||
}
|
||||
return { journeyId: journey.id, ids };
|
||||
}
|
||||
|
||||
/** A promise the case settles when it likes. */
|
||||
function deferred<T>() {
|
||||
let resolve: (value: T) => void = () => undefined;
|
||||
const promise = new Promise<T>((r) => {
|
||||
resolve = r;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe('PhotoSignalsService', () => {
|
||||
it('PHSVC-001: measures a gallery photo once, keeps it, and leaves the videos out', async () => {
|
||||
const empty = await gallery([]);
|
||||
expect(await main.signals.galleryFacts(empty.journeyId)).toEqual([]);
|
||||
expect(await main.signals.signalsFor([])).toEqual({ byPhoto: new Map(), pending: 0 });
|
||||
const { journeyId, ids } = await gallery(['image', 'video']);
|
||||
const id = ids[0] ?? 0;
|
||||
resolver.readForSignals.mockResolvedValue({ kind: 'original', bytes: photo });
|
||||
|
||||
const facts = await main.signals.galleryFacts(journeyId);
|
||||
expect(facts.map((f) => f.photo_id)).toEqual([id]);
|
||||
const first = await main.signals.signalsFor(facts.map((f) => f.photo_id));
|
||||
expect(first.pending).toBe(0);
|
||||
expect(first.byPhoto.get(id)).toMatchObject({ origWidth: 300, origHeight: 200, clipped: 0 });
|
||||
expect(first.byPhoto.get(id)?.phash).toMatch(/^[0-9a-f]{16}$/);
|
||||
expect(resolver.readForSignals).toHaveBeenCalledWith(id, MAX_ORIGINAL_BYTES);
|
||||
|
||||
const again = await main.signals.signalsFor([id]);
|
||||
expect(again.byPhoto.get(id)).toEqual(first.byPhoto.get(id));
|
||||
expect(resolver.readForSignals).toHaveBeenCalledTimes(1);
|
||||
expect(await countRows(orm, TrekPhotoSignals, { photo: ids[0], algo: PHOTO_SIGNALS_ALGO })).toBe(1);
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(1);
|
||||
});
|
||||
|
||||
it("PHSVC-002: a provider's thumbnail keeps the original's size it reports; nothing to measure is an answer, no answer is not", async () => {
|
||||
const { ids } = await gallery(['image', 'image', 'image']);
|
||||
const [thumb, none, offline] = ids as [number, number, number];
|
||||
const sources = new Map<number, PhotoSignalSource>([
|
||||
[thumb, { kind: 'thumbnail', bytes: photo, width: 6000, height: 4000 }],
|
||||
[none, { kind: 'none' }],
|
||||
]);
|
||||
resolver.readForSignals.mockImplementation((photoId) =>
|
||||
Promise.resolve<PhotoSignalSource>(sources.get(photoId) ?? { kind: 'unavailable' }),
|
||||
);
|
||||
|
||||
const first = await main.signals.signalsFor(ids);
|
||||
expect(first.pending).toBe(1);
|
||||
expect(first.byPhoto.get(thumb)).toMatchObject({ origWidth: 6000, origHeight: 4000 });
|
||||
expect(first.byPhoto.get(thumb)?.sharpness).toEqual(expect.any(Number));
|
||||
expect(first.byPhoto.get(none)).toEqual({
|
||||
phash: null,
|
||||
sharpness: null,
|
||||
clipped: null,
|
||||
origWidth: null,
|
||||
origHeight: null,
|
||||
});
|
||||
expect(first.byPhoto.has(offline)).toBe(false);
|
||||
|
||||
await main.signals.signalsFor(ids);
|
||||
expect(resolver.readForSignals.mock.calls.map(([id]) => id)).toEqual([thumb, none, offline, offline]);
|
||||
});
|
||||
|
||||
it('PHSVC-003: a measurement of an older version counts as missing and is made again', async () => {
|
||||
const { ids } = await gallery(['image']);
|
||||
const id = ids[0] ?? 0;
|
||||
await insertRow(orm, TrekPhotoSignals, { photo: id, algo: PHOTO_SIGNALS_ALGO - 1, sharpness: 1 });
|
||||
resolver.readForSignals.mockResolvedValue({ kind: 'original', bytes: photo });
|
||||
|
||||
const { byPhoto } = await main.signals.signalsFor([id]);
|
||||
expect(byPhoto.get(id)?.origWidth).toBe(300);
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(1);
|
||||
});
|
||||
|
||||
it('PHSVC-004: waits at most as long as told, and a photo asked for twice is in line once', async () => {
|
||||
const { ids } = await gallery(['image']);
|
||||
const id = ids[0] ?? 0;
|
||||
const bytes = deferred<PhotoSignalSource>();
|
||||
resolver.readForSignals.mockReturnValue(bytes.promise);
|
||||
|
||||
const [a, b] = await Promise.all([
|
||||
main.signals.signalsFor([id], { waitMs: 20 }),
|
||||
main.signals.signalsFor([id], { waitMs: 20 }),
|
||||
]);
|
||||
expect([a.pending, b.pending]).toEqual([1, 1]);
|
||||
expect(resolver.readForSignals).toHaveBeenCalledTimes(1);
|
||||
|
||||
bytes.resolve({ kind: 'none' });
|
||||
await vi.waitFor(async () => expect(await countRows(orm, TrekPhotoSignals)).toBe(1));
|
||||
expect((await main.signals.signalsFor([id], { waitMs: 0 })).pending).toBe(0);
|
||||
expect(resolver.readForSignals).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PhotoSignalsService, the line', () => {
|
||||
it('PHSVC-005: fetches two photos at a time and keeps no more than the line holds', async () => {
|
||||
const bytes = deferred<PhotoSignalSource>();
|
||||
resolver.readForSignals.mockReturnValue(bytes.promise);
|
||||
const ids = Array.from({ length: MAX_QUEUED_PHOTOS + 5 }, (_, i) => 900_000 + i);
|
||||
|
||||
const { pending } = await main.signals.signalsFor(ids, { waitMs: 0 });
|
||||
expect(pending).toBe(ids.length);
|
||||
expect(resolver.readForSignals).toHaveBeenCalledTimes(PARALLEL_FETCHES);
|
||||
|
||||
// Nothing answers: the line drains without writing, the photos past its end were never in it.
|
||||
bytes.resolve({ kind: 'unavailable' });
|
||||
await vi.waitFor(() => expect(resolver.readForSignals).toHaveBeenCalledTimes(MAX_QUEUED_PHOTOS));
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(0);
|
||||
});
|
||||
|
||||
it('PHSVC-006: a photo that times out twice is measured as nothing, once is tried again; busy or unavailable is left for later', async () => {
|
||||
const runner = { measure: vi.fn<PhotoSignalsRunner['measure']>(), close: vi.fn(() => Promise.resolve()) };
|
||||
const own = await boot(runner);
|
||||
try {
|
||||
const { ids } = await gallery(['image', 'image', 'image']);
|
||||
const [slow, busy, unavailable] = ids as [number, number, number];
|
||||
const reasons = new Map([
|
||||
[slow, 'timeout'],
|
||||
[busy, 'busy'],
|
||||
[unavailable, 'unavailable'],
|
||||
]);
|
||||
// Each photo's bytes carry its place in `ids`, so the runner knows which one it holds.
|
||||
resolver.readForSignals.mockImplementation((photoId) =>
|
||||
Promise.resolve<PhotoSignalSource>({ kind: 'original', bytes: Buffer.from([ids.indexOf(photoId)]) }),
|
||||
);
|
||||
runner.measure.mockImplementation((bytes) =>
|
||||
Promise.reject(new Error(reasons.get(ids[bytes[0] ?? 0] ?? 0), { cause: new Error('no thread script') })),
|
||||
);
|
||||
|
||||
const first = await own.signals.signalsFor(ids);
|
||||
expect(first.pending).toBe(3);
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(0);
|
||||
const second = await own.signals.signalsFor(ids);
|
||||
expect(second.byPhoto.get(slow)).toMatchObject({ sharpness: null, origWidth: null });
|
||||
expect([second.byPhoto.has(busy), second.byPhoto.has(unavailable)]).toEqual([false, false]);
|
||||
expect(second.pending).toBe(2);
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(1);
|
||||
expect(runner.measure).toHaveBeenCalledTimes(6);
|
||||
} finally {
|
||||
await own.t.close();
|
||||
}
|
||||
expect(runner.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('PHSVC-008: a photo whose bytes cannot be read stays pending, and the line goes on', async () => {
|
||||
const { ids } = await gallery(['image', 'image']);
|
||||
const [broken, fine] = ids as [number, number];
|
||||
resolver.readForSignals.mockImplementation((photoId) =>
|
||||
photoId === broken
|
||||
? Promise.reject(new Error('disk gone'))
|
||||
: Promise.resolve<PhotoSignalSource>({ kind: 'none' }),
|
||||
);
|
||||
|
||||
const { byPhoto, pending } = await main.signals.signalsFor([broken, fine]);
|
||||
expect(pending).toBe(1);
|
||||
expect(byPhoto.has(fine)).toBe(true);
|
||||
expect(await countRows(orm, TrekPhotoSignals)).toBe(1);
|
||||
});
|
||||
|
||||
it('PHSVC-007: once the module is shutting down a photo is no longer put in line', async () => {
|
||||
const own = await boot();
|
||||
try {
|
||||
const { ids } = await gallery(['image']);
|
||||
await own.signals.onModuleDestroy();
|
||||
const { pending } = await own.signals.signalsFor(ids, { waitMs: 0 });
|
||||
expect(pending).toBe(1);
|
||||
expect(resolver.readForSignals).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await own.t.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('PHSVC-009: photos asked for later answer what is stored and queue behind, neither waited for nor counted', async () => {
|
||||
const { ids } = await gallery(['image', 'image', 'image']);
|
||||
const [asked, stored, behind] = ids as [number, number, number];
|
||||
await insertRow(orm, TrekPhotoSignals, { photo: stored, algo: PHOTO_SIGNALS_ALGO, sharpness: 42 });
|
||||
const slow = deferred<PhotoSignalSource>();
|
||||
resolver.readForSignals.mockImplementation((photoId) =>
|
||||
photoId === behind ? slow.promise : Promise.resolve<PhotoSignalSource>({ kind: 'none' }),
|
||||
);
|
||||
|
||||
const { byPhoto, pending } = await main.signals.signalsFor([asked], { later: [stored, behind, asked] });
|
||||
expect(pending).toBe(0);
|
||||
expect(byPhoto.get(stored)?.sharpness).toBe(42);
|
||||
expect(byPhoto.has(behind)).toBe(false);
|
||||
expect(resolver.readForSignals.mock.calls.map(([id]) => id)).toEqual([asked, behind]);
|
||||
|
||||
slow.resolve({ kind: 'none' });
|
||||
await vi.waitFor(async () => expect(await countRows(orm, TrekPhotoSignals)).toBe(3));
|
||||
});
|
||||
|
||||
it("PHSVC-010: a provider's sides that disagree with its upright thumbnail on the long side are turned to match", async () => {
|
||||
const { ids } = await gallery(['image', 'image', 'image', 'image']);
|
||||
const [turned, square, undecoded, unsized] = ids as [number, number, number, number];
|
||||
// An undecoded thumbnail (Immich's WebP) cannot say which side is long, so the provider's sides stay.
|
||||
const sources = new Map<number, PhotoSignalSource>([
|
||||
[turned, { kind: 'thumbnail', bytes: photo, width: 4000, height: 6000 }],
|
||||
[square, { kind: 'thumbnail', bytes: photo, width: 5000, height: 5000 }],
|
||||
[undecoded, { kind: 'thumbnail', bytes: Buffer.from('RIFF....WEBP'), width: 4000, height: 6000 }],
|
||||
[unsized, { kind: 'thumbnail', bytes: photo, width: null, height: null }],
|
||||
]);
|
||||
resolver.readForSignals.mockImplementation((photoId) =>
|
||||
Promise.resolve<PhotoSignalSource>(sources.get(photoId) ?? { kind: 'none' }),
|
||||
);
|
||||
|
||||
const { byPhoto } = await main.signals.signalsFor(ids);
|
||||
expect(byPhoto.get(turned)).toMatchObject({ origWidth: 6000, origHeight: 4000 });
|
||||
expect(byPhoto.get(square)).toMatchObject({ origWidth: 5000, origHeight: 5000 });
|
||||
expect(byPhoto.get(undecoded)).toMatchObject({ sharpness: null, origWidth: 4000, origHeight: 6000 });
|
||||
expect(byPhoto.get(unsized)).toMatchObject({ origWidth: null, origHeight: null });
|
||||
expect(byPhoto.get(unsized)?.sharpness).toEqual(expect.any(Number));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,370 @@
|
||||
import {
|
||||
bookPhotoSuggestionsSchema,
|
||||
hashDistance,
|
||||
PHOTO_PICK,
|
||||
photoMoment,
|
||||
photosInScope,
|
||||
pickBookPhotos,
|
||||
type PhotoPickCandidate,
|
||||
type PhotoPickEntry,
|
||||
type PhotoPickSignals,
|
||||
} from './photo-pick';
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
/**
|
||||
* Picking photos for a book without looking at them: days, places, bursts,
|
||||
* look-alikes by hash, the doubtful ones set aside, a cover's shape, the limit
|
||||
* shared out over the groups. Lisbon's Alfama and Belem are 6 km apart; two
|
||||
* points 0.001 degrees of latitude apart are 111 m apart.
|
||||
*/
|
||||
|
||||
const ALFAMA = { lat: 38.7118, lng: -9.13 };
|
||||
const BELEM = { lat: 38.6916, lng: -9.216 };
|
||||
|
||||
const signals = (over: Partial<PhotoPickSignals> = {}): PhotoPickSignals => ({
|
||||
phash: null,
|
||||
sharpness: null,
|
||||
clipped: null,
|
||||
origWidth: null,
|
||||
origHeight: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
const photo = (photoId: number, over: Partial<PhotoPickCandidate> = {}): PhotoPickCandidate => ({
|
||||
photoId,
|
||||
takenAt: null,
|
||||
lat: null,
|
||||
lng: null,
|
||||
width: null,
|
||||
height: null,
|
||||
caption: null,
|
||||
entryIds: [],
|
||||
inBook: false,
|
||||
signals: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
const entry = (id: number, over: Partial<PhotoPickEntry> = {}): PhotoPickEntry => ({
|
||||
id,
|
||||
date: '2026-07-03',
|
||||
title: null,
|
||||
location: null,
|
||||
lat: null,
|
||||
lng: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
/** A photo in the Alfama on 3 July at hh:mm:ss local time (no zone). */
|
||||
const at = (photoId: number, time: string, over: Partial<PhotoPickCandidate> = {}) =>
|
||||
photo(photoId, { takenAt: `2026-07-03T${time}`, ...ALFAMA, ...over });
|
||||
|
||||
const ids = (result: ReturnType<typeof pickBookPhotos>) => result.groups.map((g) => g.picks.map((p) => p.photoId));
|
||||
|
||||
describe('photoMoment', () => {
|
||||
it('PHOTOPICK-001: a time without a zone is the wall clock, a UTC time moves by the longitude, an offset is taken as written', () => {
|
||||
const wall = Date.UTC(2026, 6, 3, 23, 30) / 1000;
|
||||
expect(photoMoment('2026-07-03T23:30:00', 139.7)).toEqual({ day: '2026-07-03', at: wall });
|
||||
expect(photoMoment('2026-07-03T20:00:00.000Z', 139.7)?.day).toBe('2026-07-04');
|
||||
expect(photoMoment('2026-07-03T20:00:00Z', null)?.day).toBe('2026-07-03');
|
||||
expect(photoMoment('2026-07-03T01:00:00Z', -122.4)?.day).toBe('2026-07-02');
|
||||
expect(photoMoment('2026-07-03 23:30:00+02:00', 139.7)).toEqual({
|
||||
day: '2026-07-03',
|
||||
at: Date.UTC(2026, 6, 3, 21, 30) / 1000,
|
||||
});
|
||||
expect(photoMoment('2026-07-03', 10)).toEqual({ day: '2026-07-03', at: null });
|
||||
expect(photoMoment('yesterday', 10)).toBeNull();
|
||||
expect(photoMoment(null, 10)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('hashDistance', () => {
|
||||
it('PHOTOPICK-002: counts the differing bits of two 64-bit hex hashes and refuses anything else', () => {
|
||||
expect(hashDistance('0123456789abcdef', '0123456789abcdef')).toBe(0);
|
||||
expect(hashDistance('0000000000000000', '000000000000000f')).toBe(4);
|
||||
expect(hashDistance('ffffffffffffffff', '0000000000000000')).toBe(64);
|
||||
expect(hashDistance('0000', '0000000000000000')).toBeNull();
|
||||
expect(hashDistance('zzzzzzzzzzzzzzzz', '0000000000000000')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickBookPhotos', () => {
|
||||
it('PHOTOPICK-003: groups by the linked entry day first, then the capture day, undated last, and answers the shared shape', () => {
|
||||
const entries = [entry(7, { date: '2026-07-01', location: 'Porto' })];
|
||||
const late = '2026-07-05T10:00:00';
|
||||
const photos = [photo(1), photo(2, { takenAt: late }), photo(3, { takenAt: late, entryIds: [7] })];
|
||||
const result = pickBookPhotos(photos, entries, { signalsPending: 2 });
|
||||
expect(bookPhotoSuggestionsSchema.parse(result)).toEqual(result);
|
||||
expect(result.groups.map((g) => [g.day, g.entryId, g.place])).toEqual([
|
||||
['2026-07-01', 7, 'Porto'],
|
||||
['2026-07-05', null, null],
|
||||
['undated', null, null],
|
||||
]);
|
||||
expect(ids(result)).toEqual([[3], [2], [1]]);
|
||||
expect(result.signals_pending).toBe(2);
|
||||
expect(result.groups_more).toBe(0);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-004: splits a day into places 250 m apart, named after the nearest entry within 2 km, with its entry only on that day', () => {
|
||||
const entries = [
|
||||
entry(1, { location: 'Alfama', ...ALFAMA }),
|
||||
entry(2, { date: '2026-07-04', title: 'Belem tower', ...BELEM }),
|
||||
];
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
at(10, '09:00:00'),
|
||||
at(11, '09:10:00', { lat: ALFAMA.lat + 0.001 }),
|
||||
at(12, '15:00:00', BELEM),
|
||||
at(13, '16:00:00', { lat: ALFAMA.lat + 0.01 }),
|
||||
],
|
||||
entries,
|
||||
);
|
||||
expect(result.groups.map((g) => [g.place, g.entryId, g.picks.map((p) => p.photoId)])).toEqual([
|
||||
['Alfama', 1, [10, 11]],
|
||||
['Belem tower', null, [12]],
|
||||
['Alfama', 1, [13]],
|
||||
]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-005: a burst keeps its sharpest photo and names the others near duplicates of it', () => {
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
at(1, '10:00:00', { signals: signals({ sharpness: 200 }) }),
|
||||
at(2, '10:00:04', { signals: signals({ sharpness: 400 }) }),
|
||||
at(3, '10:00:09', { signals: signals({ sharpness: 300 }) }),
|
||||
at(4, '10:05:00', { signals: signals({ sharpness: 250 }) }),
|
||||
],
|
||||
[],
|
||||
);
|
||||
const [group] = result.groups;
|
||||
expect(group?.picks.map((p) => p.photoId)).toEqual([2, 4]);
|
||||
expect(group?.picks[0]?.why[0]).toBe('sharpest of 3 similar');
|
||||
expect(group?.dropped).toEqual([
|
||||
{ photoId: 3, reason: 'near_duplicate_of', of: 2 },
|
||||
{ photoId: 1, reason: 'near_duplicate_of', of: 2 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-006: photos far apart in time look alike when their hashes differ in at most 10 bits', () => {
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
at(1, '10:00:00', { signals: signals({ phash: '0000000000000000', sharpness: 100 }) }),
|
||||
at(2, '12:00:00', { signals: signals({ phash: '00000000000003ff', sharpness: 150 }) }),
|
||||
at(3, '14:00:00', { signals: signals({ phash: '0000000000000fff', sharpness: 120 }) }),
|
||||
],
|
||||
[],
|
||||
);
|
||||
// 1 and 2 differ in 10 bits, 2 and 3 in 2, so all three are one set even though 1 and 3 differ in 12.
|
||||
expect(ids(result)).toEqual([[2]]);
|
||||
expect(result.groups[0]?.dropped.map((d) => d.photoId).sort()).toEqual([1, 3]);
|
||||
const apart = pickBookPhotos(
|
||||
[
|
||||
at(1, '10:00:00', { signals: signals({ phash: '0000000000000000' }) }),
|
||||
at(2, '12:00:00', { signals: signals({ phash: '0000000000000fff' }) }),
|
||||
],
|
||||
[],
|
||||
);
|
||||
expect(ids(apart)).toEqual([[1, 2]]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-007: a possibly blurry photo is set aside and said so, unless it is all its group has', () => {
|
||||
const sharp = (id: number, time: string, sharpness: number) => at(id, time, { signals: signals({ sharpness }) });
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
sharp(1, '09:00:00', 300),
|
||||
sharp(2, '10:00:00', 280),
|
||||
sharp(3, '11:00:00', 20),
|
||||
photo(4, { takenAt: '2026-07-04T10:00:00', signals: signals({ sharpness: 15 }) }),
|
||||
],
|
||||
[],
|
||||
);
|
||||
expect(result.groups[0]?.picks.map((p) => p.photoId)).toEqual([1, 2]);
|
||||
expect(result.groups[0]?.dropped).toEqual([{ photoId: 3, reason: 'possibly_blurry' }]);
|
||||
expect(result.groups[1]?.picks).toEqual([{ photoId: 4, why: ['possibly blurry', 'not in the book yet'] }]);
|
||||
// Under 35 % of the typical photo but above the floor: not blurry.
|
||||
const soft = pickBookPhotos([sharp(1, '09:00:00', 900), sharp(2, '10:00:00', 1000), sharp(3, '11:00:00', 100)], []);
|
||||
expect(soft.groups[0]?.dropped).toEqual([]);
|
||||
// Fewer than three measured photos say nothing about what is typical.
|
||||
const two = pickBookPhotos([sharp(1, '09:00:00', 300), sharp(2, '10:00:00', 5)], []);
|
||||
expect(two.groups[0]?.dropped).toEqual([]);
|
||||
expect(PHOTO_PICK.minForMedian).toBe(3);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-008: an original too small for print is set aside, a cover asks for more pixels, an unknown size counts as fine', () => {
|
||||
const sized = (id: number, time: string, w: number | null) =>
|
||||
at(id, time, { signals: signals({ origWidth: w, origHeight: w === null ? null : Math.round(w * 0.75) }) });
|
||||
const photos = [
|
||||
sized(1, '09:00:00', 4000),
|
||||
sized(2, '10:00:00', 1000),
|
||||
sized(3, '11:00:00', 2000),
|
||||
sized(4, '12:00:00', null),
|
||||
];
|
||||
const page = pickBookPhotos(photos, []);
|
||||
expect(page.groups[0]?.dropped).toEqual([{ photoId: 2, reason: 'low_resolution' }]);
|
||||
const cover = pickBookPhotos(photos, [], { purpose: 'cover' });
|
||||
expect(cover.groups[0]?.dropped.map((d) => d.photoId).sort()).toEqual([2, 3]);
|
||||
expect(cover.groups[0]?.picks.map((p) => p.photoId)).toEqual([1, 4]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-009: a cover puts a wide or square photo before a sharper upright one, a page goes by sharpness', () => {
|
||||
const photos = [
|
||||
at(1, '09:00:00', { width: 600, height: 800, signals: signals({ sharpness: 900 }) }),
|
||||
at(2, '10:00:00', { width: 800, height: 600, signals: signals({ sharpness: 300 }) }),
|
||||
at(3, '11:00:00', { width: 800, height: 800, signals: signals({ sharpness: 200 }) }),
|
||||
];
|
||||
const cover = pickBookPhotos(photos, [], { purpose: 'cover', limit: 2 });
|
||||
expect(cover.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([
|
||||
[2, 'landscape'],
|
||||
[3, 'square'],
|
||||
]);
|
||||
expect(cover.groups[0]?.more).toBe(1);
|
||||
const page = pickBookPhotos(photos, [], { limit: 1 });
|
||||
expect(page.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([[1, 'portrait']]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-010: without signals a caption wins, then a photo the book does not have yet, best first', () => {
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
at(1, '09:00:00', { inBook: true }),
|
||||
at(2, '10:00:00'),
|
||||
at(3, '11:00:00', { caption: 'Tram 28', inBook: true }),
|
||||
],
|
||||
[],
|
||||
{ limit: 2 },
|
||||
);
|
||||
expect(result.groups[0]?.picks).toEqual([
|
||||
{ photoId: 3, why: ['in the book already', 'has a caption'] },
|
||||
{ photoId: 2, why: ['not in the book yet'] },
|
||||
]);
|
||||
expect(result.groups[0]?.more).toBe(1);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-011: the limit goes round the groups, so every group gets its best before any gets a second', () => {
|
||||
const day = (d: string, n: number) =>
|
||||
Array.from({ length: 3 }, (_, i) => photo(n + i, { takenAt: `2026-07-0${d}T1${i}:00:00` }));
|
||||
const photos = [...day('1', 10), ...day('2', 20), ...day('3', 30)];
|
||||
const four = pickBookPhotos(photos, [], { limit: 4 });
|
||||
expect(ids(four)).toEqual([[10, 11], [20], [30]]);
|
||||
expect(four.groups.map((g) => g.more)).toEqual([1, 2, 2]);
|
||||
const two = pickBookPhotos(photos, [], { limit: 2 });
|
||||
expect(ids(two)).toEqual([[10], [20]]);
|
||||
expect(two.groups_more).toBe(1);
|
||||
expect(pickBookPhotos(photos, [], { limit: 500 }).groups.flatMap((g) => g.picks)).toHaveLength(9);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-012: an entry narrows to its linked photos, or to its day when none is linked; a date narrows to that day', () => {
|
||||
const entries = [entry(1, { date: '2026-07-01' }), entry(2, { date: '2026-07-02' })];
|
||||
const photos = [
|
||||
photo(1, { entryIds: [1] }),
|
||||
photo(2, { takenAt: '2026-07-01T10:00:00' }),
|
||||
photo(3, { takenAt: '2026-07-02T10:00:00' }),
|
||||
];
|
||||
expect(ids(pickBookPhotos(photos, entries, { entryId: 1 }))).toEqual([[1]]);
|
||||
expect(ids(pickBookPhotos(photos, entries, { entryId: 2 }))).toEqual([[3]]);
|
||||
expect(ids(pickBookPhotos(photos, entries, { entryId: 99 }))).toEqual([]);
|
||||
// Photo 1 has no capture time of its own, so it comes after the one that does.
|
||||
expect(ids(pickBookPhotos(photos, entries, { date: '2026-07-01' }))).toEqual([[2, 1]]);
|
||||
// The same narrowing, for the server to measure those photos first.
|
||||
expect([...photosInScope(photos, entries, { entryId: 2 })]).toEqual([3]);
|
||||
expect([...photosInScope(photos, entries, { date: '2026-07-01' })]).toEqual([1, 2]);
|
||||
expect([...photosInScope(photos, entries)]).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-013: a photo without a place takes its entry place, and a placeless group takes the name its one entry has', () => {
|
||||
const entries = [entry(5, { location: 'Sintra', lat: 38.7976, lng: -9.3906 }), entry(6, { location: 'Cascais' })];
|
||||
const result = pickBookPhotos(
|
||||
[photo(1, { entryIds: [5] }), photo(2, { entryIds: [6] }), photo(3, { entryIds: [6] })],
|
||||
entries,
|
||||
);
|
||||
expect(result.groups.map((g) => [g.place, g.entryId, g.picks.map((p) => p.photoId)])).toEqual([
|
||||
['Sintra', 5, [1]],
|
||||
['Cascais', 6, [2, 3]],
|
||||
]);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-014: photos about as sharp compare by what comes next: clipping, then a caption, then not in the book', () => {
|
||||
const shot = (
|
||||
id: number,
|
||||
time: string,
|
||||
sharpness: number,
|
||||
clipped: number,
|
||||
over: Partial<PhotoPickCandidate> = {},
|
||||
) => at(id, time, { inBook: true, signals: signals({ sharpness, clipped }), ...over });
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
shot(1, '09:00:00', 310, 0),
|
||||
shot(2, '10:00:00', 300, 0.012, { caption: 'Tram 28' }),
|
||||
shot(3, '11:00:00', 305, 0.005, { inBook: false }),
|
||||
shot(4, '12:00:00', 600, 0.05),
|
||||
shot(5, '13:00:00', 305, 0.1, { caption: 'Miradouro', inBook: false }),
|
||||
],
|
||||
[],
|
||||
);
|
||||
// 4 is clearly the sharpest; 1, 2, 3 and 5 are within 15 % of each other, and 5 burns out a tenth of its pixels.
|
||||
expect(ids(result)).toEqual([[4, 2, 3, 1, 5]]);
|
||||
expect(PHOTO_PICK.bands).toEqual({ sharpness: 1.15, clipped: 0.02, pixels: 1.5 });
|
||||
});
|
||||
|
||||
it('PHOTOPICK-015: shots seconds apart whose hashes clearly differ stay apart; a burst within 20 bits is one set', () => {
|
||||
const hashed = (id: number, time: string, phash: string) => at(id, time, { signals: signals({ phash }) });
|
||||
const walk = pickBookPhotos(
|
||||
[
|
||||
hashed(1, '10:00:00', '0000000000000000'),
|
||||
hashed(2, '10:00:08', 'ffffffff00000000'),
|
||||
hashed(3, '10:00:16', 'ffffffffffffffff'),
|
||||
],
|
||||
[],
|
||||
);
|
||||
expect(ids(walk)).toEqual([[1, 2, 3]]);
|
||||
expect(walk.groups[0]?.dropped).toEqual([]);
|
||||
const burst = pickBookPhotos(
|
||||
[hashed(1, '10:00:00', '0000000000000000'), hashed(2, '10:00:08', '00000000000fffff')],
|
||||
[],
|
||||
);
|
||||
expect(ids(burst)).toEqual([[1]]);
|
||||
expect(burst.groups[0]?.picks[0]?.why[0]).toBe('best of 2 similar');
|
||||
expect(burst.groups[0]?.dropped).toEqual([{ photoId: 2, reason: 'near_duplicate_of', of: 1 }]);
|
||||
// The same 20 bits an hour apart are no longer alike.
|
||||
const later = pickBookPhotos(
|
||||
[hashed(1, '10:00:00', '0000000000000000'), hashed(2, '11:00:00', '00000000000fffff')],
|
||||
[],
|
||||
);
|
||||
expect(ids(later)).toEqual([[1, 2]]);
|
||||
expect(PHOTO_PICK.burstHashDistance).toBe(20);
|
||||
});
|
||||
|
||||
it('PHOTOPICK-016: dropped names at most 30 per group and 120 in all, and dropped_more counts the rest', () => {
|
||||
const burst = (day: number, n: number) =>
|
||||
Array.from({ length: n }, (_, i) =>
|
||||
photo(day * 100 + i + 1, { takenAt: `2026-07-0${day}T10:00:${String(i).padStart(2, '0')}` }),
|
||||
);
|
||||
const one = pickBookPhotos(burst(1, 36), []);
|
||||
expect(ids(one)).toEqual([[101]]);
|
||||
expect(one.groups[0]?.dropped).toHaveLength(30);
|
||||
expect(one.groups[0]?.dropped_more).toBe(5);
|
||||
const five = pickBookPhotos([1, 2, 3, 4, 5].flatMap((day) => burst(day, 31)), []);
|
||||
expect(bookPhotoSuggestionsSchema.parse(five)).toEqual(five);
|
||||
expect(five.groups.map((g) => [g.dropped.length, g.dropped_more])).toEqual([
|
||||
[30, 0],
|
||||
[30, 0],
|
||||
[30, 0],
|
||||
[30, 0],
|
||||
[0, 30],
|
||||
]);
|
||||
});
|
||||
|
||||
it("PHOTOPICK-017: a photo's shape comes from its row before the original's size, which a provider may report turned", () => {
|
||||
const result = pickBookPhotos(
|
||||
[
|
||||
at(1, '09:00:00', { width: 600, height: 800, signals: signals({ origWidth: 4000, origHeight: 3000 }) }),
|
||||
at(2, '10:00:00', { signals: signals({ origWidth: 3000, origHeight: 2000 }) }),
|
||||
],
|
||||
[],
|
||||
{ purpose: 'cover' },
|
||||
);
|
||||
expect(result.groups[0]?.picks.map((p) => [p.photoId, p.why[0]])).toEqual([
|
||||
[2, 'landscape'],
|
||||
[1, 'portrait'],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,579 @@
|
||||
import { haversineMetres } from '../geo/haversine';
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Photos for a book page, picked without a model looking at them: what the
|
||||
* rows know (day, place, capture time, size, caption, whether the book has the
|
||||
* photo already) and two cheap pixel signals measured once per photo on the
|
||||
* server (a perceptual hash and how sharp it is, photo-signals.service.ts).
|
||||
*
|
||||
* Pure: the server's `suggest_book_photos` tool calls it with the journey's
|
||||
* gallery, and a Studio button can call it later with the same data.
|
||||
*
|
||||
* What it cannot do, and the tool says so: judge beauty, find faces or closed
|
||||
* eyes, tell who is in a photo or what it shows, or know two photos show the
|
||||
* same scene from another angle (the hash only catches near-identical framing).
|
||||
*/
|
||||
|
||||
export const PHOTO_PICK = {
|
||||
/** Photos within this many metres of a place's centre belong to it. */
|
||||
placeRadiusM: 250,
|
||||
/** A place is named after the nearest entry location within this many metres. */
|
||||
labelRadiusM: 2000,
|
||||
/** Two photos taken this close together are one burst... */
|
||||
burstSeconds: 10,
|
||||
/** ...unless both hashes are measured and differ in more than this many bits (a walk shot every few seconds). */
|
||||
burstHashDistance: 20,
|
||||
/** Two photos whose hashes differ in at most this many of 64 bits look the same. */
|
||||
hashDistance: 10,
|
||||
/** Possibly blurry: sharper than less than this share of the journey's typical photo... */
|
||||
blurRatio: 0.35,
|
||||
/** ...and under this absolute floor (the variance of the grey Laplacian at a 240 px long edge). */
|
||||
blurFloor: 60,
|
||||
/**
|
||||
* Ranking compares in bands, so a later criterion still counts between two
|
||||
* photos that are about as good: sharpness within this factor of each other,
|
||||
* clipping within this share of the pixels, size within this factor.
|
||||
*/
|
||||
bands: { sharpness: 1.15, clipped: 0.02, pixels: 1.5 },
|
||||
/** The typical photo needs this many measured photos before anything counts as blurry. */
|
||||
minForMedian: 3,
|
||||
/** The long edge in pixels a photo needs for print: a part of a page, or a whole cover. */
|
||||
minLongEdge: { page: 1200, cover: 2400 },
|
||||
defaultLimit: 24,
|
||||
maxLimit: 60,
|
||||
/** What one group lists as left out, at most... */
|
||||
droppedPerGroup: 30,
|
||||
/** ...and all groups together; the rest is counted in each group's `dropped_more`. */
|
||||
droppedTotal: 120,
|
||||
/** Reasons one pick carries, at most. */
|
||||
whyPerPick: 6,
|
||||
} as const;
|
||||
|
||||
/** What the server measured on a photo's pixels; a null is a signal that could not be measured. */
|
||||
export interface PhotoPickSignals {
|
||||
phash: string | null;
|
||||
sharpness: number | null;
|
||||
clipped: number | null;
|
||||
/** The original's size in pixels, never a thumbnail's. */
|
||||
origWidth: number | null;
|
||||
origHeight: number | null;
|
||||
}
|
||||
|
||||
/** A photo of the journey's gallery as the picker reads it. */
|
||||
export interface PhotoPickCandidate {
|
||||
photoId: number;
|
||||
/** As the row has it: ISO with or without a zone, or a bare date. */
|
||||
takenAt: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
/** Any size with the photo's shape (a thumbnail's will do); null when unknown. */
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
caption: string | null;
|
||||
/** The journey entries the photo is linked to. */
|
||||
entryIds: readonly number[];
|
||||
inBook: boolean;
|
||||
signals: PhotoPickSignals | null;
|
||||
}
|
||||
|
||||
/** A journey entry: its day and where it was. */
|
||||
export interface PhotoPickEntry {
|
||||
id: number;
|
||||
date: string;
|
||||
title: string | null;
|
||||
location: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
}
|
||||
|
||||
export interface PhotoPickOptions {
|
||||
/** Only the photos linked to this entry (or, when it has none, taken on its day). */
|
||||
entryId?: number;
|
||||
/** Only the photos of this day, `YYYY-MM-DD`. */
|
||||
date?: string;
|
||||
/** A cover wants a wide or square photo and more pixels. */
|
||||
purpose?: 'page' | 'cover';
|
||||
limit?: number;
|
||||
/** Photos whose signals were still being measured; passed through to the answer. */
|
||||
signalsPending?: number;
|
||||
}
|
||||
|
||||
const photoIdSchema = z.number().int().positive();
|
||||
|
||||
export const photoPickDroppedSchema = z.discriminatedUnion('reason', [
|
||||
z.object({ photoId: photoIdSchema, reason: z.literal('near_duplicate_of'), of: photoIdSchema }),
|
||||
z.object({ photoId: photoIdSchema, reason: z.literal('possibly_blurry') }),
|
||||
z.object({ photoId: photoIdSchema, reason: z.literal('low_resolution') }),
|
||||
]);
|
||||
export type PhotoPickDropped = z.infer<typeof photoPickDroppedSchema>;
|
||||
|
||||
/** What `suggest_book_photos` answers: groups by day and place, each with its picks and what it left out. */
|
||||
export const bookPhotoSuggestionsSchema = z.object({
|
||||
groups: z.array(
|
||||
z.object({
|
||||
/** `YYYY-MM-DD`, or `undated` for photos with neither an entry nor a capture time. */
|
||||
day: z.string(),
|
||||
place: z.string().nullable(),
|
||||
entryId: z.number().int().nullable(),
|
||||
picks: z.array(z.object({ photoId: photoIdSchema, why: z.array(z.string()).max(PHOTO_PICK.whyPerPick) })),
|
||||
dropped: z.array(photoPickDroppedSchema).max(PHOTO_PICK.droppedPerGroup),
|
||||
/** Photos left out of this group that `dropped` has no room to name. */
|
||||
dropped_more: z.number().int().nonnegative(),
|
||||
/** Good photos of this group the limit left out. */
|
||||
more: z.number().int().nonnegative(),
|
||||
}),
|
||||
),
|
||||
/** Groups left out whole because the limit was reached. */
|
||||
groups_more: z.number().int().nonnegative(),
|
||||
/** Photos judged on their rows alone, their pixels not measured yet. */
|
||||
signals_pending: z.number().int().nonnegative(),
|
||||
});
|
||||
export type BookPhotoSuggestions = z.infer<typeof bookPhotoSuggestionsSchema>;
|
||||
|
||||
// ── Time ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const STAMP = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?\s*(Z|[+-]\d{2}:?\d{2})?$/i;
|
||||
const BARE_DATE = /^(\d{4}-\d{2}-\d{2})$/;
|
||||
|
||||
interface Moment {
|
||||
/** The local day the photo was taken on. */
|
||||
day: string;
|
||||
/** Seconds, comparable between photos; null for a bare date. */
|
||||
at: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* When a photo was taken. A time without a zone is the camera's wall clock and
|
||||
* its date is the day. A time in UTC is moved by the photo's longitude (an
|
||||
* hour per 15 degrees, the sun's time) when it has one, since the day a
|
||||
* traveller lived it is the local one; an explicit offset is taken as written.
|
||||
*/
|
||||
export function photoMoment(takenAt: string | null, lng: number | null): Moment | null {
|
||||
if (!takenAt) return null;
|
||||
const bare = BARE_DATE.exec(takenAt.trim());
|
||||
if (bare?.[1]) return { day: bare[1], at: null };
|
||||
const m = STAMP.exec(takenAt.trim());
|
||||
if (!m) return null;
|
||||
const [, y, mo, d, h, mi, s, zone] = m;
|
||||
const wall = Date.UTC(Number(y), Number(mo) - 1, Number(d), Number(h), Number(mi), Number(s ?? 0));
|
||||
if (Number.isNaN(wall)) return null;
|
||||
const offsetMin = zone ? zoneMinutes(zone) : 0;
|
||||
const at = (wall - offsetMin * 60_000) / 1000;
|
||||
if (zone === undefined || offsetMin !== 0) return { day: takenAt.trim().slice(0, 10), at };
|
||||
const shift = lng !== null ? Math.round(lng / 15) * 3600 : 0;
|
||||
return { day: new Date((at + shift) * 1000).toISOString().slice(0, 10), at };
|
||||
}
|
||||
|
||||
function zoneMinutes(zone: string): number {
|
||||
if (zone.toUpperCase() === 'Z') return 0;
|
||||
const digits = zone.replace(':', '');
|
||||
const sign = digits.startsWith('-') ? -1 : 1;
|
||||
return sign * (Number(digits.slice(1, 3)) * 60 + Number(digits.slice(3, 5)));
|
||||
}
|
||||
|
||||
// ── Likeness ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** A 64-bit hash as its two 32-bit halves, read once per photo rather than once per pair. */
|
||||
type Hash = readonly [number, number];
|
||||
|
||||
const HASH = /^[0-9a-f]{16}$/i;
|
||||
|
||||
function parseHash(hex: string | null | undefined): Hash | null {
|
||||
if (!hex || !HASH.test(hex)) return null;
|
||||
return [parseInt(hex.slice(0, 8), 16), parseInt(hex.slice(8), 16)];
|
||||
}
|
||||
|
||||
/** The set bits of a 32-bit number. */
|
||||
function bitCount(x: number): number {
|
||||
let v = x - ((x >>> 1) & 0x55555555);
|
||||
v = (v & 0x33333333) + ((v >>> 2) & 0x33333333);
|
||||
return Math.imul((v + (v >>> 4)) & 0x0f0f0f0f, 0x01010101) >>> 24;
|
||||
}
|
||||
|
||||
function bitsApart(a: Hash, b: Hash): number {
|
||||
return bitCount(a[0] ^ b[0]) + bitCount(a[1] ^ b[1]);
|
||||
}
|
||||
|
||||
/** How many of the 64 bits two hashes (16 hex digits each) differ in; null when either is not one. */
|
||||
export function hashDistance(a: string, b: string): number | null {
|
||||
const ha = parseHash(a);
|
||||
const hb = parseHash(b);
|
||||
return ha && hb ? bitsApart(ha, hb) : null;
|
||||
}
|
||||
|
||||
// ── Picking ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** A candidate with what the picker worked out about it. */
|
||||
interface Placed {
|
||||
photo: PhotoPickCandidate;
|
||||
day: string;
|
||||
at: number | null;
|
||||
pos: { lat: number; lng: number } | null;
|
||||
}
|
||||
|
||||
interface Group {
|
||||
day: string;
|
||||
members: Placed[];
|
||||
centre: { lat: number; lng: number } | null;
|
||||
}
|
||||
|
||||
interface Ranked {
|
||||
photoId: number;
|
||||
at: number | null;
|
||||
why: string[];
|
||||
}
|
||||
|
||||
interface Judged {
|
||||
picks: Ranked[];
|
||||
dropped: PhotoPickDropped[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The photos `pickBookPhotos` judges under these options: those of the entry or
|
||||
* the day asked for, or all of them. The server measures these first.
|
||||
*/
|
||||
export function photosInScope(
|
||||
candidates: readonly PhotoPickCandidate[],
|
||||
entries: readonly PhotoPickEntry[],
|
||||
options: Pick<PhotoPickOptions, 'entryId' | 'date'> = {},
|
||||
): Set<number> {
|
||||
const byEntry = new Map(entries.map((e) => [e.id, e]));
|
||||
const placed = candidates.map((photo) => place(photo, byEntry));
|
||||
return new Set(filterFor(placed, options, byEntry).map((p) => p.photo.photoId));
|
||||
}
|
||||
|
||||
/** Photos for a page or the cover, grouped by day and place, best first within each group. */
|
||||
export function pickBookPhotos(
|
||||
candidates: readonly PhotoPickCandidate[],
|
||||
entries: readonly PhotoPickEntry[],
|
||||
options: PhotoPickOptions = {},
|
||||
): BookPhotoSuggestions {
|
||||
const purpose = options.purpose ?? 'page';
|
||||
const limit = Math.min(PHOTO_PICK.maxLimit, Math.max(1, options.limit ?? PHOTO_PICK.defaultLimit));
|
||||
const byEntry = new Map(entries.map((e) => [e.id, e]));
|
||||
const median = medianSharpness(candidates);
|
||||
const placed = candidates.map((photo) => place(photo, byEntry));
|
||||
const chosen = filterFor(placed, options, byEntry);
|
||||
const groups = groupByDayAndPlace(chosen);
|
||||
const judged = groups.map((g) => judge(g, purpose, median));
|
||||
const shown = takeRoundRobin(judged.map((j) => j.picks), limit);
|
||||
const out: BookPhotoSuggestions['groups'] = [];
|
||||
let groupsMore = 0;
|
||||
let droppedRoom: number = PHOTO_PICK.droppedTotal;
|
||||
groups.forEach((group, i) => {
|
||||
const picks = shown[i] ?? [];
|
||||
const all = judged[i];
|
||||
if (!all || picks.length === 0) {
|
||||
if (all && all.picks.length > 0) groupsMore++;
|
||||
return;
|
||||
}
|
||||
const label = labelOf(group, entries);
|
||||
const dropped = all.dropped.slice(0, Math.min(PHOTO_PICK.droppedPerGroup, droppedRoom));
|
||||
droppedRoom -= dropped.length;
|
||||
out.push({
|
||||
day: group.day,
|
||||
place: label.place,
|
||||
entryId: label.entryId,
|
||||
picks: picks.map((p) => ({ photoId: p.photoId, why: p.why.slice(0, PHOTO_PICK.whyPerPick) })),
|
||||
dropped,
|
||||
dropped_more: all.dropped.length - dropped.length,
|
||||
more: all.picks.length - picks.length,
|
||||
});
|
||||
});
|
||||
return { groups: out, groups_more: groupsMore, signals_pending: options.signalsPending ?? 0 };
|
||||
}
|
||||
|
||||
/** The sharpness of the journey's typical photo, over every measured one; null with too few to say. */
|
||||
function medianSharpness(candidates: readonly PhotoPickCandidate[]): number | null {
|
||||
const values = candidates
|
||||
.map((c) => c.signals?.sharpness)
|
||||
.filter((v): v is number => typeof v === 'number' && Number.isFinite(v))
|
||||
.sort((a, b) => a - b);
|
||||
if (values.length < PHOTO_PICK.minForMedian) return null;
|
||||
const mid = Math.floor(values.length / 2);
|
||||
const upper = values[mid] ?? 0;
|
||||
return values.length % 2 === 1 ? upper : ((values[mid - 1] ?? upper) + upper) / 2;
|
||||
}
|
||||
|
||||
/** The photo's day (its entry's first, else when it was taken) and where it was (its own place, else its entry's). */
|
||||
function place(photo: PhotoPickCandidate, byEntry: ReadonlyMap<number, PhotoPickEntry>): Placed {
|
||||
const entry = photo.entryIds.map((id) => byEntry.get(id)).find((e) => e !== undefined);
|
||||
const moment = photoMoment(photo.takenAt, photo.lng);
|
||||
const day = entry?.date ?? moment?.day ?? 'undated';
|
||||
const own = photo.lat !== null && photo.lng !== null ? { lat: photo.lat, lng: photo.lng } : null;
|
||||
const fromEntry = entry && entry.lat !== null && entry.lng !== null ? { lat: entry.lat, lng: entry.lng } : null;
|
||||
return { photo, day, at: moment?.at ?? null, pos: own ?? fromEntry };
|
||||
}
|
||||
|
||||
function filterFor(
|
||||
placed: readonly Placed[],
|
||||
options: PhotoPickOptions,
|
||||
byEntry: ReadonlyMap<number, PhotoPickEntry>,
|
||||
): Placed[] {
|
||||
let out = [...placed];
|
||||
if (options.entryId !== undefined) {
|
||||
const entryId = options.entryId;
|
||||
const linked = out.filter((p) => p.photo.entryIds.includes(entryId));
|
||||
const entryDay = byEntry.get(entryId)?.date;
|
||||
out = linked.length > 0 || !entryDay ? linked : out.filter((p) => p.day === entryDay);
|
||||
}
|
||||
if (options.date !== undefined) out = out.filter((p) => p.day === options.date);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Days in order (undated last); within a day, greedy places of `placeRadiusM`, the photos without one together. */
|
||||
function groupByDayAndPlace(placed: readonly Placed[]): Group[] {
|
||||
const days = new Map<string, Placed[]>();
|
||||
for (const p of [...placed].sort(byTime)) {
|
||||
const list = days.get(p.day);
|
||||
if (list) list.push(p);
|
||||
else days.set(p.day, [p]);
|
||||
}
|
||||
const out: Group[] = [];
|
||||
for (const day of [...days.keys()].sort(byDay)) {
|
||||
const located: Group[] = [];
|
||||
const nowhere: Group = { day, members: [], centre: null };
|
||||
for (const p of days.get(day) ?? []) {
|
||||
if (!p.pos) {
|
||||
nowhere.members.push(p);
|
||||
continue;
|
||||
}
|
||||
const pos = p.pos;
|
||||
const near = located.find(
|
||||
(g) => g.centre && haversineMetres(g.centre.lat, g.centre.lng, pos.lat, pos.lng) <= PHOTO_PICK.placeRadiusM,
|
||||
);
|
||||
if (!near) {
|
||||
located.push({ day, members: [p], centre: { ...pos } });
|
||||
continue;
|
||||
}
|
||||
near.members.push(p);
|
||||
const n = near.members.length;
|
||||
const c = near.centre ?? pos;
|
||||
near.centre = { lat: c.lat + (pos.lat - c.lat) / n, lng: c.lng + (pos.lng - c.lng) / n };
|
||||
}
|
||||
out.push(...located);
|
||||
if (nowhere.members.length > 0) out.push(nowhere);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Near duplicates folded onto their best, then the doubtful ones set aside unless nothing else is left. */
|
||||
function judge(group: Group, purpose: 'page' | 'cover', median: number | null): Judged {
|
||||
const sets = similarSets(group.members);
|
||||
const dropped: PhotoPickDropped[] = [];
|
||||
const kept: { p: Placed; why: string[] }[] = [];
|
||||
for (const set of sets) {
|
||||
const ranked = [...set].sort((a, b) => compare(a, b, purpose, median));
|
||||
const best = ranked[0];
|
||||
if (!best) continue;
|
||||
const why: string[] = [];
|
||||
if (set.length > 1) {
|
||||
const measured = set.filter((p) => typeof p.photo.signals?.sharpness === 'number').length;
|
||||
why.push(measured >= 2 ? `sharpest of ${set.length} similar` : `best of ${set.length} similar`);
|
||||
for (const other of ranked.slice(1)) {
|
||||
dropped.push({ photoId: other.photo.photoId, reason: 'near_duplicate_of', of: best.photo.photoId });
|
||||
}
|
||||
}
|
||||
kept.push({ p: best, why });
|
||||
}
|
||||
const minEdge = PHOTO_PICK.minLongEdge[purpose];
|
||||
const doubts = kept.map(({ p }) => doubtOf(p.photo, median, minEdge));
|
||||
const sound = kept.filter((_, i) => doubts[i] === null);
|
||||
const keepDoubtful = sound.length === 0;
|
||||
const picks: Ranked[] = [];
|
||||
kept.forEach(({ p, why }, i) => {
|
||||
const doubt = doubts[i] ?? null;
|
||||
if (doubt && !keepDoubtful) {
|
||||
dropped.push({ photoId: p.photo.photoId, reason: doubt });
|
||||
return;
|
||||
}
|
||||
if (doubt === 'possibly_blurry') why.push('possibly blurry');
|
||||
if (doubt === 'low_resolution') why.push('low resolution for print');
|
||||
picks.push({ photoId: p.photo.photoId, at: p.at, why: [...why, ...plainWhy(p.photo)] });
|
||||
});
|
||||
const rank = new Map(
|
||||
kept
|
||||
.map(({ p }) => p)
|
||||
.sort((a, b) => compare(a, b, purpose, median))
|
||||
.map((p, i) => [p.photo.photoId, i]),
|
||||
);
|
||||
picks.sort((a, b) => (rank.get(a.photoId) ?? 0) - (rank.get(b.photoId) ?? 0));
|
||||
return { picks, dropped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Photos that look alike, joined transitively: a burst (taken within
|
||||
* `burstSeconds`) whose hashes, where both are measured, are within
|
||||
* `burstHashDistance`, or near-identical hashes at any time. `members` come in
|
||||
* time order (`groupByDayAndPlace`), so once a photo without a hash is past
|
||||
* its burst window no later one can match it. Every hash is read once, and a
|
||||
* pair costs two XORs, so a day of thousands of photos stays quick.
|
||||
*/
|
||||
function similarSets(members: readonly Placed[]): Placed[][] {
|
||||
const parent = members.map((_, i) => i);
|
||||
const root = (i: number): number => {
|
||||
let r = i;
|
||||
while (parent[r] !== r) r = parent[r] ?? r;
|
||||
// Every index on the way now points straight at the root.
|
||||
for (let k = i; parent[k] !== r; ) {
|
||||
const next = parent[k] ?? r;
|
||||
parent[k] = r;
|
||||
k = next;
|
||||
}
|
||||
return r;
|
||||
};
|
||||
const hashes = members.map((p) => parseHash(p.photo.signals?.phash));
|
||||
members.forEach((a, i) => {
|
||||
const ha = hashes[i] ?? null;
|
||||
for (let j = i + 1; j < members.length; j++) {
|
||||
const b = members[j];
|
||||
if (!b) continue;
|
||||
const burst = a.at !== null && b.at !== null && b.at - a.at <= PHOTO_PICK.burstSeconds;
|
||||
if (!burst && ha === null) break;
|
||||
if (alike(burst, ha, hashes[j] ?? null)) parent[root(j)] = root(i);
|
||||
}
|
||||
});
|
||||
const sets = new Map<number, Placed[]>();
|
||||
members.forEach((p, i) => sets.set(root(i), [...(sets.get(root(i)) ?? []), p]));
|
||||
return [...sets.values()];
|
||||
}
|
||||
|
||||
function alike(burst: boolean, ha: Hash | null, hb: Hash | null): boolean {
|
||||
const distance = ha && hb ? bitsApart(ha, hb) : null;
|
||||
if (burst) return distance === null || distance <= PHOTO_PICK.burstHashDistance;
|
||||
return distance !== null && distance <= PHOTO_PICK.hashDistance;
|
||||
}
|
||||
|
||||
/** Which band of `PHOTO_PICK.bands` a ratio falls in, centred on 1; zero or nothing is the lowest band. */
|
||||
function band(ratio: number, step: number): number {
|
||||
return ratio > 0 ? Math.round(Math.log(ratio) / Math.log(step)) : -Infinity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best first: for a cover a wide or square photo, then the sharper relative to
|
||||
* the journey, then fewer burnt-out or black pixels, then more pixels, then a
|
||||
* caption, then one the book does not have yet, each compared in bands so the
|
||||
* next one decides between two photos about as good. An unknown sharpness
|
||||
* counts as typical, an unknown clipping as none, an unknown size as the
|
||||
* smallest.
|
||||
*/
|
||||
function compare(a: Placed, b: Placed, purpose: 'page' | 'cover', median: number | null): number {
|
||||
const { bands } = PHOTO_PICK;
|
||||
const keys: Array<(p: Placed) => number> = [
|
||||
(p) => (purpose === 'cover' && shapeOf(p.photo) === 'portrait' ? 1 : 0),
|
||||
(p) => {
|
||||
const s = p.photo.signals?.sharpness;
|
||||
return -band(typeof s === 'number' && median ? s / median : 1, bands.sharpness);
|
||||
},
|
||||
(p) => Math.floor((p.photo.signals?.clipped ?? 0) / bands.clipped),
|
||||
(p) => -band((p.photo.signals?.origWidth ?? 0) * (p.photo.signals?.origHeight ?? 0), bands.pixels),
|
||||
(p) => (p.photo.caption?.trim() ? 0 : 1),
|
||||
(p) => (p.photo.inBook ? 1 : 0),
|
||||
];
|
||||
for (const key of keys) {
|
||||
const ka = key(a);
|
||||
const kb = key(b);
|
||||
if (ka !== kb) return ka - kb;
|
||||
}
|
||||
return byTime(a, b);
|
||||
}
|
||||
|
||||
function doubtOf(
|
||||
photo: PhotoPickCandidate,
|
||||
median: number | null,
|
||||
minEdge: number,
|
||||
): 'possibly_blurry' | 'low_resolution' | null {
|
||||
const s = photo.signals;
|
||||
if (s && typeof s.sharpness === 'number' && median !== null) {
|
||||
if (s.sharpness < median * PHOTO_PICK.blurRatio && s.sharpness < PHOTO_PICK.blurFloor) return 'possibly_blurry';
|
||||
}
|
||||
if (s?.origWidth && s.origHeight && Math.max(s.origWidth, s.origHeight) < minEdge) return 'low_resolution';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The photo's shape: from its row's size (drawn upright), else from the original's size the server measured. */
|
||||
function shapeOf(photo: PhotoPickCandidate): 'landscape' | 'portrait' | 'square' | null {
|
||||
const own = photo.width && photo.height;
|
||||
const w = own ? photo.width : photo.signals?.origWidth;
|
||||
const h = own ? photo.height : photo.signals?.origHeight;
|
||||
if (!w || !h) return null;
|
||||
const ratio = w / h;
|
||||
if (ratio >= 1.1) return 'landscape';
|
||||
return ratio <= 0.9 ? 'portrait' : 'square';
|
||||
}
|
||||
|
||||
function plainWhy(photo: PhotoPickCandidate): string[] {
|
||||
const why: string[] = [];
|
||||
const shape = shapeOf(photo);
|
||||
if (shape) why.push(shape);
|
||||
why.push(photo.inBook ? 'in the book already' : 'not in the book yet');
|
||||
if (photo.caption?.trim()) why.push('has a caption');
|
||||
return why;
|
||||
}
|
||||
|
||||
/** Up to `limit` picks, one from each group in turn, so every day and place gets its best before any gets a second. */
|
||||
function takeRoundRobin(lists: readonly Ranked[][], limit: number): Ranked[][] {
|
||||
const out = lists.map((): Ranked[] => []);
|
||||
let taken = 0;
|
||||
for (let round = 0; taken < limit; round++) {
|
||||
let tookOne = false;
|
||||
for (let i = 0; i < lists.length && taken < limit; i++) {
|
||||
const next = lists[i]?.[round];
|
||||
if (!next) continue;
|
||||
out[i]?.push(next);
|
||||
taken++;
|
||||
tookOne = true;
|
||||
}
|
||||
if (!tookOne) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A group's place and entry: the entry most of its photos are linked to, else
|
||||
* the nearest entry location within `labelRadiusM` (its entry only when it is
|
||||
* of the same day). A group without a place is named after its one entry, if
|
||||
* all its photos share one.
|
||||
*/
|
||||
function labelOf(group: Group, entries: readonly PhotoPickEntry[]): { place: string | null; entryId: number | null } {
|
||||
const counts = new Map<number, number>();
|
||||
for (const m of group.members) for (const id of m.photo.entryIds) counts.set(id, (counts.get(id) ?? 0) + 1);
|
||||
const linked = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0])[0]?.[0] ?? null;
|
||||
const linkedEntry = linked === null ? undefined : entries.find((e) => e.id === linked);
|
||||
const centre = group.centre;
|
||||
if (!centre) {
|
||||
const all = linked !== null && group.members.every((m) => m.photo.entryIds.includes(linked));
|
||||
return { place: all ? (linkedEntry?.location ?? null) : null, entryId: linked };
|
||||
}
|
||||
let nearest: { entry: PhotoPickEntry; d: number } | null = null;
|
||||
for (const e of entries) {
|
||||
if (e.lat === null || e.lng === null) continue;
|
||||
const d = haversineMetres(centre.lat, centre.lng, e.lat, e.lng);
|
||||
if (d <= PHOTO_PICK.labelRadiusM && (!nearest || d < nearest.d)) nearest = { entry: e, d };
|
||||
}
|
||||
const name = nearest ? (nearest.entry.location ?? nearest.entry.title) : null;
|
||||
const sameDay = nearest && nearest.entry.date === group.day ? nearest.entry.id : null;
|
||||
return { place: name ?? linkedEntry?.location ?? null, entryId: linked ?? sameDay };
|
||||
}
|
||||
|
||||
/** Days in calendar order, `undated` last. */
|
||||
function byDay(a: string, b: string): number {
|
||||
if (a === 'undated' || b === 'undated') return Number(a === 'undated') - Number(b === 'undated');
|
||||
return a < b ? -1 : 1;
|
||||
}
|
||||
|
||||
/** Something taken at a time: a pick (its own id) or a placed photo (its photo's). */
|
||||
interface Timed {
|
||||
at: number | null;
|
||||
photoId?: number;
|
||||
photo?: { photoId: number };
|
||||
}
|
||||
|
||||
function byTime(a: Timed, b: Timed): number {
|
||||
if (a.at !== null && b.at !== null && a.at !== b.at) return a.at - b.at;
|
||||
if (a.at === null && b.at !== null) return 1;
|
||||
if (a.at !== null && b.at === null) return -1;
|
||||
return (a.photoId ?? a.photo?.photoId ?? 0) - (b.photoId ?? b.photo?.photoId ?? 0);
|
||||
}
|
||||
Reference in New Issue
Block a user