feat(photos): a photo knows its colours and its calm areas

Photo signals move to version 3: a deterministic palette, a busy grid and
the point of most detail, stored as measured values, never as image bytes.

server/src/db/entities/TrekPhotoSignals.entity.ts

@@ -16,6 +16,10 @@ export class TrekPhotoSignals {
   orig_width?: number | null;
   orig_height?: number | null;
   computed_at!: string & Opt;
+  palette?: string | null;
+  busy?: string | null;
+  sal_x?: number | null;
+  sal_y?: number | null;
 }
 
 export const TrekPhotoSignalsSchema = defineEntity({
@@ -34,5 +38,9 @@ export const TrekPhotoSignalsSchema = defineEntity({
     orig_width: p.integer().nullable(),
     orig_height: p.integer().nullable(),
     computed_at: p.type(DbTimestampType).defaultRaw(`CURRENT_TIMESTAMP`),
+    palette: p.text().nullable(),
+    busy: p.text().nullable(),
+    sal_x: p.double().nullable(),
+    sal_y: p.double().nullable(),
   },
 });

server/src/db/kysely/TrekPhotoSignals.table.ts

@@ -13,4 +13,8 @@ export interface TrekPhotoSignalsTable {
   orig_width: InsertOptional<number | null>;
   orig_height: InsertOptional<number | null>;
   computed_at: InsertOptional<string>;
+  palette: InsertOptional<string | null>;
+  busy: InsertOptional<string | null>;
+  sal_x: InsertOptional<number | null>;
+  sal_y: InsertOptional<number | null>;
 }

server/src/db/migrations/Migration20200101046270_a_photo_knows_its_colours_and_calm_areas.ts

@@ -0,0 +1,20 @@
+import { Migration } from '@mikro-orm/migrations';
+
+/**
+ * What else a photo's pixels say, for the assistant's book design: its main
+ * colours (`palette`, JSON of up to five `"#rrggbb:share"`), where it is busy
+ * and where calm (`busy`, an 8 x 8 grid of edge density, one hex digit per
+ * cell, row by row) and where its detail gathers (`sal_x`, `sal_y`, 0 to 1 of
+ * each side; an estimate, no face detection). Measured with the rest of the
+ * signals; a row of an older measurement is measured again.
+ */
+export class Migration20200101046270_a_photo_knows_its_colours_and_calm_areas extends Migration {
+  override name = 'Migration20200101046270_a_photo_knows_its_colours_and_calm_areas';
+
+  override up(): void {
+    this.addSql('ALTER TABLE trek_photo_signals ADD COLUMN palette TEXT');
+    this.addSql('ALTER TABLE trek_photo_signals ADD COLUMN busy TEXT');
+    this.addSql('ALTER TABLE trek_photo_signals ADD COLUMN sal_x REAL');
+    this.addSql('ALTER TABLE trek_photo_signals ADD COLUMN sal_y REAL');
+  }
+}

server/src/db/repositories/TrekPhotoSignals.repository.ts

@@ -15,6 +15,13 @@ export interface TrekPhotoSignalWrite {
   clipped: number | null;
   orig_width: number | null;
   orig_height: number | null;
+  /** The main colours as JSON of up to five `"#rrggbb:share"`; absent or null when not measured. */
+  palette?: string | null;
+  /** An 8 x 8 grid of edge density, one hex digit per cell, row by row. */
+  busy?: string | null;
+  /** Where the detail gathers, 0 to 1 of each side. */
+  sal_x?: number | null;
+  sal_y?: number | null;
 }
 
 /** A stored measurement, with when it was made. */
@@ -68,6 +75,10 @@ export class TrekPhotoSignalsRepository extends TrekRepository<TrekPhotoSignals>
             'orig_width',
             'orig_height',
             'computed_at',
+            'palette',
+            'busy',
+            'sal_x',
+            'sal_y',
           ])
           .where('photo_id', 'in', chunk)
           .execute(),
@@ -81,7 +92,7 @@ export class TrekPhotoSignalsRepository extends TrekRepository<TrekPhotoSignals>
     const computedAt = dbNow();
     await this.db()
       .insertInto('trek_photo_signals')
-      .values({ ...row, computed_at: computedAt })
+      .values({ ...row, ...colours(row), computed_at: computedAt })
       .onConflict((oc) =>
         oc.column('photo_id').doUpdateSet({
           algo: row.algo,
@@ -91,6 +102,7 @@ export class TrekPhotoSignalsRepository extends TrekRepository<TrekPhotoSignals>
           clipped: row.clipped,
           orig_width: row.orig_width,
           orig_height: row.orig_height,
+          ...colours(row),
           computed_at: computedAt,
         }),
       )
@@ -124,3 +136,13 @@ export class TrekPhotoSignalsRepository extends TrekRepository<TrekPhotoSignals>
       .execute();
   }
 }
+
+/** The colour signals of a measurement, null where it has none, so an update never keeps an older one's. */
+function colours(row: TrekPhotoSignalWrite) {
+  return {
+    palette: row.palette ?? null,
+    busy: row.busy ?? null,
+    sal_x: row.sal_x ?? null,
+    sal_y: row.sal_y ?? null,
+  };
+}

server/src/nest/photos/photo-colour-signals.ts

@@ -0,0 +1,167 @@
+/**
+ * What a photo's colours and edges say about where words and frames can go,
+ * for the assistant's book design (journey-studio-wp13.md 5.1): its main
+ * colours, where it is busy and where calm, and where its detail gathers.
+ * Pure arithmetic on the RGBA pixels the measurement already decoded and
+ * scaled (photo-signals-runner.ts), run in the same worker thread.
+ *
+ * None of this sees what is in a photo: the salient point is the centroid of
+ * the edges, not a face or a subject, and the palette is colour clustering,
+ * not taste.
+ */
+
+/** Clusters the palette is made of at most. */
+const PALETTE_SIZE = 5;
+/** Pixels the clustering looks at at most; a larger picture is sampled evenly. */
+const MAX_SAMPLES = 6000;
+/** Rounds of k-means after the start from the histogram. */
+const ROUNDS = 6;
+/** A cluster holding less of the picture than this is left out of the palette. */
+const MIN_SHARE = 0.02;
+/** Two starting colours at least this far apart (RGB distance), so the palette is not five shades of one sky. */
+const MIN_START_DISTANCE = 48;
+/** The busy grid's cells per side. */
+const GRID = 8;
+/** Mean Laplacian magnitude per step of the busy grid's 0..15 scale: 45 and more is 15. */
+const BUSY_STEP = 3;
+
+export interface ColourSignals {
+  /** Up to five `#rrggbb:share`, most of the picture first; null for a picture with no pixels to read. */
+  palette: string[] | null;
+  /** 8 x 8 cells, row by row, one hex digit each: 0 calm to f busy. */
+  busy: string | null;
+  /** The edge-energy centroid, 0..1 of the width and the height; null for a picture without edges. */
+  salX: number | null;
+  salY: number | null;
+}
+
+type Rgb = [number, number, number];
+
+const hex2 = (v: number) =>
+  Math.max(0, Math.min(255, Math.round(v)))
+    .toString(16)
+    .padStart(2, '0');
+
+const distance2 = (a: Rgb, b: Rgb) => (a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2;
+
+/** Every n-th pixel's colour, n chosen so at most MAX_SAMPLES are read. */
+function samples(data: Uint8Array, total: number): Rgb[] {
+  const stride = Math.max(1, Math.floor(total / MAX_SAMPLES));
+  const out: Rgb[] = [];
+  for (let i = 0; i < total; i += stride) {
+    const o = i * 4;
+    out.push([data[o] ?? 0, data[o + 1] ?? 0, data[o + 2] ?? 0]);
+  }
+  return out;
+}
+
+/** The starting colours: the fullest 4-bit histogram bins, each far enough from the ones before it. */
+function startColours(pixels: readonly Rgb[]): Rgb[] {
+  const counts = new Map<number, number>();
+  for (const [r, g, b] of pixels) {
+    const bin = ((r >> 4) << 8) | ((g >> 4) << 4) | (b >> 4);
+    counts.set(bin, (counts.get(bin) ?? 0) + 1);
+  }
+  const bins = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0] - b[0]);
+  const starts: Rgb[] = [];
+  for (const [bin] of bins) {
+    const colour: Rgb = [((bin >> 8) & 15) * 16 + 8, ((bin >> 4) & 15) * 16 + 8, (bin & 15) * 16 + 8];
+    if (starts.every((s) => distance2(s, colour) >= MIN_START_DISTANCE ** 2)) starts.push(colour);
+    if (starts.length === PALETTE_SIZE) break;
+  }
+  return starts;
+}
+
+/** The index of the centre nearest to a colour. */
+function nearest(centres: readonly Rgb[], colour: Rgb): number {
+  let best = 0;
+  let bestDistance = Infinity;
+  centres.forEach((centre, i) => {
+    const d = distance2(centre, colour);
+    if (d < bestDistance) {
+      best = i;
+      bestDistance = d;
+    }
+  });
+  return best;
+}
+
+/** A few rounds of k-means from the histogram's colours; deterministic for the same pixels. */
+export function paletteOf(data: Uint8Array, width: number, height: number): string[] | null {
+  const pixels = samples(data, width * height);
+  if (pixels.length === 0) return null;
+  const centres = startColours(pixels);
+  let members = new Array<number>(centres.length).fill(0);
+  for (let round = 0; round <= ROUNDS; round++) {
+    const sums = centres.map((): Rgb => [0, 0, 0]);
+    members = new Array<number>(centres.length).fill(0);
+    for (const colour of pixels) {
+      const at = nearest(centres, colour);
+      const sum = sums[at];
+      if (!sum) continue;
+      sum[0] += colour[0];
+      sum[1] += colour[1];
+      sum[2] += colour[2];
+      members[at] = (members[at] ?? 0) + 1;
+    }
+    // The last round only counts: the centres stay where the counting found them.
+    if (round === ROUNDS) break;
+    sums.forEach((sum, i) => {
+      const n = members[i] ?? 0;
+      if (n > 0) centres[i] = [sum[0] / n, sum[1] / n, sum[2] / n];
+    });
+  }
+  return centres
+    .map((centre, i) => ({ centre, share: (members[i] ?? 0) / pixels.length, i }))
+    .filter((c) => c.share >= MIN_SHARE)
+    .sort((a, b) => b.share - a.share || a.i - b.i)
+    .map((c) => `#${hex2(c.centre[0])}${hex2(c.centre[1])}${hex2(c.centre[2])}:${c.share.toFixed(2)}`);
+}
+
+/**
+ * The busy grid and the salient point from one pass of the grey Laplacian:
+ * each cell's mean magnitude on a 0..15 scale, and the centroid of all of it.
+ * The caller hands in at least 3 x 3 pixels.
+ */
+export function edgeSignals(data: Uint8Array, width: number, height: number): Omit<ColourSignals, 'palette'> {
+  const grey = new Float32Array(width * height);
+  for (let i = 0; i < grey.length; i++) {
+    const o = i * 4;
+    grey[i] = 0.299 * (data[o] ?? 0) + 0.587 * (data[o + 1] ?? 0) + 0.114 * (data[o + 2] ?? 0);
+  }
+  const at = (k: number) => grey[k] ?? 0;
+  const sums = new Array<number>(GRID * GRID).fill(0);
+  const counts = new Array<number>(GRID * GRID).fill(0);
+  let energy = 0;
+  let sx = 0;
+  let sy = 0;
+  for (let row = 1; row < height - 1; row++) {
+    const cellRow = Math.min(GRID - 1, Math.floor((row * GRID) / height));
+    for (let col = 1; col < width - 1; col++) {
+      const i = row * width + col;
+      const lap = Math.abs(at(i - 1) + at(i + 1) + at(i - width) + at(i + width) - 4 * at(i));
+      const cell = cellRow * GRID + Math.min(GRID - 1, Math.floor((col * GRID) / width));
+      sums[cell] = (sums[cell] ?? 0) + lap;
+      counts[cell] = (counts[cell] ?? 0) + 1;
+      energy += lap;
+      sx += lap * ((col + 0.5) / width);
+      sy += lap * ((row + 0.5) / height);
+    }
+  }
+  const busy = sums
+    .map((sum, cell) => {
+      const n = counts[cell] ?? 0;
+      const level = n > 0 ? Math.min(15, Math.round(sum / n / BUSY_STEP)) : 0;
+      return level.toString(16);
+    })
+    .join('');
+  if (energy <= 0) return { busy, salX: null, salY: null };
+  return { busy, salX: round3(sx / energy), salY: round3(sy / energy) };
+}
+
+const round3 = (v: number) => Math.round(v * 1000) / 1000;
+
+/** Everything above, for one decoded and scaled picture. */
+export function colourSignals(data: Uint8Array, width: number, height: number): ColourSignals {
+  return { palette: paletteOf(data, width, height), ...edgeSignals(data, width, height) };
+}

server/src/nest/photos/photo-signals-runner.ts

@@ -1,11 +1,14 @@
+import { colourSignals, type ColourSignals } from './photo-colour-signals';
+
 import { Jimp } from 'jimp';
 
 /**
  * The version of the measurement below. A stored row of another version is
  * measured again, so a better measurement never mixes with an older one.
- * Version 2 measures every photo at the same `SIGNAL_EDGE`, up or down.
+ * Version 2 measures every photo at the same `SIGNAL_EDGE`, up or down;
+ * version 3 adds the colours, the busy grid and the salient point.
  */
-export const PHOTO_SIGNALS_ALGO = 2;
+export const PHOTO_SIGNALS_ALGO = 3;
 
 /**
  * The long edge every photo is scaled to, up or down, before it is measured. A
@@ -23,7 +26,7 @@ const CLIP_LOW = 5;
 const CLIP_HIGH = 250;
 
 /** What a photo's pixels say. `width`/`height` are the decoded image's, before it is scaled to measure. */
-export interface PhotoPixels {
+export interface PhotoPixels extends ColourSignals {
   /** The perceptual hash, 64 bits as 16 hex digits; null if the hash came out in another shape. */
   phash: string | null;
   /** The variance of the grey Laplacian at `SIGNAL_EDGE`: the higher, the sharper. */
@@ -51,8 +54,9 @@ export async function measurePhoto(bytes: Uint8Array): Promise<PhotoPixels | nul
     image.scaleToFit({ w: SIGNAL_EDGE, h: SIGNAL_EDGE });
     // A strip far longer than it is wide ends up under 3 pixels across, which leaves no inner pixel.
     if (image.bitmap.width < 3 || image.bitmap.height < 3) return null;
-    const grey = greyStats(image.bitmap.data, image.bitmap.width, image.bitmap.height);
-    return { phash: bitsToHex(image.pHash()), ...grey, width, height };
+    const { data, width: w, height: h } = image.bitmap;
+    const grey = greyStats(data, w, h);
+    return { phash: bitsToHex(image.pHash()), ...grey, ...colourSignals(data, w, h), width, height };
   } catch {
     return null;
   }

server/src/nest/photos/photo-signals.service.ts

@@ -1,9 +1,11 @@
 import { TrekPhotoSignals } from '../../db/entities/TrekPhotoSignals.entity';
+import { TREK_PHOTO_SIGNAL_PALETTE } from '../../db/json-columns';
 import type {
   GalleryPhotoFacts,
   TrekPhotoSignalRow,
   TrekPhotoSignalsRepository,
 } from '../../db/repositories/TrekPhotoSignals.repository';
+import { decodeJson, encodeJson } from '../../utils/json-column';
 import { logWarn } from '../audit/audit-log.logger';
 import { withRequestContext } from '../database/request-context';
 import { PhotoResolverService, type PhotoSignalSource } from '../memories/photo-resolver.service';
@@ -16,7 +18,7 @@ import {
 import { MikroORM } from '@mikro-orm/core';
 import { InjectRepository } from '@mikro-orm/nestjs';
 import { Inject, Injectable, type OnModuleDestroy } from '@nestjs/common';
-import type { PhotoPickSignals } from '@trek/shared';
+import type { BookPhotoInfo, PhotoPickSignals } from '@trek/shared';
 
 /** How long a caller waits for photos still being measured before it answers on what the rows know. */
 const SIGNALS_WAIT_MS = 8000;
@@ -27,6 +29,9 @@ export const PARALLEL_FETCHES = 2;
 /** A local original larger than this is not read for its signals. */
 export const MAX_ORIGINAL_BYTES = 40 * 1024 * 1024;
 
+/** What the book engine reads of a photo's measurement: the original's size, brightness, colours and calm areas. */
+type BookPhotoSignals = Pick<BookPhotoInfo, 'width' | 'height' | 'luma' | 'palette' | 'busy' | 'salient'>;
+
 /** Errors of the runner that say nothing about the photo: it is measured again on a later call. */
 const TRANSIENT = new Set(['busy', 'closed', 'unavailable']);
 
@@ -107,6 +112,19 @@ export class PhotoSignalsService implements OnModuleDestroy {
     return { byPhoto, pending: ids.filter((id) => !byPhoto.has(id)).length };
   }
 
+  /**
+   * What the book engine reads of these photos, from whatever measurement is
+   * stored, none measured or waited for: the original's size for the print
+   * check, and from the current measurement the colours, the busy grid and
+   * the salient point. A photo without a row is left out.
+   */
+  async bookSignals(photoIds: readonly number[]): Promise<Map<number, BookPhotoSignals>> {
+    const ids = [...new Set(photoIds)];
+    if (ids.length === 0) return new Map();
+    const rows = await this.signalsRepo.listForPhotos(ids);
+    return new Map(rows.map((row) => [row.photo_id, toBookSignals(row)]));
+  }
+
   /** The signals stored for these photos by the current measurement; older ones count as missing. */
   private async stored(ids: number[]): Promise<Map<number, PhotoPickSignals>> {
     if (ids.length === 0) return new Map();
@@ -183,6 +201,10 @@ export class PhotoSignalsService implements OnModuleDestroy {
       luma: pixels?.luma ?? null,
       clipped: pixels?.clipped ?? null,
       ...originalSize(source, pixels),
+      palette: pixels?.palette ? encodeJson(TREK_PHOTO_SIGNAL_PALETTE, pixels.palette) : null,
+      busy: pixels?.busy ?? null,
+      sal_x: pixels?.salX ?? null,
+      sal_y: pixels?.salY ?? null,
     });
   }
 
@@ -222,6 +244,32 @@ function toSignals(row: TrekPhotoSignalRow): PhotoPickSignals {
   };
 }
 
+const BUSY_GRID = /^[0-9a-f]{64}$/;
+
+/** A stored palette as the engine reads it; anything that is not the measurement's own shape reads as none. */
+function paletteOf(row: TrekPhotoSignalRow): string[] | undefined {
+  const entries = decodeJson(TREK_PHOTO_SIGNAL_PALETTE, row.palette, `photo ${row.photo_id}`);
+  return entries.length > 0 ? entries : undefined;
+}
+
+function toBookSignals(row: TrekPhotoSignalRow): BookPhotoSignals {
+  const fresh = row.algo === PHOTO_SIGNALS_ALGO;
+  const palette = fresh ? paletteOf(row) : undefined;
+  const busy = fresh && row.busy && BUSY_GRID.test(row.busy) ? row.busy : undefined;
+  const salient =
+    fresh && typeof row.sal_x === 'number' && typeof row.sal_y === 'number'
+      ? { x: row.sal_x, y: row.sal_y }
+      : undefined;
+  return {
+    width: row.orig_width,
+    height: row.orig_height,
+    ...(typeof row.luma === 'number' ? { luma: Math.round(row.luma * 255) } : {}),
+    ...(palette ? { palette } : {}),
+    ...(busy ? { busy } : {}),
+    ...(salient ? { salient } : {}),
+  };
+}
+
 /** Resolves when `work` settles or after `ms`, whichever comes first; never rejects. */
 function waitAtMost(work: Promise<unknown>, ms: number): Promise<void> {
   return new Promise((resolve) => {