TREK/server/tests/unit/nest/storage-reference-sources.test.ts

/**
 * The reference sources the owner domains hand the storage reclaim pass
 * (#2924), each against the migrated schema: which of a page of keys its rows
 * still name, in every spelling those columns hold. One object kept too many
 * costs a few bytes; one called unreferenced while a row names it is lost, so
 * every case here is a spelling a real row can carry. A source answers twice:
 * `namedKeys`, every key of a category its rows name (one read, for a pass),
 * and `referenced`, a few keys asked fresh and narrowed to the rows holding
 * their plain runs (the re-check before a delete, and a release).
 * RSRC-001 to RSRC-017.
 */
import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity';
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
import { Users } from '../../../src/db/entities/Users.entity';
import { AuthStorageRefsService } from '../../../src/nest/auth/auth-storage-refs.service';
import { CollectionsStorageRefsService } from '../../../src/nest/collections/collections-storage-refs.service';
import { FilesStorageRefsService } from '../../../src/nest/files/files-storage-refs.service';
import { JourneyStorageRefsService } from '../../../src/nest/journey/journey-storage-refs.service';
import { PhotosStorageRefsService } from '../../../src/nest/photos/photos-storage-refs.service';
import { PlaceContribStorageRefsService } from '../../../src/nest/place-contrib/place-contrib-storage-refs.service';
import { PlacesStorageRefsService } from '../../../src/nest/places/places-storage-refs.service';
import { StorageReclaimRegistryService } from '../../../src/nest/storage/storage-reclaim-registry.service';
import { likeAny, MAX_LIKE_TERMS } from '../../../src/nest/storage/storage-reclaim-source.service';
import {
  STORAGE_RECLAIMABLE_CATEGORIES,
  STORAGE_REFERENCE_SOURCE_IDS,
  type StorageReferenceSource,
} from '../../../src/nest/storage/storage-reclaim.types';
import type { ServedCategory } from '../../../src/nest/storage/storage.types';
import { TripsStorageRefsService } from '../../../src/nest/trips/trips-storage-refs.service';
import { createSnapshotTestDb } from '../../helpers/db-mock';
import { makeCollection, makeCollectionPlace } from '../../helpers/factories/collections';
import { makeTripFile } from '../../helpers/factories/files';
import { makeJourney } from '../../helpers/factories/journeys';
import { makePlace } from '../../helpers/factories/places';
import { insertRow, updateRows } from '../../helpers/factories/rows';
import { makeTrip } from '../../helpers/factories/trips';
import { makeUser } from '../../helpers/factories/users';
import { createTestModule, type TestModule } from '../../helpers/test-module';
import { createTestOrm, type TestOrm } from '../../helpers/test-orm';

import { afterAll, beforeAll, describe, expect, it } from 'vitest';

const SOURCES = [
  TripsStorageRefsService,
  CollectionsStorageRefsService,
  PlacesStorageRefsService,
  PlaceContribStorageRefsService,
  JourneyStorageRefsService,
  PhotosStorageRefsService,
  FilesStorageRefsService,
  AuthStorageRefsService,
];

const db = createSnapshotTestDb();
let orm: TestOrm;
let t: TestModule;
let userId: number;
let tripId: number;

beforeAll(async () => {
  orm = await createTestOrm(db);
  t = await createTestModule({ db, imports: [], providers: [StorageReclaimRegistryService, ...SOURCES], init: true });
  userId = (await makeUser(orm)).user.id;
  tripId = (await makeTrip(orm, userId)).id;
});

afterAll(async () => {
  await t.close();
  await orm.close();
  db.close();
});

/** The sorted keys a source still references. */
async function asked(source: StorageReferenceSource, category: ServedCategory, keys: string[]): Promise<string[]> {
  return [...(await source.referenced(category, keys))].sort();
}

describe('reference sources', () => {
  it('RSRC-001: every source registers itself once the app boots, under the ids the ledger knows', () => {
    const registry = t.get(StorageReclaimRegistryService);
    expect([...registry.ids()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort());
    for (const source of registry.sources()) {
      const claimed = source.claims.map((claim) => `${claim.category}:${claim.prefix}`).sort();
      if (source.id === 'files') expect(claimed).toEqual(['files:']);
      else if (source.id === 'auth') expect(claimed).toEqual(['avatars:']);
      else expect(claimed).toEqual(STORAGE_RECLAIMABLE_CATEGORIES.map((category) => `${category}:`).sort());
    }
  });

  it('RSRC-002: a trip cover keeps its object in every spelling, and in whatever category it points into', async () => {
    await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-a.jpg' });
    await makeTrip(orm, userId, { cover_image: 'https://trip.example.test/uploads/places/rsrc-b.jpg?v=2' });
    await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Mixed.JPG' });
    await makeTrip(orm, userId, { cover_image: 'https://images.unsplash.com/rsrc-z.jpg' });
    const trips = t.get(TripsStorageRefsService);
    expect(await asked(trips, 'covers', ['rsrc-a.jpg', 'rsrc-mixed.jpg', 'rsrc-z.jpg', 'rsrc-b.jpg'])).toEqual([
      'rsrc-a.jpg',
      'rsrc-mixed.jpg',
    ]);
    expect(await asked(trips, 'places', ['rsrc-b.jpg', 'rsrc-a.jpg'])).toEqual(['rsrc-b.jpg']);
    expect(await asked(trips, 'covers', [])).toEqual([]);
  });

  it('RSRC-003: a list cover and a saved place image both keep their objects', async () => {
    const collection = await makeCollection(orm, userId, { cover_image: '/uploads/covers/rsrc-c.jpg' });
    await makeCollectionPlace(orm, collection.id, userId, { image_url: '/uploads/places/rsrc-d.jpg' });
    await makeCollectionPlace(orm, collection.id, userId, { image_url: 'https://example.test/rsrc-y.jpg' });
    const collections = t.get(CollectionsStorageRefsService);
    expect(await asked(collections, 'covers', ['rsrc-c.jpg', 'rsrc-d.jpg'])).toEqual(['rsrc-c.jpg']);
    expect(await asked(collections, 'places', ['rsrc-d.jpg', 'rsrc-y.jpg'])).toEqual(['rsrc-d.jpg']);
  });

  it('RSRC-004: a place image keeps its object; a photo-proxy path or an inline image names none', async () => {
    await makePlace(orm, tripId, { image_url: '/uploads/places/rsrc-e.jpg' });
    await makePlace(orm, tripId, { image_url: '/uploads/covers/rsrc-e2.jpg' });
    await makePlace(orm, tripId, { image_url: '/api/maps/place-photo/rsrc-x/bytes' });
    await makePlace(orm, tripId, { image_url: 'data:image/png;base64,AAAA' });
    const places = t.get(PlacesStorageRefsService);
    expect(await asked(places, 'places', ['rsrc-e.jpg', 'rsrc-x', 'bytes'])).toEqual(['rsrc-e.jpg']);
    expect(await asked(places, 'covers', ['rsrc-e2.jpg'])).toEqual(['rsrc-e2.jpg']);
  });

  it('RSRC-005: a queued contribution keeps the image it has yet to send', async () => {
    const place = await makePlace(orm, tripId, { image_url: null });
    await insertRow(orm, PlaceContribOutbox, {
      contrib_key: 'osm:node/2924',
      field: 'image',
      value: '/uploads/places/rsrc-f.jpg',
      place: place.id,
      place_name: 'Queued',
      cid: 'cid-rsrc-0001',
      eligible_at: '2026-01-01 00:00:00',
      next_attempt_at: '2026-01-01 00:00:00',
    });
    const outbox = t.get(PlaceContribStorageRefsService);
    expect(await asked(outbox, 'places', ['rsrc-f.jpg', 'rsrc-f2.jpg'])).toEqual(['rsrc-f.jpg']);
  });

  it('RSRC-006: a journey cover keeps a shared trip cover and a gallery photo by their relative spelling', async () => {
    await makeJourney(orm, userId, { cover_image: 'covers/rsrc-g.jpg' });
    await makeJourney(orm, userId, { cover_image: 'journey/rsrc-h.jpg' });
    await makeJourney(orm, userId, { cover_image: '/uploads/covers/rsrc-g2.jpg' });
    const journeys = t.get(JourneyStorageRefsService);
    expect(await asked(journeys, 'covers', ['rsrc-g.jpg', 'rsrc-g2.jpg', 'rsrc-h.jpg'])).toEqual([
      'rsrc-g.jpg',
      'rsrc-g2.jpg',
    ]);
    expect(await asked(journeys, 'journey', ['rsrc-h.jpg', 'rsrc-g.jpg'])).toEqual(['rsrc-h.jpg']);
  });

  it('RSRC-007: every photo row with a path keeps it, whatever its provider and links', async () => {
    // A local photo retargeted to a remote provider is still served from its bytes.
    await insertRow(orm, TrekPhotos, {
      provider: 'immich',
      asset_id: 'rsrc-asset',
      owner: userId,
      file_path: 'journey/rsrc-i.jpg',
      thumbnail_path: 'journey/rsrc-i-poster.jpg',
    });
    await insertRow(orm, TrekPhotos, { provider: 'local', owner: userId, file_path: 'legacy/rsrc-j.jpg' });
    await insertRow(orm, TrekPhotos, { provider: 'immich', asset_id: 'rsrc-remote', owner: userId });
    const photos = t.get(PhotosStorageRefsService);
    expect(
      await asked(photos, 'journey', ['rsrc-i.jpg', 'rsrc-i-poster.jpg', 'rsrc-j.jpg', 'rsrc-remote', 'rsrc-k.jpg']),
    ).toEqual(['rsrc-i-poster.jpg', 'rsrc-i.jpg', 'rsrc-j.jpg']);
    expect(await asked(photos, 'covers', ['rsrc-i.jpg'])).toEqual([]);
  });

  it('RSRC-008: a trip file keeps its object, trashed or carrying a legacy path', async () => {
    await makeTripFile(orm, tripId, { filename: 'rsrc-k.pdf' });
    await makeTripFile(orm, tripId, { filename: 'rsrc-l.pdf', deleted_at: '2026-01-01 00:00:00' });
    await makeTripFile(orm, tripId, { filename: 'files/rsrc-m.pdf' });
    await makeTripFile(orm, tripId, { filename: '/uploads/files/rsrc-n.pdf' });
    await makeTripFile(orm, tripId, { filename: 'uploads\\files\\rsrc-p.pdf' });
    const files = t.get(FilesStorageRefsService);
    expect(
      await asked(files, 'files', ['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf', 'rsrc-o.pdf']),
    ).toEqual(['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf']);
  });

  it('RSRC-009: a source asked about a category it does not claim answers nothing', async () => {
    await makeTripFile(orm, tripId, { filename: 'rsrc-q.jpg' });
    expect(await asked(t.get(FilesStorageRefsService), 'covers', ['rsrc-q.jpg'])).toEqual([]);
    expect(await asked(t.get(FilesStorageRefsService), 'files', [])).toEqual([]);
    expect(await asked(t.get(AuthStorageRefsService), 'files', ['rsrc-q.jpg'])).toEqual([]);
  });

  it('RSRC-010: an uploaded avatar keeps its object; an OIDC picture names none', async () => {
    const uploaded = (await makeUser(orm)).user;
    const oidc = (await makeUser(orm)).user;
    await updateRows(orm, Users, { id: uploaded.id }, { avatar: 'rsrc-r.png' });
    await updateRows(orm, Users, { id: oidc.id }, { avatar: 'https://idp.example.test/pictures/rsrc-s.png' });
    const auth = t.get(AuthStorageRefsService);
    expect(await asked(auth, 'avatars', ['rsrc-r.png', 'rsrc-s.png'])).toEqual(['rsrc-r.png']);
    expect(await asked(auth, 'avatars', [])).toEqual([]);
  });

  it('RSRC-011: every source asked with no keys answers nothing', async () => {
    const answers = await Promise.all(
      SOURCES.map((Source) => asked(t.get<StorageReferenceSource>(Source), 'covers', [])),
    );
    expect(answers).toEqual(SOURCES.map(() => []));
  });

  it('RSRC-012: a narrowed question still finds a uuid-named object in every spelling a row holds', async () => {
    const id = '7b1f0c2e-5a4d-4e3f-8a9b-0c1d2e3f4a5b';
    await makeTrip(orm, userId, {
      cover_image: `https://trip.example.test/uploads/covers/${id.toUpperCase()}.JPG?v=3`,
    });
    await makePlace(orm, tripId, { image_url: `/uploads/places/${id}-p.jpg#frag` });
    const trips = t.get(TripsStorageRefsService);
    expect(await asked(trips, 'covers', [`${id}.jpg`, `${id}-other.jpg`])).toEqual([`${id}.jpg`]);
    expect(await asked(t.get(PlacesStorageRefsService), 'places', [`${id}-p.jpg`])).toEqual([`${id}-p.jpg`]);
  });

  it('RSRC-013: namedKeys answers every key of the category its rows name, lowercased, in one read', async () => {
    await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Named-A.jpg' });
    await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-named-b.jpg' });
    const trips = t.get(TripsStorageRefsService);
    const covers = await trips.namedKeys('covers');
    expect(covers.has('rsrc-named-a.jpg')).toBe(true);
    expect(covers.has('rsrc-named-b.jpg')).toBe(false);
    expect((await trips.namedKeys('places')).has('rsrc-named-b.jpg')).toBe(true);
    expect(await t.get(FilesStorageRefsService).namedKeys('covers')).toEqual(new Set());
  });

  it('RSRC-014: a trip file whose name holds a #, a ? or a trailing blank keeps its object both ways', async () => {
    const id = '3c9d8e7f-6a5b-4c4d-9e3f-2a1b0c9d8e7f';
    const names = [`${id}. #4711`, `${id}.c#`, `${id}.v2?final`, `${id}.pdf `];
    await Promise.all(names.map((filename) => makeTripFile(orm, tripId, { filename })));
    const files = t.get(FilesStorageRefsService);
    expect(await asked(files, 'files', names)).toEqual([...names].sort());
    const named = await files.namedKeys('files');
    for (const name of names) expect(named.has(name.toLowerCase())).toBe(true);
  });

  it('RSRC-015: a question about many keys is split into reads of at most MAX_LIKE_TERMS terms each', async () => {
    const ids = Array.from({ length: 60 }, (_unused, i) => `9d8c7b6a-5f4e-4d3c-8b2a-${String(i).padStart(12, '0')}`);
    await Promise.all(ids.slice(0, 30).map((id) => makePlace(orm, tripId, { image_url: `/uploads/places/${id}.jpg` })));
    const keys = ids.map((id) => `${id}.jpg`);
    expect(await asked(t.get(PlacesStorageRefsService), 'places', keys)).toEqual(keys.slice(0, 30).sort());
    // The photos source reads two columns, still within the bound.
    expect(await asked(t.get(PhotosStorageRefsService), 'journey', keys)).toEqual([]);
  });

  it('RSRC-016: a filter past MAX_LIKE_TERMS terms is refused before it reaches SQLite', () => {
    const runs = Array.from({ length: MAX_LIKE_TERMS + 1 }, (_unused, i) => `run-${String(i).padStart(8, '0')}`);
    expect(() => likeAny<{ a: string }>(['a'], runs)).toThrow(/at most/);
    expect(() => likeAny<{ a: string; b: string }>(['a', 'b'], runs.slice(0, 25))).not.toThrow();
    expect(() => likeAny<{ a: string; b: string }>(['a', 'b'], runs.slice(0, 26))).toThrow(/at most/);
  });

  it('RSRC-017: namedKeysByCategory reads once and answers every category the source claims, none it does not', async () => {
    await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-once-a.jpg' });
    await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-once-b.jpg' });
    const answer = await t.get(TripsStorageRefsService).namedKeysByCategory(['covers', 'places', 'photos']);
    expect(answer.get('covers')?.has('rsrc-once-a.jpg')).toBe(true);
    expect(answer.get('covers')?.has('rsrc-once-b.jpg')).toBe(false);
    expect(answer.get('places')?.has('rsrc-once-b.jpg')).toBe(true);
    expect(answer.has('photos')).toBe(false);
    expect(await t.get(FilesStorageRefsService).namedKeysByCategory(['covers'])).toEqual(new Map());
  });
});