From 095bed2db835192cf4ae2900656ee62519c84b00 Mon Sep 17 00:00:00 2001 From: Maurice Date: Sun, 1 Nov 2026 13:59:19 +0100 Subject: [PATCH] feat(storage): let every owner of an upload column tell the reclaim pass what it still names Trips, collections, places, the contribution queue, journeys, photos, trip files and users each register a reference source from their own domain. A source reads its own columns and parses every spelling they hold through the storage helpers, and the free-form columns claim every reclaimable category, since a cover or an image can point into any of them. The contribution queue joins the ledger as its own source, because place-contrib owns that table. --- .../nest/auth/auth-storage-refs.service.ts | 21 ++ server/src/nest/auth/auth.module.ts | 2 + .../collections-storage-refs.service.ts | 28 +++ .../nest/collections/collections.module.ts | 9 +- .../nest/files/files-storage-refs.service.ts | 26 +++ server/src/nest/files/files.module.ts | 3 +- .../journey/journey-storage-refs.service.ts | 24 +++ server/src/nest/journey/journey.module.ts | 10 +- .../photos/photos-storage-refs.service.ts | 32 +++ server/src/nest/photos/photos.module.ts | 6 +- .../place-contrib-storage-refs.service.ts | 22 ++ .../place-contrib/place-contrib.module.ts | 10 +- .../places/places-storage-refs.service.ts | 23 ++ server/src/nest/places/places.module.ts | 3 +- .../storage/storage-reclaim-source.service.ts | 49 +++++ .../nest/storage/storage-reclaim.ledger.ts | 12 +- .../src/nest/storage/storage-reclaim.types.ts | 6 + .../nest/trips/trips-storage-refs.service.ts | 22 ++ server/src/nest/trips/trips.module.ts | 3 +- .../nest/storage-reference-sources.test.ts | 200 ++++++++++++++++++ .../storage/storage-reclaim.ledger.test.ts | 2 +- 21 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 server/src/nest/auth/auth-storage-refs.service.ts create mode 100644 server/src/nest/collections/collections-storage-refs.service.ts create mode 100644 server/src/nest/files/files-storage-refs.service.ts create mode 100644 server/src/nest/journey/journey-storage-refs.service.ts create mode 100644 server/src/nest/photos/photos-storage-refs.service.ts create mode 100644 server/src/nest/place-contrib/place-contrib-storage-refs.service.ts create mode 100644 server/src/nest/places/places-storage-refs.service.ts create mode 100644 server/src/nest/storage/storage-reclaim-source.service.ts create mode 100644 server/src/nest/trips/trips-storage-refs.service.ts create mode 100644 server/tests/unit/nest/storage-reference-sources.test.ts diff --git a/server/src/nest/auth/auth-storage-refs.service.ts b/server/src/nest/auth/auth-storage-refs.service.ts new file mode 100644 index 000000000..9af40df6c --- /dev/null +++ b/server/src/nest/auth/auth-storage-refs.service.ts @@ -0,0 +1,21 @@ +import { Users } from '../../db/entities/Users.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which avatar objects a user row still names + * (#2924). `users.avatar` holds the bare object name of an uploaded avatar; + * an https picture taken from the OIDC claim names no object of ours and + * parses as nothing. + */ +@Injectable() +export class AuthStorageRefsService extends StorageReclaimSource { + readonly id = 'auth'; + readonly claims = [{ category: 'avatars', prefix: '' }] as const; + protected override readonly keyOptions = { bare: 'avatars' } as const; + + protected async storedValues(): Promise> { + const rows = await this.em.getRepository(Users).find({ avatar: { $ne: null } }, { fields: ['avatar'] }); + return rows.map((row) => row.avatar); + } +} diff --git a/server/src/nest/auth/auth.module.ts b/server/src/nest/auth/auth.module.ts index 3888836ad..fad90130b 100644 --- a/server/src/nest/auth/auth.module.ts +++ b/server/src/nest/auth/auth.module.ts @@ -34,6 +34,7 @@ import { StorageService } from '../storage/storage.service'; import { TokensModule } from '../tokens/tokens.module'; import { TripMembershipModule } from '../trip-membership/trip-membership.module'; import { AuthPublicController } from './auth-public.controller'; +import { AuthStorageRefsService } from './auth-storage-refs.service'; import { AuthController } from './auth.controller'; import { AVATAR_FILE_FILTER, MAX_AVATAR_SIZE } from './auth.controller'; import { AuthMcp } from './auth.mcp'; @@ -163,6 +164,7 @@ import { MulterModule } from '@nestjs/platform-express'; UserCleanupService, WebauthnConfigService, AuthMcp, + AuthStorageRefsService, ], exports: [AuthService, RegistrationInvitesService, PasskeyService, UserCleanupService], }) diff --git a/server/src/nest/collections/collections-storage-refs.service.ts b/server/src/nest/collections/collections-storage-refs.service.ts new file mode 100644 index 000000000..93a8e89f6 --- /dev/null +++ b/server/src/nest/collections/collections-storage-refs.service.ts @@ -0,0 +1,28 @@ +import { CollectionPlaces } from '../../db/entities/CollectionPlaces.entity'; +import { Collections } from '../../db/entities/Collections.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects a collection still names + * (#2924): the list's own cover and every saved place's image. Saving a place + * into a list and copying it to a trip share the uploaded image by reference, + * and both columns take any string the client sends, so every category is + * claimed and every spelling is parsed. + */ +@Injectable() +export class CollectionsStorageRefsService extends StorageReclaimSource { + readonly id = 'collections'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + + protected async storedValues(): Promise> { + const covers = await this.em + .getRepository(Collections) + .find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + const images = await this.em + .getRepository(CollectionPlaces) + .find({ image_url: { $ne: null } }, { fields: ['image_url'] }); + return [...covers.map((row) => row.cover_image), ...images.map((row) => row.image_url)]; + } +} diff --git a/server/src/nest/collections/collections.module.ts b/server/src/nest/collections/collections.module.ts index 9950aa145..835b1fabf 100644 --- a/server/src/nest/collections/collections.module.ts +++ b/server/src/nest/collections/collections.module.ts @@ -22,6 +22,7 @@ import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; import { StorageModule } from '../storage/storage.module'; import { StorageService } from '../storage/storage.service'; import { CollectionsItemsService } from './collections-items.service'; +import { CollectionsStorageRefsService } from './collections-storage-refs.service'; import { CollectionsController } from './collections.controller'; import { MAX_COVER_SIZE } from './collections.controller'; import { CollectionsMcp } from './collections.mcp'; @@ -77,7 +78,13 @@ import { MulterModule } from '@nestjs/platform-express'; ]), ], controllers: [CollectionsController], - providers: [CollectionsService, CollectionsItemsService, CollectionsMcp, CollectionsRpc], + providers: [ + CollectionsService, + CollectionsItemsService, + CollectionsMcp, + CollectionsRpc, + CollectionsStorageRefsService, + ], exports: [CollectionsService], }) export class CollectionsModule {} diff --git a/server/src/nest/files/files-storage-refs.service.ts b/server/src/nest/files/files-storage-refs.service.ts new file mode 100644 index 000000000..83df3de53 --- /dev/null +++ b/server/src/nest/files/files-storage-refs.service.ts @@ -0,0 +1,26 @@ +import { TripFiles } from '../../db/entities/TripFiles.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which objects of the files category a trip + * file row still names (#2924). `trip_files.filename` is the bare object name; + * rows from before the storage layer may still carry a path in front of it, + * which every reader strips with basename(), so the last segment counts too. + * Trashed rows count: a restore brings the file back, and doc-sync keeps its + * superseded copies in the trash on purpose. Collab note files and chat images + * are trip_files rows as well. The pass only ever asks about the category's + * root (the assistant's subfolder has a sweep of its own), so a basename never + * reaches into a subfolder. + */ +@Injectable() +export class FilesStorageRefsService extends StorageReclaimSource { + readonly id = 'files'; + readonly claims = [{ category: 'files', prefix: '' }] as const; + protected override readonly keyOptions = { bare: 'files', basename: true } as const; + + protected async storedValues(): Promise> { + const rows = await this.em.getRepository(TripFiles).find({}, { fields: ['filename'] }); + return rows.map((row) => row.filename); + } +} diff --git a/server/src/nest/files/files.module.ts b/server/src/nest/files/files.module.ts index 33cd5ed51..7bfec61e1 100644 --- a/server/src/nest/files/files.module.ts +++ b/server/src/nest/files/files.module.ts @@ -19,6 +19,7 @@ import { TripMembershipModule } from '../trip-membership/trip-membership.module' import { AllowedFileTypesModule } from './allowed-file-types.module'; import { AllowedFileTypesService } from './allowed-file-types.service'; import { FilesDownloadController } from './files-download.controller'; +import { FilesStorageRefsService } from './files-storage-refs.service'; import { MAX_FILE_SIZE, MAX_VIDEO_SIZE } from './files.constants'; import { FilesController } from './files.controller'; import { filesUploadFileFilter } from './files.controller'; @@ -65,7 +66,7 @@ import { MulterModule } from '@nestjs/platform-express'; AllowedFileTypesModule, ], controllers: [FilesController, FilesDownloadController], - providers: [FilesService, FilesRpc, FilesMcp], + providers: [FilesService, FilesRpc, FilesMcp, FilesStorageRefsService], exports: [FilesService], }) export class FilesModule {} diff --git a/server/src/nest/journey/journey-storage-refs.service.ts b/server/src/nest/journey/journey-storage-refs.service.ts new file mode 100644 index 000000000..a591c246a --- /dev/null +++ b/server/src/nest/journey/journey-storage-refs.service.ts @@ -0,0 +1,24 @@ +import { Journeys } from '../../db/entities/Journeys.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects a journey still names + * (#2924). `journeys.cover_image` is inherited from the first linked trip as + * `covers/` (the trip and the journey share the object), set to a + * gallery photo's `journey/`, or whatever a PATCH sent. Every category + * is claimed; the photos themselves are the photos domain's rows. + */ +@Injectable() +export class JourneyStorageRefsService extends StorageReclaimSource { + readonly id = 'journey'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + + protected async storedValues(): Promise> { + const rows = await this.em + .getRepository(Journeys) + .find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + return rows.map((row) => row.cover_image); + } +} diff --git a/server/src/nest/journey/journey.module.ts b/server/src/nest/journey/journey.module.ts index 4aef50d2d..8e8de9a4a 100644 --- a/server/src/nest/journey/journey.module.ts +++ b/server/src/nest/journey/journey.module.ts @@ -16,6 +16,7 @@ import { JourneyMapMcp } from './journey-map.mcp'; import { JourneyPhotoCaptureModule } from './journey-photo-capture.module'; import { JourneyPhotoLinksMcp } from './journey-photos.mcp'; import { JourneyPublicController } from './journey-public.controller'; +import { JourneyStorageRefsService } from './journey-storage-refs.service'; import { JourneyTracksMcp } from './journey-tracks.mcp'; import { JourneyController } from './journey.controller'; import { journeyImageFileFilter, journeyUploadFilename } from './journey.controller'; @@ -62,6 +63,13 @@ import { MulterModule } from '@nestjs/platform-express'; MikroOrmModule.forFeature([Users]), ], controllers: [JourneyController, JourneyMapController, JourneyPublicController], - providers: [JourneyService, JourneyMcp, JourneyMapMcp, JourneyTracksMcp, JourneyPhotoLinksMcp], + providers: [ + JourneyService, + JourneyMcp, + JourneyMapMcp, + JourneyTracksMcp, + JourneyPhotoLinksMcp, + JourneyStorageRefsService, + ], }) export class JourneyModule {} diff --git a/server/src/nest/photos/photos-storage-refs.service.ts b/server/src/nest/photos/photos-storage-refs.service.ts new file mode 100644 index 000000000..b8f8ad992 --- /dev/null +++ b/server/src/nest/photos/photos-storage-refs.service.ts @@ -0,0 +1,32 @@ +import { TrekPhotos } from '../../db/entities/TrekPhotos.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects a photo row still names + * (#2924). Every `trek_photos` row with a path is live, whatever its provider + * and whether or not a trip or journey links it: the bytes are served by + * photo id (book pages, public journey shares), and a photo retargeted to a + * remote provider is still read from its local copy. `file_path` and + * `thumbnail_path` (a video's poster) are `journey/` as every writer + * stores them; rows carried over from older tables may hold another + * spelling, so a bare or prefixed name keeps the journey object of that name + * instead of guessing it away. + */ +@Injectable() +export class PhotosStorageRefsService extends StorageReclaimSource { + readonly id = 'photos'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + protected override readonly keyOptions = { bare: 'journey', basename: true } as const; + + protected async storedValues(): Promise> { + const rows = await this.em + .getRepository(TrekPhotos) + .find( + { $or: [{ file_path: { $ne: null } }, { thumbnail_path: { $ne: null } }] }, + { fields: ['file_path', 'thumbnail_path'] }, + ); + return rows.flatMap((row) => [row.file_path, row.thumbnail_path]); + } +} diff --git a/server/src/nest/photos/photos.module.ts b/server/src/nest/photos/photos.module.ts index 69bf0826e..aa79e06ca 100644 --- a/server/src/nest/photos/photos.module.ts +++ b/server/src/nest/photos/photos.module.ts @@ -1,4 +1,6 @@ import { MemoriesModule } from '../memories/memories.module'; +import { StorageModule } from '../storage/storage.module'; +import { PhotosStorageRefsService } from './photos-storage-refs.service'; import { PhotosController } from './photos.controller'; import { PhotosService } from './photos.service'; import { TrekPhotosModule } from './trek-photos.module'; @@ -11,9 +13,9 @@ import { Module } from '@nestjs/common'; * shared leaf is what keeps this a straight line instead of a forwardRef. */ @Module({ - imports: [TrekPhotosModule, MemoriesModule], + imports: [TrekPhotosModule, MemoriesModule, StorageModule], controllers: [PhotosController], - providers: [PhotosService], + providers: [PhotosService, PhotosStorageRefsService], exports: [TrekPhotosModule], }) export class PhotosModule {} diff --git a/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts b/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts new file mode 100644 index 000000000..36da56c41 --- /dev/null +++ b/server/src/nest/place-contrib/place-contrib-storage-refs.service.ts @@ -0,0 +1,22 @@ +import { PlaceContribOutbox } from '../../db/entities/PlaceContribOutbox.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects the contribution queue + * still names (#2924). A queued image value is the place's `image_url` as it + * was when the user set it, kept until the sender reads the bytes, so the + * object must outlive a place that changed or dropped its image meanwhile. + * The value copies a free-form column: every category is claimed. + */ +@Injectable() +export class PlaceContribStorageRefsService extends StorageReclaimSource { + readonly id = 'place-contrib'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + + protected async storedValues(): Promise> { + const rows = await this.em.getRepository(PlaceContribOutbox).find({}, { fields: ['value'] }); + return rows.map((row) => row.value); + } +} diff --git a/server/src/nest/place-contrib/place-contrib.module.ts b/server/src/nest/place-contrib/place-contrib.module.ts index f0fdff980..9c0f51c63 100644 --- a/server/src/nest/place-contrib/place-contrib.module.ts +++ b/server/src/nest/place-contrib/place-contrib.module.ts @@ -8,6 +8,7 @@ import { SettingsModule } from '../settings/settings.module'; import { StorageModule } from '../storage/storage.module'; import { PlaceContribSender } from './place-contrib-sender.service'; import { PlaceContribSettings } from './place-contrib-settings'; +import { PlaceContribStorageRefsService } from './place-contrib-storage-refs.service'; import { PlaceContribClient } from './place-contrib.client'; import { PlaceContribController } from './place-contrib.controller'; import { PlaceContribJob } from './place-contrib.job'; @@ -32,7 +33,14 @@ import { Module } from '@nestjs/common'; MikroOrmModule.forFeature([PlaceContribOutbox, PlaceContribLog]), ], controllers: [PlaceContribController], - providers: [PlaceContribService, PlaceContribSettings, PlaceContribClient, PlaceContribSender, PlaceContribJob], + providers: [ + PlaceContribService, + PlaceContribSettings, + PlaceContribClient, + PlaceContribSender, + PlaceContribJob, + PlaceContribStorageRefsService, + ], exports: [PlaceContribService], }) export class PlaceContribModule {} diff --git a/server/src/nest/places/places-storage-refs.service.ts b/server/src/nest/places/places-storage-refs.service.ts new file mode 100644 index 000000000..739daa7b4 --- /dev/null +++ b/server/src/nest/places/places-storage-refs.service.ts @@ -0,0 +1,23 @@ +import { Places } from '../../db/entities/Places.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects a trip place still names + * (#2924). `places.image_url` admits any `/uploads/` path and any https URL, + * a trip copy and copy-to-trip share it by reference, and a bulk update writes + * one value onto many places, so every category is claimed and every spelling + * is parsed. The photo-proxy paths name the place-photo cache, which keeps + * itself. + */ +@Injectable() +export class PlacesStorageRefsService extends StorageReclaimSource { + readonly id = 'places'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + + protected async storedValues(): Promise> { + const rows = await this.em.getRepository(Places).find({ image_url: { $ne: null } }, { fields: ['image_url'] }); + return rows.map((row) => row.image_url); + } +} diff --git a/server/src/nest/places/places.module.ts b/server/src/nest/places/places.module.ts index 7e485782c..6ecc18660 100644 --- a/server/src/nest/places/places.module.ts +++ b/server/src/nest/places/places.module.ts @@ -28,6 +28,7 @@ import { StorageService } from '../storage/storage.service'; import { TripMembershipModule } from '../trip-membership/trip-membership.module'; import { UnsplashModule } from '../unsplash/unsplash.module'; import { PlaceOriginsService } from './place-origins.service'; +import { PlacesStorageRefsService } from './places-storage-refs.service'; import { PlacesController } from './places.controller'; import { PlacesMcp } from './places.mcp'; import { PlacesRpc } from './places.rpc'; @@ -105,7 +106,7 @@ import { MulterModule } from '@nestjs/platform-express'; ]), ], controllers: [PlacesController], - providers: [PlacesService, PlacesMcp, PlacesRpc, PlaceOriginsService], + providers: [PlacesService, PlacesMcp, PlacesRpc, PlaceOriginsService, PlacesStorageRefsService], // The origin bookkeeping too: the booking import writes places as well. exports: [PlacesService, PlaceOriginsService], }) diff --git a/server/src/nest/storage/storage-reclaim-source.service.ts b/server/src/nest/storage/storage-reclaim-source.service.ts new file mode 100644 index 000000000..09ec2812d --- /dev/null +++ b/server/src/nest/storage/storage-reclaim-source.service.ts @@ -0,0 +1,49 @@ +import { StorageReclaimRegistryService } from './storage-reclaim-registry.service'; +import { + referencedAmong, + type StorageKeyOptions, + type StorageReferenceClaim, + type StorageReferenceSource, + type StorageReferenceSourceId, +} from './storage-reclaim.types'; +import type { ServedCategory } from './storage.types'; +import { EntityManager } from '@mikro-orm/core'; +import { Injectable, type OnApplicationBootstrap } from '@nestjs/common'; + +/** + * The shape every owner's reference source shares: it registers itself with + * the reclaim registry once the app is up, and answers a page of keys by + * reading the values its own columns hold and parsing them with + * `referencedAmong`, so no source carries a parser of its own. + * + * An owner extends this in its own domain, names its id and claims, and reads + * its columns through `em.getRepository()`; storage never + * learns which tables those are. The values are read whole on every call: + * a column that takes any string (an absolute URL of this instance, a path + * with a query, another category's object) cannot be matched with `IN`, and + * a re-check right before a delete must see the rows as they are now. + */ +@Injectable() +export abstract class StorageReclaimSource implements StorageReferenceSource, OnApplicationBootstrap { + abstract readonly id: StorageReferenceSourceId; + abstract readonly claims: ReadonlyArray; + /** How the values name objects besides the `/uploads//` and `/` spellings. */ + protected readonly keyOptions: StorageKeyOptions = {}; + + constructor( + private readonly registry: StorageReclaimRegistryService, + protected readonly em: EntityManager, + ) {} + + onApplicationBootstrap(): void { + this.registry.register(this); + } + + async referenced(category: ServedCategory, keys: readonly string[]): Promise> { + if (keys.length === 0 || !this.claims.some((claim) => claim.category === category)) return new Set(); + return referencedAmong(category, keys, await this.storedValues(), this.keyOptions); + } + + /** Every value the owner's columns hold that may name a stored object; null and empty ones may be among them. */ + protected abstract storedValues(): Promise>; +} diff --git a/server/src/nest/storage/storage-reclaim.ledger.ts b/server/src/nest/storage/storage-reclaim.ledger.ts index bdde765c0..36b0b7747 100644 --- a/server/src/nest/storage/storage-reclaim.ledger.ts +++ b/server/src/nest/storage/storage-reclaim.ledger.ts @@ -9,7 +9,14 @@ export type ReclaimRule = | { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] }; /** The owners of columns that hold a free-form upload path, which can point into any category. */ -const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos']; +const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = [ + 'trips', + 'collections', + 'places', + 'place-contrib', + 'journey', + 'photos', +]; /** * What the reclaim pass may do per category, and which reference sources must @@ -27,7 +34,8 @@ export const RECLAIM_LEDGER: Readonly> = { // Trip, collection and journey covers; a trip copy and a journey share one object. covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES }, avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] }, - // places.image_url, collection_places.image_url and the contribution outbox. + // places.image_url, collection_places.image_url and the contribution outbox + // (place-contrib's queue holds an image path until it is sent). places: { reclaim: true, requiredSources: FREE_FORM_SOURCES }, photos: { reclaim: false, diff --git a/server/src/nest/storage/storage-reclaim.types.ts b/server/src/nest/storage/storage-reclaim.types.ts index ee2fc3806..aef5f4f32 100644 --- a/server/src/nest/storage/storage-reclaim.types.ts +++ b/server/src/nest/storage/storage-reclaim.types.ts @@ -16,6 +16,7 @@ export const STORAGE_REFERENCE_SOURCE_IDS = [ 'trips', 'collections', 'places', + 'place-contrib', 'journey', 'photos', 'files', @@ -38,6 +39,11 @@ export interface StorageReferenceClaim { readonly prefix: string; } +/** What a source whose columns hold free-form upload paths claims: the whole of every reclaimable category. */ +export const EVERY_RECLAIMABLE_CATEGORY: readonly StorageReferenceClaim[] = STORAGE_RECLAIMABLE_CATEGORIES.map( + (category) => ({ category, prefix: '' }), +); + export interface StorageReferenceSource { readonly id: StorageReferenceSourceId; /** diff --git a/server/src/nest/trips/trips-storage-refs.service.ts b/server/src/nest/trips/trips-storage-refs.service.ts new file mode 100644 index 000000000..c1e9a5441 --- /dev/null +++ b/server/src/nest/trips/trips-storage-refs.service.ts @@ -0,0 +1,22 @@ +import { Trips } from '../../db/entities/Trips.entity'; +import { StorageReclaimSource } from '../storage/storage-reclaim-source.service'; +import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types'; +import { Injectable } from '@nestjs/common'; + +/** + * Tells the storage reclaim pass which stored objects a trip still names + * (#2924). `trips.cover_image` is whatever the client sent: usually + * `/uploads/covers/`, but just as well an absolute URL of this instance + * or a path into another category, and a trip copy shares its source's cover. + * So every category is claimed and every spelling is parsed. + */ +@Injectable() +export class TripsStorageRefsService extends StorageReclaimSource { + readonly id = 'trips'; + readonly claims = EVERY_RECLAIMABLE_CATEGORY; + + protected async storedValues(): Promise> { + const rows = await this.em.getRepository(Trips).find({ cover_image: { $ne: null } }, { fields: ['cover_image'] }); + return rows.map((row) => row.cover_image); + } +} diff --git a/server/src/nest/trips/trips.module.ts b/server/src/nest/trips/trips.module.ts index 08c8e9832..cda8f003a 100644 --- a/server/src/nest/trips/trips.module.ts +++ b/server/src/nest/trips/trips.module.ts @@ -26,6 +26,7 @@ import { UnsplashModule } from '../unsplash/unsplash.module'; import { VacayModule } from '../vacay/vacay.module'; import { TripPromptsMcp } from './trip-prompts.mcp'; import { TripWritesService } from './trip-writes.service'; +import { TripsStorageRefsService } from './trips-storage-refs.service'; import { TripsController } from './trips.controller'; import { MAX_COVER_SIZE, TRIP_COVER_FILE_FILTER } from './trips.controller'; import { TripsMcp } from './trips.mcp'; @@ -74,7 +75,7 @@ import { MulterModule } from '@nestjs/platform-express'; SettingsModule, ], controllers: [TripsController], - providers: [TripsService, TripWritesService, TripsMcp, TripPromptsMcp, TripsRpc], + providers: [TripsService, TripWritesService, TripsMcp, TripPromptsMcp, TripsRpc, TripsStorageRefsService], // Exported for FeedsModule (ICS feeds) and PluginsModule (RPC host injection). exports: [TripsService], }) diff --git a/server/tests/unit/nest/storage-reference-sources.test.ts b/server/tests/unit/nest/storage-reference-sources.test.ts new file mode 100644 index 000000000..483779c86 --- /dev/null +++ b/server/tests/unit/nest/storage-reference-sources.test.ts @@ -0,0 +1,200 @@ +/** + * The reference sources the owner domains hand the storage reclaim pass + * (#2924), each against the migrated schema: which of a page of keys its rows + * still name, in every spelling those columns hold. One object kept too many + * costs a few bytes; one called unreferenced while a row names it is lost, so + * every case here is a spelling a real row can carry. RSRC-001 to RSRC-011. + */ +import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity'; +import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity'; +import { Users } from '../../../src/db/entities/Users.entity'; +import { AuthStorageRefsService } from '../../../src/nest/auth/auth-storage-refs.service'; +import { CollectionsStorageRefsService } from '../../../src/nest/collections/collections-storage-refs.service'; +import { FilesStorageRefsService } from '../../../src/nest/files/files-storage-refs.service'; +import { JourneyStorageRefsService } from '../../../src/nest/journey/journey-storage-refs.service'; +import { PhotosStorageRefsService } from '../../../src/nest/photos/photos-storage-refs.service'; +import { PlaceContribStorageRefsService } from '../../../src/nest/place-contrib/place-contrib-storage-refs.service'; +import { PlacesStorageRefsService } from '../../../src/nest/places/places-storage-refs.service'; +import { StorageReclaimRegistryService } from '../../../src/nest/storage/storage-reclaim-registry.service'; +import { + STORAGE_RECLAIMABLE_CATEGORIES, + STORAGE_REFERENCE_SOURCE_IDS, + type StorageReferenceSource, +} from '../../../src/nest/storage/storage-reclaim.types'; +import type { ServedCategory } from '../../../src/nest/storage/storage.types'; +import { TripsStorageRefsService } from '../../../src/nest/trips/trips-storage-refs.service'; +import { createSnapshotTestDb } from '../../helpers/db-mock'; +import { makeCollection, makeCollectionPlace } from '../../helpers/factories/collections'; +import { makeTripFile } from '../../helpers/factories/files'; +import { makeJourney } from '../../helpers/factories/journeys'; +import { makePlace } from '../../helpers/factories/places'; +import { insertRow, updateRows } from '../../helpers/factories/rows'; +import { makeTrip } from '../../helpers/factories/trips'; +import { makeUser } from '../../helpers/factories/users'; +import { createTestModule, type TestModule } from '../../helpers/test-module'; +import { createTestOrm, type TestOrm } from '../../helpers/test-orm'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const SOURCES = [ + TripsStorageRefsService, + CollectionsStorageRefsService, + PlacesStorageRefsService, + PlaceContribStorageRefsService, + JourneyStorageRefsService, + PhotosStorageRefsService, + FilesStorageRefsService, + AuthStorageRefsService, +]; + +const db = createSnapshotTestDb(); +let orm: TestOrm; +let t: TestModule; +let userId: number; +let tripId: number; + +beforeAll(async () => { + orm = await createTestOrm(db); + t = await createTestModule({ db, imports: [], providers: [StorageReclaimRegistryService, ...SOURCES], init: true }); + userId = (await makeUser(orm)).user.id; + tripId = (await makeTrip(orm, userId)).id; +}); + +afterAll(async () => { + await t.close(); + await orm.close(); + db.close(); +}); + +/** The sorted keys a source still references. */ +async function asked(source: StorageReferenceSource, category: ServedCategory, keys: string[]): Promise { + return [...(await source.referenced(category, keys))].sort(); +} + +describe('reference sources', () => { + it('RSRC-001: every source registers itself once the app boots, under the ids the ledger knows', () => { + const registry = t.get(StorageReclaimRegistryService); + expect([...registry.ids()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort()); + for (const source of registry.sources()) { + const claimed = source.claims.map((claim) => `${claim.category}:${claim.prefix}`).sort(); + if (source.id === 'files') expect(claimed).toEqual(['files:']); + else if (source.id === 'auth') expect(claimed).toEqual(['avatars:']); + else expect(claimed).toEqual(STORAGE_RECLAIMABLE_CATEGORIES.map((category) => `${category}:`).sort()); + } + }); + + it('RSRC-002: a trip cover keeps its object in every spelling, and in whatever category it points into', async () => { + await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-a.jpg' }); + await makeTrip(orm, userId, { cover_image: 'https://trip.example.test/uploads/places/rsrc-b.jpg?v=2' }); + await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Mixed.JPG' }); + await makeTrip(orm, userId, { cover_image: 'https://images.unsplash.com/rsrc-z.jpg' }); + const trips = t.get(TripsStorageRefsService); + expect(await asked(trips, 'covers', ['rsrc-a.jpg', 'rsrc-mixed.jpg', 'rsrc-z.jpg', 'rsrc-b.jpg'])).toEqual([ + 'rsrc-a.jpg', + 'rsrc-mixed.jpg', + ]); + expect(await asked(trips, 'places', ['rsrc-b.jpg', 'rsrc-a.jpg'])).toEqual(['rsrc-b.jpg']); + expect(await asked(trips, 'covers', [])).toEqual([]); + }); + + it('RSRC-003: a list cover and a saved place image both keep their objects', async () => { + const collection = await makeCollection(orm, userId, { cover_image: '/uploads/covers/rsrc-c.jpg' }); + await makeCollectionPlace(orm, collection.id, userId, { image_url: '/uploads/places/rsrc-d.jpg' }); + await makeCollectionPlace(orm, collection.id, userId, { image_url: 'https://example.test/rsrc-y.jpg' }); + const collections = t.get(CollectionsStorageRefsService); + expect(await asked(collections, 'covers', ['rsrc-c.jpg', 'rsrc-d.jpg'])).toEqual(['rsrc-c.jpg']); + expect(await asked(collections, 'places', ['rsrc-d.jpg', 'rsrc-y.jpg'])).toEqual(['rsrc-d.jpg']); + }); + + it('RSRC-004: a place image keeps its object; a photo-proxy path or an inline image names none', async () => { + await makePlace(orm, tripId, { image_url: '/uploads/places/rsrc-e.jpg' }); + await makePlace(orm, tripId, { image_url: '/uploads/covers/rsrc-e2.jpg' }); + await makePlace(orm, tripId, { image_url: '/api/maps/place-photo/rsrc-x/bytes' }); + await makePlace(orm, tripId, { image_url: 'data:image/png;base64,AAAA' }); + const places = t.get(PlacesStorageRefsService); + expect(await asked(places, 'places', ['rsrc-e.jpg', 'rsrc-x', 'bytes'])).toEqual(['rsrc-e.jpg']); + expect(await asked(places, 'covers', ['rsrc-e2.jpg'])).toEqual(['rsrc-e2.jpg']); + }); + + it('RSRC-005: a queued contribution keeps the image it has yet to send', async () => { + const place = await makePlace(orm, tripId, { image_url: null }); + await insertRow(orm, PlaceContribOutbox, { + contrib_key: 'osm:node/2924', + field: 'image', + value: '/uploads/places/rsrc-f.jpg', + place: place.id, + place_name: 'Queued', + cid: 'cid-rsrc-0001', + eligible_at: '2026-01-01 00:00:00', + next_attempt_at: '2026-01-01 00:00:00', + }); + const outbox = t.get(PlaceContribStorageRefsService); + expect(await asked(outbox, 'places', ['rsrc-f.jpg', 'rsrc-f2.jpg'])).toEqual(['rsrc-f.jpg']); + }); + + it('RSRC-006: a journey cover keeps a shared trip cover and a gallery photo by their relative spelling', async () => { + await makeJourney(orm, userId, { cover_image: 'covers/rsrc-g.jpg' }); + await makeJourney(orm, userId, { cover_image: 'journey/rsrc-h.jpg' }); + await makeJourney(orm, userId, { cover_image: '/uploads/covers/rsrc-g2.jpg' }); + const journeys = t.get(JourneyStorageRefsService); + expect(await asked(journeys, 'covers', ['rsrc-g.jpg', 'rsrc-g2.jpg', 'rsrc-h.jpg'])).toEqual([ + 'rsrc-g.jpg', + 'rsrc-g2.jpg', + ]); + expect(await asked(journeys, 'journey', ['rsrc-h.jpg', 'rsrc-g.jpg'])).toEqual(['rsrc-h.jpg']); + }); + + it('RSRC-007: every photo row with a path keeps it, whatever its provider and links', async () => { + // A local photo retargeted to a remote provider is still served from its bytes. + await insertRow(orm, TrekPhotos, { + provider: 'immich', + asset_id: 'rsrc-asset', + owner: userId, + file_path: 'journey/rsrc-i.jpg', + thumbnail_path: 'journey/rsrc-i-poster.jpg', + }); + await insertRow(orm, TrekPhotos, { provider: 'local', owner: userId, file_path: 'legacy/rsrc-j.jpg' }); + await insertRow(orm, TrekPhotos, { provider: 'immich', asset_id: 'rsrc-remote', owner: userId }); + const photos = t.get(PhotosStorageRefsService); + expect( + await asked(photos, 'journey', ['rsrc-i.jpg', 'rsrc-i-poster.jpg', 'rsrc-j.jpg', 'rsrc-remote', 'rsrc-k.jpg']), + ).toEqual(['rsrc-i-poster.jpg', 'rsrc-i.jpg', 'rsrc-j.jpg']); + expect(await asked(photos, 'covers', ['rsrc-i.jpg'])).toEqual([]); + }); + + it('RSRC-008: a trip file keeps its object, trashed or carrying a legacy path', async () => { + await makeTripFile(orm, tripId, { filename: 'rsrc-k.pdf' }); + await makeTripFile(orm, tripId, { filename: 'rsrc-l.pdf', deleted_at: '2026-01-01 00:00:00' }); + await makeTripFile(orm, tripId, { filename: 'files/rsrc-m.pdf' }); + await makeTripFile(orm, tripId, { filename: '/uploads/files/rsrc-n.pdf' }); + await makeTripFile(orm, tripId, { filename: 'uploads\\files\\rsrc-p.pdf' }); + const files = t.get(FilesStorageRefsService); + expect( + await asked(files, 'files', ['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf', 'rsrc-o.pdf']), + ).toEqual(['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf']); + }); + + it('RSRC-009: a source asked about a category it does not claim answers nothing', async () => { + await makeTripFile(orm, tripId, { filename: 'rsrc-q.jpg' }); + expect(await asked(t.get(FilesStorageRefsService), 'covers', ['rsrc-q.jpg'])).toEqual([]); + expect(await asked(t.get(FilesStorageRefsService), 'files', [])).toEqual([]); + expect(await asked(t.get(AuthStorageRefsService), 'files', ['rsrc-q.jpg'])).toEqual([]); + }); + + it('RSRC-010: an uploaded avatar keeps its object; an OIDC picture names none', async () => { + const uploaded = (await makeUser(orm)).user; + const oidc = (await makeUser(orm)).user; + await updateRows(orm, Users, { id: uploaded.id }, { avatar: 'rsrc-r.png' }); + await updateRows(orm, Users, { id: oidc.id }, { avatar: 'https://idp.example.test/pictures/rsrc-s.png' }); + const auth = t.get(AuthStorageRefsService); + expect(await asked(auth, 'avatars', ['rsrc-r.png', 'rsrc-s.png'])).toEqual(['rsrc-r.png']); + expect(await asked(auth, 'avatars', [])).toEqual([]); + }); + + it('RSRC-011: every source asked with no keys answers nothing', async () => { + const answers = await Promise.all( + SOURCES.map((Source) => asked(t.get(Source), 'covers', [])), + ); + expect(answers).toEqual(SOURCES.map(() => [])); + }); +}); diff --git a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts index b40d4ee9a..455ee22a0 100644 --- a/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts +++ b/server/tests/unit/nest/storage/storage-reclaim.ledger.test.ts @@ -63,7 +63,7 @@ describe('RECLAIM_LEDGER', () => { }); it('RCLL-002: every reclaimable category waits for all the free-form owners, files and avatars for their own too', () => { - const freeForm = ['trips', 'collections', 'places', 'journey', 'photos']; + const freeForm = ['trips', 'collections', 'places', 'place-contrib', 'journey', 'photos']; for (const category of STORAGE_RECLAIMABLE_CATEGORIES) { const rule = RECLAIM_LEDGER[category]; expect(rule.reclaim).toBe(true);