mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-14 00:01:03 +00:00
feat(storage): let every owner of an upload column tell the reclaim pass what it still names
Trips, collections, places, the contribution queue, journeys, photos, trip files and users each register a reference source from their own domain. A source reads its own columns and parses every spelling they hold through the storage helpers, and the free-form columns claim every reclaimable category, since a cover or an image can point into any of them. The contribution queue joins the ledger as its own source, because place-contrib owns that table.
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
import { Users } from '../../db/entities/Users.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which avatar objects a user row still names
|
||||
* (#2924). `users.avatar` holds the bare object name of an uploaded avatar;
|
||||
* an https picture taken from the OIDC claim names no object of ours and
|
||||
* parses as nothing.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AuthStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'auth';
|
||||
readonly claims = [{ category: 'avatars', prefix: '' }] as const;
|
||||
protected override readonly keyOptions = { bare: 'avatars' } as const;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em.getRepository(Users).find({ avatar: { $ne: null } }, { fields: ['avatar'] });
|
||||
return rows.map((row) => row.avatar);
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ import { StorageService } from '../storage/storage.service';
|
||||
import { TokensModule } from '../tokens/tokens.module';
|
||||
import { TripMembershipModule } from '../trip-membership/trip-membership.module';
|
||||
import { AuthPublicController } from './auth-public.controller';
|
||||
import { AuthStorageRefsService } from './auth-storage-refs.service';
|
||||
import { AuthController } from './auth.controller';
|
||||
import { AVATAR_FILE_FILTER, MAX_AVATAR_SIZE } from './auth.controller';
|
||||
import { AuthMcp } from './auth.mcp';
|
||||
@@ -163,6 +164,7 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
UserCleanupService,
|
||||
WebauthnConfigService,
|
||||
AuthMcp,
|
||||
AuthStorageRefsService,
|
||||
],
|
||||
exports: [AuthService, RegistrationInvitesService, PasskeyService, UserCleanupService],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { CollectionPlaces } from '../../db/entities/CollectionPlaces.entity';
|
||||
import { Collections } from '../../db/entities/Collections.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects a collection still names
|
||||
* (#2924): the list's own cover and every saved place's image. Saving a place
|
||||
* into a list and copying it to a trip share the uploaded image by reference,
|
||||
* and both columns take any string the client sends, so every category is
|
||||
* claimed and every spelling is parsed.
|
||||
*/
|
||||
@Injectable()
|
||||
export class CollectionsStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'collections';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const covers = await this.em
|
||||
.getRepository(Collections)
|
||||
.find({ cover_image: { $ne: null } }, { fields: ['cover_image'] });
|
||||
const images = await this.em
|
||||
.getRepository(CollectionPlaces)
|
||||
.find({ image_url: { $ne: null } }, { fields: ['image_url'] });
|
||||
return [...covers.map((row) => row.cover_image), ...images.map((row) => row.image_url)];
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { StorageService } from '../storage/storage.service';
|
||||
import { CollectionsItemsService } from './collections-items.service';
|
||||
import { CollectionsStorageRefsService } from './collections-storage-refs.service';
|
||||
import { CollectionsController } from './collections.controller';
|
||||
import { MAX_COVER_SIZE } from './collections.controller';
|
||||
import { CollectionsMcp } from './collections.mcp';
|
||||
@@ -77,7 +78,13 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
]),
|
||||
],
|
||||
controllers: [CollectionsController],
|
||||
providers: [CollectionsService, CollectionsItemsService, CollectionsMcp, CollectionsRpc],
|
||||
providers: [
|
||||
CollectionsService,
|
||||
CollectionsItemsService,
|
||||
CollectionsMcp,
|
||||
CollectionsRpc,
|
||||
CollectionsStorageRefsService,
|
||||
],
|
||||
exports: [CollectionsService],
|
||||
})
|
||||
export class CollectionsModule {}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { TripFiles } from '../../db/entities/TripFiles.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which objects of the files category a trip
|
||||
* file row still names (#2924). `trip_files.filename` is the bare object name;
|
||||
* rows from before the storage layer may still carry a path in front of it,
|
||||
* which every reader strips with basename(), so the last segment counts too.
|
||||
* Trashed rows count: a restore brings the file back, and doc-sync keeps its
|
||||
* superseded copies in the trash on purpose. Collab note files and chat images
|
||||
* are trip_files rows as well. The pass only ever asks about the category's
|
||||
* root (the assistant's subfolder has a sweep of its own), so a basename never
|
||||
* reaches into a subfolder.
|
||||
*/
|
||||
@Injectable()
|
||||
export class FilesStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'files';
|
||||
readonly claims = [{ category: 'files', prefix: '' }] as const;
|
||||
protected override readonly keyOptions = { bare: 'files', basename: true } as const;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em.getRepository(TripFiles).find({}, { fields: ['filename'] });
|
||||
return rows.map((row) => row.filename);
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import { TripMembershipModule } from '../trip-membership/trip-membership.module'
|
||||
import { AllowedFileTypesModule } from './allowed-file-types.module';
|
||||
import { AllowedFileTypesService } from './allowed-file-types.service';
|
||||
import { FilesDownloadController } from './files-download.controller';
|
||||
import { FilesStorageRefsService } from './files-storage-refs.service';
|
||||
import { MAX_FILE_SIZE, MAX_VIDEO_SIZE } from './files.constants';
|
||||
import { FilesController } from './files.controller';
|
||||
import { filesUploadFileFilter } from './files.controller';
|
||||
@@ -65,7 +66,7 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
AllowedFileTypesModule,
|
||||
],
|
||||
controllers: [FilesController, FilesDownloadController],
|
||||
providers: [FilesService, FilesRpc, FilesMcp],
|
||||
providers: [FilesService, FilesRpc, FilesMcp, FilesStorageRefsService],
|
||||
exports: [FilesService],
|
||||
})
|
||||
export class FilesModule {}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Journeys } from '../../db/entities/Journeys.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects a journey still names
|
||||
* (#2924). `journeys.cover_image` is inherited from the first linked trip as
|
||||
* `covers/<name>` (the trip and the journey share the object), set to a
|
||||
* gallery photo's `journey/<name>`, or whatever a PATCH sent. Every category
|
||||
* is claimed; the photos themselves are the photos domain's rows.
|
||||
*/
|
||||
@Injectable()
|
||||
export class JourneyStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'journey';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em
|
||||
.getRepository(Journeys)
|
||||
.find({ cover_image: { $ne: null } }, { fields: ['cover_image'] });
|
||||
return rows.map((row) => row.cover_image);
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import { JourneyMapMcp } from './journey-map.mcp';
|
||||
import { JourneyPhotoCaptureModule } from './journey-photo-capture.module';
|
||||
import { JourneyPhotoLinksMcp } from './journey-photos.mcp';
|
||||
import { JourneyPublicController } from './journey-public.controller';
|
||||
import { JourneyStorageRefsService } from './journey-storage-refs.service';
|
||||
import { JourneyTracksMcp } from './journey-tracks.mcp';
|
||||
import { JourneyController } from './journey.controller';
|
||||
import { journeyImageFileFilter, journeyUploadFilename } from './journey.controller';
|
||||
@@ -62,6 +63,13 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
MikroOrmModule.forFeature([Users]),
|
||||
],
|
||||
controllers: [JourneyController, JourneyMapController, JourneyPublicController],
|
||||
providers: [JourneyService, JourneyMcp, JourneyMapMcp, JourneyTracksMcp, JourneyPhotoLinksMcp],
|
||||
providers: [
|
||||
JourneyService,
|
||||
JourneyMcp,
|
||||
JourneyMapMcp,
|
||||
JourneyTracksMcp,
|
||||
JourneyPhotoLinksMcp,
|
||||
JourneyStorageRefsService,
|
||||
],
|
||||
})
|
||||
export class JourneyModule {}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { TrekPhotos } from '../../db/entities/TrekPhotos.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects a photo row still names
|
||||
* (#2924). Every `trek_photos` row with a path is live, whatever its provider
|
||||
* and whether or not a trip or journey links it: the bytes are served by
|
||||
* photo id (book pages, public journey shares), and a photo retargeted to a
|
||||
* remote provider is still read from its local copy. `file_path` and
|
||||
* `thumbnail_path` (a video's poster) are `journey/<name>` as every writer
|
||||
* stores them; rows carried over from older tables may hold another
|
||||
* spelling, so a bare or prefixed name keeps the journey object of that name
|
||||
* instead of guessing it away.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PhotosStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'photos';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
protected override readonly keyOptions = { bare: 'journey', basename: true } as const;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em
|
||||
.getRepository(TrekPhotos)
|
||||
.find(
|
||||
{ $or: [{ file_path: { $ne: null } }, { thumbnail_path: { $ne: null } }] },
|
||||
{ fields: ['file_path', 'thumbnail_path'] },
|
||||
);
|
||||
return rows.flatMap((row) => [row.file_path, row.thumbnail_path]);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { MemoriesModule } from '../memories/memories.module';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { PhotosStorageRefsService } from './photos-storage-refs.service';
|
||||
import { PhotosController } from './photos.controller';
|
||||
import { PhotosService } from './photos.service';
|
||||
import { TrekPhotosModule } from './trek-photos.module';
|
||||
@@ -11,9 +13,9 @@ import { Module } from '@nestjs/common';
|
||||
* shared leaf is what keeps this a straight line instead of a forwardRef.
|
||||
*/
|
||||
@Module({
|
||||
imports: [TrekPhotosModule, MemoriesModule],
|
||||
imports: [TrekPhotosModule, MemoriesModule, StorageModule],
|
||||
controllers: [PhotosController],
|
||||
providers: [PhotosService],
|
||||
providers: [PhotosService, PhotosStorageRefsService],
|
||||
exports: [TrekPhotosModule],
|
||||
})
|
||||
export class PhotosModule {}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { PlaceContribOutbox } from '../../db/entities/PlaceContribOutbox.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects the contribution queue
|
||||
* still names (#2924). A queued image value is the place's `image_url` as it
|
||||
* was when the user set it, kept until the sender reads the bytes, so the
|
||||
* object must outlive a place that changed or dropped its image meanwhile.
|
||||
* The value copies a free-form column: every category is claimed.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PlaceContribStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'place-contrib';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em.getRepository(PlaceContribOutbox).find({}, { fields: ['value'] });
|
||||
return rows.map((row) => row.value);
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import { SettingsModule } from '../settings/settings.module';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { PlaceContribSender } from './place-contrib-sender.service';
|
||||
import { PlaceContribSettings } from './place-contrib-settings';
|
||||
import { PlaceContribStorageRefsService } from './place-contrib-storage-refs.service';
|
||||
import { PlaceContribClient } from './place-contrib.client';
|
||||
import { PlaceContribController } from './place-contrib.controller';
|
||||
import { PlaceContribJob } from './place-contrib.job';
|
||||
@@ -32,7 +33,14 @@ import { Module } from '@nestjs/common';
|
||||
MikroOrmModule.forFeature([PlaceContribOutbox, PlaceContribLog]),
|
||||
],
|
||||
controllers: [PlaceContribController],
|
||||
providers: [PlaceContribService, PlaceContribSettings, PlaceContribClient, PlaceContribSender, PlaceContribJob],
|
||||
providers: [
|
||||
PlaceContribService,
|
||||
PlaceContribSettings,
|
||||
PlaceContribClient,
|
||||
PlaceContribSender,
|
||||
PlaceContribJob,
|
||||
PlaceContribStorageRefsService,
|
||||
],
|
||||
exports: [PlaceContribService],
|
||||
})
|
||||
export class PlaceContribModule {}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { Places } from '../../db/entities/Places.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects a trip place still names
|
||||
* (#2924). `places.image_url` admits any `/uploads/` path and any https URL,
|
||||
* a trip copy and copy-to-trip share it by reference, and a bulk update writes
|
||||
* one value onto many places, so every category is claimed and every spelling
|
||||
* is parsed. The photo-proxy paths name the place-photo cache, which keeps
|
||||
* itself.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PlacesStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'places';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em.getRepository(Places).find({ image_url: { $ne: null } }, { fields: ['image_url'] });
|
||||
return rows.map((row) => row.image_url);
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import { StorageService } from '../storage/storage.service';
|
||||
import { TripMembershipModule } from '../trip-membership/trip-membership.module';
|
||||
import { UnsplashModule } from '../unsplash/unsplash.module';
|
||||
import { PlaceOriginsService } from './place-origins.service';
|
||||
import { PlacesStorageRefsService } from './places-storage-refs.service';
|
||||
import { PlacesController } from './places.controller';
|
||||
import { PlacesMcp } from './places.mcp';
|
||||
import { PlacesRpc } from './places.rpc';
|
||||
@@ -105,7 +106,7 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
]),
|
||||
],
|
||||
controllers: [PlacesController],
|
||||
providers: [PlacesService, PlacesMcp, PlacesRpc, PlaceOriginsService],
|
||||
providers: [PlacesService, PlacesMcp, PlacesRpc, PlaceOriginsService, PlacesStorageRefsService],
|
||||
// The origin bookkeeping too: the booking import writes places as well.
|
||||
exports: [PlacesService, PlaceOriginsService],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
|
||||
import {
|
||||
referencedAmong,
|
||||
type StorageKeyOptions,
|
||||
type StorageReferenceClaim,
|
||||
type StorageReferenceSource,
|
||||
type StorageReferenceSourceId,
|
||||
} from './storage-reclaim.types';
|
||||
import type { ServedCategory } from './storage.types';
|
||||
import { EntityManager } from '@mikro-orm/core';
|
||||
import { Injectable, type OnApplicationBootstrap } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* The shape every owner's reference source shares: it registers itself with
|
||||
* the reclaim registry once the app is up, and answers a page of keys by
|
||||
* reading the values its own columns hold and parsing them with
|
||||
* `referencedAmong`, so no source carries a parser of its own.
|
||||
*
|
||||
* An owner extends this in its own domain, names its id and claims, and reads
|
||||
* its columns through `em.getRepository(<its own entity>)`; storage never
|
||||
* learns which tables those are. The values are read whole on every call:
|
||||
* a column that takes any string (an absolute URL of this instance, a path
|
||||
* with a query, another category's object) cannot be matched with `IN`, and
|
||||
* a re-check right before a delete must see the rows as they are now.
|
||||
*/
|
||||
@Injectable()
|
||||
export abstract class StorageReclaimSource implements StorageReferenceSource, OnApplicationBootstrap {
|
||||
abstract readonly id: StorageReferenceSourceId;
|
||||
abstract readonly claims: ReadonlyArray<StorageReferenceClaim>;
|
||||
/** How the values name objects besides the `/uploads/<category>/<key>` and `<category>/<key>` spellings. */
|
||||
protected readonly keyOptions: StorageKeyOptions = {};
|
||||
|
||||
constructor(
|
||||
private readonly registry: StorageReclaimRegistryService,
|
||||
protected readonly em: EntityManager,
|
||||
) {}
|
||||
|
||||
onApplicationBootstrap(): void {
|
||||
this.registry.register(this);
|
||||
}
|
||||
|
||||
async referenced(category: ServedCategory, keys: readonly string[]): Promise<Set<string>> {
|
||||
if (keys.length === 0 || !this.claims.some((claim) => claim.category === category)) return new Set();
|
||||
return referencedAmong(category, keys, await this.storedValues(), this.keyOptions);
|
||||
}
|
||||
|
||||
/** Every value the owner's columns hold that may name a stored object; null and empty ones may be among them. */
|
||||
protected abstract storedValues(): Promise<Iterable<string | null | undefined>>;
|
||||
}
|
||||
@@ -9,7 +9,14 @@ export type ReclaimRule =
|
||||
| { readonly reclaim: true; readonly requiredSources: readonly StorageReferenceSourceId[] };
|
||||
|
||||
/** The owners of columns that hold a free-form upload path, which can point into any category. */
|
||||
const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = ['trips', 'collections', 'places', 'journey', 'photos'];
|
||||
const FREE_FORM_SOURCES: readonly StorageReferenceSourceId[] = [
|
||||
'trips',
|
||||
'collections',
|
||||
'places',
|
||||
'place-contrib',
|
||||
'journey',
|
||||
'photos',
|
||||
];
|
||||
|
||||
/**
|
||||
* What the reclaim pass may do per category, and which reference sources must
|
||||
@@ -27,7 +34,8 @@ export const RECLAIM_LEDGER: Readonly<Record<ServedCategory, ReclaimRule>> = {
|
||||
// Trip, collection and journey covers; a trip copy and a journey share one object.
|
||||
covers: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
avatars: { reclaim: true, requiredSources: ['auth', ...FREE_FORM_SOURCES] },
|
||||
// places.image_url, collection_places.image_url and the contribution outbox.
|
||||
// places.image_url, collection_places.image_url and the contribution outbox
|
||||
// (place-contrib's queue holds an image path until it is sent).
|
||||
places: { reclaim: true, requiredSources: FREE_FORM_SOURCES },
|
||||
photos: {
|
||||
reclaim: false,
|
||||
|
||||
@@ -16,6 +16,7 @@ export const STORAGE_REFERENCE_SOURCE_IDS = [
|
||||
'trips',
|
||||
'collections',
|
||||
'places',
|
||||
'place-contrib',
|
||||
'journey',
|
||||
'photos',
|
||||
'files',
|
||||
@@ -38,6 +39,11 @@ export interface StorageReferenceClaim {
|
||||
readonly prefix: string;
|
||||
}
|
||||
|
||||
/** What a source whose columns hold free-form upload paths claims: the whole of every reclaimable category. */
|
||||
export const EVERY_RECLAIMABLE_CATEGORY: readonly StorageReferenceClaim[] = STORAGE_RECLAIMABLE_CATEGORIES.map(
|
||||
(category) => ({ category, prefix: '' }),
|
||||
);
|
||||
|
||||
export interface StorageReferenceSource {
|
||||
readonly id: StorageReferenceSourceId;
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { Trips } from '../../db/entities/Trips.entity';
|
||||
import { StorageReclaimSource } from '../storage/storage-reclaim-source.service';
|
||||
import { EVERY_RECLAIMABLE_CATEGORY } from '../storage/storage-reclaim.types';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* Tells the storage reclaim pass which stored objects a trip still names
|
||||
* (#2924). `trips.cover_image` is whatever the client sent: usually
|
||||
* `/uploads/covers/<name>`, but just as well an absolute URL of this instance
|
||||
* or a path into another category, and a trip copy shares its source's cover.
|
||||
* So every category is claimed and every spelling is parsed.
|
||||
*/
|
||||
@Injectable()
|
||||
export class TripsStorageRefsService extends StorageReclaimSource {
|
||||
readonly id = 'trips';
|
||||
readonly claims = EVERY_RECLAIMABLE_CATEGORY;
|
||||
|
||||
protected async storedValues(): Promise<Array<string | null | undefined>> {
|
||||
const rows = await this.em.getRepository(Trips).find({ cover_image: { $ne: null } }, { fields: ['cover_image'] });
|
||||
return rows.map((row) => row.cover_image);
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import { UnsplashModule } from '../unsplash/unsplash.module';
|
||||
import { VacayModule } from '../vacay/vacay.module';
|
||||
import { TripPromptsMcp } from './trip-prompts.mcp';
|
||||
import { TripWritesService } from './trip-writes.service';
|
||||
import { TripsStorageRefsService } from './trips-storage-refs.service';
|
||||
import { TripsController } from './trips.controller';
|
||||
import { MAX_COVER_SIZE, TRIP_COVER_FILE_FILTER } from './trips.controller';
|
||||
import { TripsMcp } from './trips.mcp';
|
||||
@@ -74,7 +75,7 @@ import { MulterModule } from '@nestjs/platform-express';
|
||||
SettingsModule,
|
||||
],
|
||||
controllers: [TripsController],
|
||||
providers: [TripsService, TripWritesService, TripsMcp, TripPromptsMcp, TripsRpc],
|
||||
providers: [TripsService, TripWritesService, TripsMcp, TripPromptsMcp, TripsRpc, TripsStorageRefsService],
|
||||
// Exported for FeedsModule (ICS feeds) and PluginsModule (RPC host injection).
|
||||
exports: [TripsService],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* The reference sources the owner domains hand the storage reclaim pass
|
||||
* (#2924), each against the migrated schema: which of a page of keys its rows
|
||||
* still name, in every spelling those columns hold. One object kept too many
|
||||
* costs a few bytes; one called unreferenced while a row names it is lost, so
|
||||
* every case here is a spelling a real row can carry. RSRC-001 to RSRC-011.
|
||||
*/
|
||||
import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity';
|
||||
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
|
||||
import { Users } from '../../../src/db/entities/Users.entity';
|
||||
import { AuthStorageRefsService } from '../../../src/nest/auth/auth-storage-refs.service';
|
||||
import { CollectionsStorageRefsService } from '../../../src/nest/collections/collections-storage-refs.service';
|
||||
import { FilesStorageRefsService } from '../../../src/nest/files/files-storage-refs.service';
|
||||
import { JourneyStorageRefsService } from '../../../src/nest/journey/journey-storage-refs.service';
|
||||
import { PhotosStorageRefsService } from '../../../src/nest/photos/photos-storage-refs.service';
|
||||
import { PlaceContribStorageRefsService } from '../../../src/nest/place-contrib/place-contrib-storage-refs.service';
|
||||
import { PlacesStorageRefsService } from '../../../src/nest/places/places-storage-refs.service';
|
||||
import { StorageReclaimRegistryService } from '../../../src/nest/storage/storage-reclaim-registry.service';
|
||||
import {
|
||||
STORAGE_RECLAIMABLE_CATEGORIES,
|
||||
STORAGE_REFERENCE_SOURCE_IDS,
|
||||
type StorageReferenceSource,
|
||||
} from '../../../src/nest/storage/storage-reclaim.types';
|
||||
import type { ServedCategory } from '../../../src/nest/storage/storage.types';
|
||||
import { TripsStorageRefsService } from '../../../src/nest/trips/trips-storage-refs.service';
|
||||
import { createSnapshotTestDb } from '../../helpers/db-mock';
|
||||
import { makeCollection, makeCollectionPlace } from '../../helpers/factories/collections';
|
||||
import { makeTripFile } from '../../helpers/factories/files';
|
||||
import { makeJourney } from '../../helpers/factories/journeys';
|
||||
import { makePlace } from '../../helpers/factories/places';
|
||||
import { insertRow, updateRows } from '../../helpers/factories/rows';
|
||||
import { makeTrip } from '../../helpers/factories/trips';
|
||||
import { makeUser } from '../../helpers/factories/users';
|
||||
import { createTestModule, type TestModule } from '../../helpers/test-module';
|
||||
import { createTestOrm, type TestOrm } from '../../helpers/test-orm';
|
||||
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
|
||||
const SOURCES = [
|
||||
TripsStorageRefsService,
|
||||
CollectionsStorageRefsService,
|
||||
PlacesStorageRefsService,
|
||||
PlaceContribStorageRefsService,
|
||||
JourneyStorageRefsService,
|
||||
PhotosStorageRefsService,
|
||||
FilesStorageRefsService,
|
||||
AuthStorageRefsService,
|
||||
];
|
||||
|
||||
const db = createSnapshotTestDb();
|
||||
let orm: TestOrm;
|
||||
let t: TestModule;
|
||||
let userId: number;
|
||||
let tripId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
orm = await createTestOrm(db);
|
||||
t = await createTestModule({ db, imports: [], providers: [StorageReclaimRegistryService, ...SOURCES], init: true });
|
||||
userId = (await makeUser(orm)).user.id;
|
||||
tripId = (await makeTrip(orm, userId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await t.close();
|
||||
await orm.close();
|
||||
db.close();
|
||||
});
|
||||
|
||||
/** The sorted keys a source still references. */
|
||||
async function asked(source: StorageReferenceSource, category: ServedCategory, keys: string[]): Promise<string[]> {
|
||||
return [...(await source.referenced(category, keys))].sort();
|
||||
}
|
||||
|
||||
describe('reference sources', () => {
|
||||
it('RSRC-001: every source registers itself once the app boots, under the ids the ledger knows', () => {
|
||||
const registry = t.get(StorageReclaimRegistryService);
|
||||
expect([...registry.ids()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort());
|
||||
for (const source of registry.sources()) {
|
||||
const claimed = source.claims.map((claim) => `${claim.category}:${claim.prefix}`).sort();
|
||||
if (source.id === 'files') expect(claimed).toEqual(['files:']);
|
||||
else if (source.id === 'auth') expect(claimed).toEqual(['avatars:']);
|
||||
else expect(claimed).toEqual(STORAGE_RECLAIMABLE_CATEGORIES.map((category) => `${category}:`).sort());
|
||||
}
|
||||
});
|
||||
|
||||
it('RSRC-002: a trip cover keeps its object in every spelling, and in whatever category it points into', async () => {
|
||||
await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-a.jpg' });
|
||||
await makeTrip(orm, userId, { cover_image: 'https://trip.example.test/uploads/places/rsrc-b.jpg?v=2' });
|
||||
await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Mixed.JPG' });
|
||||
await makeTrip(orm, userId, { cover_image: 'https://images.unsplash.com/rsrc-z.jpg' });
|
||||
const trips = t.get(TripsStorageRefsService);
|
||||
expect(await asked(trips, 'covers', ['rsrc-a.jpg', 'rsrc-mixed.jpg', 'rsrc-z.jpg', 'rsrc-b.jpg'])).toEqual([
|
||||
'rsrc-a.jpg',
|
||||
'rsrc-mixed.jpg',
|
||||
]);
|
||||
expect(await asked(trips, 'places', ['rsrc-b.jpg', 'rsrc-a.jpg'])).toEqual(['rsrc-b.jpg']);
|
||||
expect(await asked(trips, 'covers', [])).toEqual([]);
|
||||
});
|
||||
|
||||
it('RSRC-003: a list cover and a saved place image both keep their objects', async () => {
|
||||
const collection = await makeCollection(orm, userId, { cover_image: '/uploads/covers/rsrc-c.jpg' });
|
||||
await makeCollectionPlace(orm, collection.id, userId, { image_url: '/uploads/places/rsrc-d.jpg' });
|
||||
await makeCollectionPlace(orm, collection.id, userId, { image_url: 'https://example.test/rsrc-y.jpg' });
|
||||
const collections = t.get(CollectionsStorageRefsService);
|
||||
expect(await asked(collections, 'covers', ['rsrc-c.jpg', 'rsrc-d.jpg'])).toEqual(['rsrc-c.jpg']);
|
||||
expect(await asked(collections, 'places', ['rsrc-d.jpg', 'rsrc-y.jpg'])).toEqual(['rsrc-d.jpg']);
|
||||
});
|
||||
|
||||
it('RSRC-004: a place image keeps its object; a photo-proxy path or an inline image names none', async () => {
|
||||
await makePlace(orm, tripId, { image_url: '/uploads/places/rsrc-e.jpg' });
|
||||
await makePlace(orm, tripId, { image_url: '/uploads/covers/rsrc-e2.jpg' });
|
||||
await makePlace(orm, tripId, { image_url: '/api/maps/place-photo/rsrc-x/bytes' });
|
||||
await makePlace(orm, tripId, { image_url: 'data:image/png;base64,AAAA' });
|
||||
const places = t.get(PlacesStorageRefsService);
|
||||
expect(await asked(places, 'places', ['rsrc-e.jpg', 'rsrc-x', 'bytes'])).toEqual(['rsrc-e.jpg']);
|
||||
expect(await asked(places, 'covers', ['rsrc-e2.jpg'])).toEqual(['rsrc-e2.jpg']);
|
||||
});
|
||||
|
||||
it('RSRC-005: a queued contribution keeps the image it has yet to send', async () => {
|
||||
const place = await makePlace(orm, tripId, { image_url: null });
|
||||
await insertRow(orm, PlaceContribOutbox, {
|
||||
contrib_key: 'osm:node/2924',
|
||||
field: 'image',
|
||||
value: '/uploads/places/rsrc-f.jpg',
|
||||
place: place.id,
|
||||
place_name: 'Queued',
|
||||
cid: 'cid-rsrc-0001',
|
||||
eligible_at: '2026-01-01 00:00:00',
|
||||
next_attempt_at: '2026-01-01 00:00:00',
|
||||
});
|
||||
const outbox = t.get(PlaceContribStorageRefsService);
|
||||
expect(await asked(outbox, 'places', ['rsrc-f.jpg', 'rsrc-f2.jpg'])).toEqual(['rsrc-f.jpg']);
|
||||
});
|
||||
|
||||
it('RSRC-006: a journey cover keeps a shared trip cover and a gallery photo by their relative spelling', async () => {
|
||||
await makeJourney(orm, userId, { cover_image: 'covers/rsrc-g.jpg' });
|
||||
await makeJourney(orm, userId, { cover_image: 'journey/rsrc-h.jpg' });
|
||||
await makeJourney(orm, userId, { cover_image: '/uploads/covers/rsrc-g2.jpg' });
|
||||
const journeys = t.get(JourneyStorageRefsService);
|
||||
expect(await asked(journeys, 'covers', ['rsrc-g.jpg', 'rsrc-g2.jpg', 'rsrc-h.jpg'])).toEqual([
|
||||
'rsrc-g.jpg',
|
||||
'rsrc-g2.jpg',
|
||||
]);
|
||||
expect(await asked(journeys, 'journey', ['rsrc-h.jpg', 'rsrc-g.jpg'])).toEqual(['rsrc-h.jpg']);
|
||||
});
|
||||
|
||||
it('RSRC-007: every photo row with a path keeps it, whatever its provider and links', async () => {
|
||||
// A local photo retargeted to a remote provider is still served from its bytes.
|
||||
await insertRow(orm, TrekPhotos, {
|
||||
provider: 'immich',
|
||||
asset_id: 'rsrc-asset',
|
||||
owner: userId,
|
||||
file_path: 'journey/rsrc-i.jpg',
|
||||
thumbnail_path: 'journey/rsrc-i-poster.jpg',
|
||||
});
|
||||
await insertRow(orm, TrekPhotos, { provider: 'local', owner: userId, file_path: 'legacy/rsrc-j.jpg' });
|
||||
await insertRow(orm, TrekPhotos, { provider: 'immich', asset_id: 'rsrc-remote', owner: userId });
|
||||
const photos = t.get(PhotosStorageRefsService);
|
||||
expect(
|
||||
await asked(photos, 'journey', ['rsrc-i.jpg', 'rsrc-i-poster.jpg', 'rsrc-j.jpg', 'rsrc-remote', 'rsrc-k.jpg']),
|
||||
).toEqual(['rsrc-i-poster.jpg', 'rsrc-i.jpg', 'rsrc-j.jpg']);
|
||||
expect(await asked(photos, 'covers', ['rsrc-i.jpg'])).toEqual([]);
|
||||
});
|
||||
|
||||
it('RSRC-008: a trip file keeps its object, trashed or carrying a legacy path', async () => {
|
||||
await makeTripFile(orm, tripId, { filename: 'rsrc-k.pdf' });
|
||||
await makeTripFile(orm, tripId, { filename: 'rsrc-l.pdf', deleted_at: '2026-01-01 00:00:00' });
|
||||
await makeTripFile(orm, tripId, { filename: 'files/rsrc-m.pdf' });
|
||||
await makeTripFile(orm, tripId, { filename: '/uploads/files/rsrc-n.pdf' });
|
||||
await makeTripFile(orm, tripId, { filename: 'uploads\\files\\rsrc-p.pdf' });
|
||||
const files = t.get(FilesStorageRefsService);
|
||||
expect(
|
||||
await asked(files, 'files', ['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf', 'rsrc-o.pdf']),
|
||||
).toEqual(['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf']);
|
||||
});
|
||||
|
||||
it('RSRC-009: a source asked about a category it does not claim answers nothing', async () => {
|
||||
await makeTripFile(orm, tripId, { filename: 'rsrc-q.jpg' });
|
||||
expect(await asked(t.get(FilesStorageRefsService), 'covers', ['rsrc-q.jpg'])).toEqual([]);
|
||||
expect(await asked(t.get(FilesStorageRefsService), 'files', [])).toEqual([]);
|
||||
expect(await asked(t.get(AuthStorageRefsService), 'files', ['rsrc-q.jpg'])).toEqual([]);
|
||||
});
|
||||
|
||||
it('RSRC-010: an uploaded avatar keeps its object; an OIDC picture names none', async () => {
|
||||
const uploaded = (await makeUser(orm)).user;
|
||||
const oidc = (await makeUser(orm)).user;
|
||||
await updateRows(orm, Users, { id: uploaded.id }, { avatar: 'rsrc-r.png' });
|
||||
await updateRows(orm, Users, { id: oidc.id }, { avatar: 'https://idp.example.test/pictures/rsrc-s.png' });
|
||||
const auth = t.get(AuthStorageRefsService);
|
||||
expect(await asked(auth, 'avatars', ['rsrc-r.png', 'rsrc-s.png'])).toEqual(['rsrc-r.png']);
|
||||
expect(await asked(auth, 'avatars', [])).toEqual([]);
|
||||
});
|
||||
|
||||
it('RSRC-011: every source asked with no keys answers nothing', async () => {
|
||||
const answers = await Promise.all(
|
||||
SOURCES.map((Source) => asked(t.get<StorageReferenceSource>(Source), 'covers', [])),
|
||||
);
|
||||
expect(answers).toEqual(SOURCES.map(() => []));
|
||||
});
|
||||
});
|
||||
@@ -63,7 +63,7 @@ describe('RECLAIM_LEDGER', () => {
|
||||
});
|
||||
|
||||
it('RCLL-002: every reclaimable category waits for all the free-form owners, files and avatars for their own too', () => {
|
||||
const freeForm = ['trips', 'collections', 'places', 'journey', 'photos'];
|
||||
const freeForm = ['trips', 'collections', 'places', 'place-contrib', 'journey', 'photos'];
|
||||
for (const category of STORAGE_RECLAIMABLE_CATEGORIES) {
|
||||
const rule = RECLAIM_LEDGER[category];
|
||||
expect(rule.reclaim).toBe(true);
|
||||
|
||||
Reference in New Issue
Block a user