fix(storage): delete what a delete or a replace left unnamed right away again

Leaving every replaced cover, place image and deleted journey photo to the
nightly pass kept them publicly fetchable by URL for a week at least, and
for good while deletion was off. Owners now hand the values a delete or a
replace left behind to StorageReleaseService once their write committed.
It asks every registered reference source, with the same parser as the
nightly pass, and deletes at once what no row names any more: an object a
trip copy, a journey or a saved place still shows stays, which the old
inline deletes got wrong.

The release holds the pass's guards (reclaimable categories at their root
only, never an owner marker, never while a restore runs, never with a
required source missing or an overlapping location), never fails the
request and logs at debug. It runs for trip covers (update, upload, trip
delete), collection covers and saved-place images, place images (update,
bulk update, delete), journey covers and journey or gallery photos
(their thumbnail goes with a file that went), avatars (change, delete,
account deletion), trip files (permanent delete, empty trash) and collab
note and chat attachments. A trip file row now goes before its bytes, so a
failing backend no longer fails the delete. The objects of a deleted trip's
places and files are left to the nightly pass: their rows are other
domains'.

PlacesService drops the collection places repository nothing read any
more. Size, warning and boundary baselines follow the shrunk files.
This commit is contained in:
Maurice
2026-11-01 16:18:21 +01:00
parent 790fccb92c
commit d17457de56
46 changed files with 995 additions and 273 deletions
+3 -3
View File
@@ -5,8 +5,8 @@
"@typescript-eslint/no-require-imports": 18,
"@typescript-eslint/no-unsafe-function-type": 1,
"@typescript-eslint/no-unused-vars": 36,
"no-await-in-loop": 557,
"no-console": 234,
"no-await-in-loop": 556,
"no-console": 232,
"no-empty": 6
},
"suppressed": {
@@ -16,7 +16,7 @@
"no-misleading-character-class": 1
},
"tests": {
"@typescript-eslint/no-explicit-any": 1559,
"@typescript-eslint/no-explicit-any": 1555,
"@typescript-eslint/no-non-null-assertion": 1755,
"@typescript-eslint/no-require-imports": 25,
"@typescript-eslint/no-unsafe-function-type": 7,
@@ -316,7 +316,6 @@
"place-shadow -> AppSettings (owned by settings)",
"places -> BudgetItems (owned by budget)",
"places -> Categories (owned by categories)",
"places -> CollectionPlaces (owned by collections)",
"places -> DayAssignments (owned by assignments)",
"places -> Tags (owned by tags)",
"places -> TripMembers (owned by trip-membership)",
+1 -1
View File
@@ -8,7 +8,7 @@
"src/nest/atlas/atlas.service.ts": 1177,
"src/nest/auth/auth.service.ts": 1143,
"src/nest/budget/budget.service.ts": 1617,
"src/nest/collections/collections.service.ts": 1427,
"src/nest/collections/collections.service.ts": 1426,
"src/nest/doc-sync/doc-sync.service.ts": 1060,
"src/nest/doc-sync/providers/synology-drive.client.ts": 1160,
"src/nest/journey/journey-domain.service.ts": 1659,
+2 -2
View File
@@ -1,9 +1,9 @@
{
"tests/unit/nest/atlas.service.test.ts": 2154,
"tests/unit/nest/budget.service.db.test.ts": 2220,
"tests/unit/nest/budget.service.db.test.ts": 2219,
"tests/unit/nest/doc-sync/doc-sync.service.test.ts": 2511,
"tests/unit/nest/journey-domain.service.test.ts": 3876,
"tests/unit/nest/maps.service.test.ts": 3680,
"tests/unit/nest/places.service.test.ts": 2561,
"tests/unit/nest/places.service.test.ts": 2545,
"tests/unit/nest/trips.service.test.ts": 2363
}
@@ -138,6 +138,12 @@ export class CollectionPlacesRepository extends TrekRepository<CollectionPlaces>
// CL47/CL48 — updatePlace
// ---------------------------------------------------------------------
/** CL47 (`updatePlace`'s pre-image read, dup text also `deletePlace`/`deletePlacesMany`/`setPlaceImage`) — `SELECT image_url FROM collection_places WHERE id=?`. Plain (non-FK) column — safe through the entity API, `fields: ['image_url']`. */
async imageUrl(id: number): Promise<string | null | undefined> {
const row = await this.findOne({ id }, { fields: ['image_url'] });
return row?.image_url;
}
/**
* CL48 (`updatePlace`, TX) — the allow-listed dynamic `UPDATE
* collection_places SET ... WHERE id=?`, including the cross-list move
@@ -230,6 +230,16 @@ export class CollectionsRepository extends TrekRepository<Collections> {
.execute();
}
/** CL31 (`setCollectionCover`) — `SELECT cover_image FROM collections WHERE id=?`. */
async coverImage(id: number): Promise<string | null | undefined> {
const row = await this.readDb()
.selectFrom('collections')
.select('cover_image')
.where('id', '=', id)
.executeTakeFirst();
return row?.cover_image;
}
/** CL32 (`setCollectionCover`) — `UPDATE collections SET cover_image=?, updated_at=<dbNow()> WHERE id=?`. */
async setCoverImage(id: number, coverUrl: string | null): Promise<void> {
await this.readDb()
@@ -28,6 +28,7 @@ import { BudgetService } from '../budget/budget.service';
import { UnitOfWork } from '../database/unit-of-work';
import { pluginsDataRoot } from '../plugins/paths';
import { enqueueHookUserDataErasures } from '../plugins/user-erasure-enqueue';
import { StorageReleaseService } from '../storage/storage-release.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
@@ -73,6 +74,7 @@ export class UserCleanupService {
@InjectRepository(Plugins) private readonly pluginsRepo: PluginsRepository,
@InjectRepository(PluginUserErasureQueue)
private readonly pluginUserErasureQueueRepo: PluginUserErasureQueueRepository,
private readonly release: StorageReleaseService,
) {}
/**
@@ -188,10 +190,15 @@ export class UserCleanupService {
* stays raw, per each method's own docstring.
*/
async deleteUserCompletely(userId: number): Promise<void> {
const avatar = await this.usersRepo.getAvatar(userId);
await this.uow.transactional(async () => {
await this.cleanupUserReferences(userId);
await this.erasePluginUserData(userId);
await this.usersRepo.deleteById(userId);
});
// After the commit, the uploaded avatar goes unless another row names it.
// What the user's trips and journeys named is the nightly reclaim pass's:
// those rows are other domains' and went with the cascade.
await this.release.releaseValues([avatar], { bare: 'avatars' });
}
}
+9 -15
View File
@@ -22,7 +22,7 @@ import {
writeInstanceApiKey,
type InstanceApiKeyName,
} from '../settings/instance-api-keys';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { EMAIL_REGEX, mask_stored_api_key } from './auth.helpers';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
@@ -30,6 +30,9 @@ import { Injectable } from '@nestjs/common';
/** A key check is a single request to a provider; one that hangs must not hold the settings page. */
const KEY_CHECK_TIMEOUT_MS = 10_000;
/** `users.avatar` holds the bare name of an uploaded avatar. */
const AVATAR_KEYS = { bare: 'avatars' } as const;
/**
* The account a user administers about themselves: display settings, avatar,
* the third-party API keys they paste in, and the directory listing other
@@ -51,7 +54,7 @@ const KEY_CHECK_TIMEOUT_MS = 10_000;
@Injectable()
export class UserProfileService {
constructor(
private readonly storage: StorageService,
private readonly release: StorageReleaseService,
private readonly uow: UnitOfWork,
@InjectRepository(AppSettings) private readonly appSettings: AppSettingsRepository,
@InjectRepository(Users) private readonly usersRepo: UsersRepository,
@@ -359,16 +362,10 @@ export class UserProfileService {
async saveAvatar(userId: number, filename: string) {
const current = await this.usersRepo.getAvatar(userId);
// Only a locally uploaded file has something to clean up. An OIDC picture URL
// (#1399) has no storage object, so skip the delete entirely.
if (current && !/^https:\/\//i.test(current)) {
// Fire-and-forget parity: leftover objects are harmless; the DB update is
// the source of truth for which avatar is current. The catch also
// swallows a hostile stored value the central key validation rejects.
await this.storage.delete('avatars', current).catch(() => {});
}
await this.usersRepo.setAvatar(userId, filename);
// The replaced upload goes once the row names the new one. An OIDC picture
// URL (#1399) names no object of ours, and a hostile value names nothing.
await this.release.releaseValues([current], AVATAR_KEYS);
const updated = await this.usersRepo.findProfileBasic(userId);
return { success: true, avatar_url: avatarUrl(updated || {}) };
@@ -376,11 +373,8 @@ export class UserProfileService {
async deleteAvatar(userId: number) {
const current = await this.usersRepo.getAvatar(userId);
// An OIDC picture URL (#1399) has no storage object — only delete an uploaded one.
if (current && !/^https:\/\//i.test(current)) {
await this.storage.delete('avatars', current).catch(() => {});
}
await this.usersRepo.setAvatar(userId, null);
await this.release.releaseValues([current], AVATAR_KEYS); // an OIDC picture URL (#1399) names no object
return { success: true };
}
+17 -22
View File
@@ -26,15 +26,16 @@ import { UnitOfWork } from '../database/unit-of-work';
import { NotificationsService } from '../notifications/notifications.service';
import { PermissionsService } from '../permissions/permissions.service';
import { RealtimeService } from '../realtime/realtime.service';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { InjectRepository } from '@mikro-orm/nestjs';
import { Injectable } from '@nestjs/common';
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
import path from 'path';
type Trip = TripAccess;
/** How a `trip_files.filename` names its object: the bare name, after any legacy path. */
const TRIP_FILE_KEYS = { bare: 'files', basename: true } as const;
// Half a megabyte of markup is far more than any og:/<title> block needs, and
// the URL is caller-supplied, so read no further than that.
const MAX_PREVIEW_BYTES = 512 * 1024;
@@ -169,7 +170,7 @@ export class CollabService {
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly notifications: NotificationsService,
private readonly storage: StorageService,
private readonly release: StorageReleaseService,
private readonly rateLimit: RateLimitService,
private readonly uow: UnitOfWork,
@InjectRepository(CollabMessageReactions) private readonly messageReactionsRepo: CollabMessageReactionsRepository,
@@ -381,20 +382,18 @@ export class CollabService {
const existing = await this.notesRepo.findInTrip(idNum, tripIdNum);
if (!existing) return false;
// Clean up attached objects first (delete-first is intentional — a failed
// row delete leaves dangling rows, never orphaned files). basename()
// tolerates any legacy 'files/'-prefixed row the boot migration has not
// seen.
// The rows go first, then the release of what they named: a failed delete
// keeps note, rows and bytes together, and an attachment another row still
// names (a trip copy's file) keeps its bytes (#2924).
const noteFiles = await this.notesRepo.listFilenamesForNote(existing.id);
for (const f of noteFiles) {
await this.storage.delete('files', path.basename(f.filename)).catch(() => {
/* ignore */
});
}
await this.uow.transactional(async () => {
await this.notesRepo.deleteAttachmentsForNote(existing.id);
await this.notesRepo.delete(existing.id);
});
await this.release.releaseValues(
noteFiles.map((f) => f.filename),
TRIP_FILE_KEYS,
);
return true;
}
@@ -449,11 +448,8 @@ export class CollabService {
const file = await this.notesRepo.findScopedForNote(fileIdNum, noteIdNum, tripIdNum);
if (!file) return false;
await this.storage.delete('files', path.basename(file.filename)).catch(() => {
/* ignore */
});
await this.notesRepo.deleteAttachmentById(file.id);
await this.release.releaseValues([file.filename], TRIP_FILE_KEYS);
return true;
}
@@ -750,11 +746,10 @@ export class CollabService {
// Only once the rows are actually gone. Dropping the blobs first left a
// live message pointing at attachments whose bytes no longer existed
// whenever the second statement failed.
for (const file of attachments) {
void this.storage.delete('files', path.basename(file.filename)).catch(() => {
/* best effort */
});
}
await this.release.releaseValues(
attachments.map((file) => file.filename),
TRIP_FILE_KEYS,
);
// Read before the delete; see the class docstring's `username` note.
return { username: author?.username };
}
@@ -5,7 +5,7 @@ import { InjectRepository } from '@mikro-orm/nestjs';
import { PermissionsService } from '../permissions/permissions.service';
import { RealtimeService } from '../realtime/realtime.service';
import { PlacesService } from '../places/places.service';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { Collections } from '../../db/entities/Collections.entity';
import { CollectionsRepository, type CollectionPlaceRow } from '../../db/repositories/Collections.repository';
import { CollectionMembers } from '../../db/entities/CollectionMembers.entity';
@@ -77,18 +77,13 @@ import { PlaceImportService, type ExportedCollectionFile } from '../place-import
import { UnitOfWork } from '../database/unit-of-work';
import { parseLinks, serializeLinks } from './collections.helpers';
// ---------------------------------------------------------------------------
// Errors — thrown as plain Errors carrying a status; TrekExceptionFilter maps
// `err.status` → that HTTP code with an `{ error: message }` body.
// ---------------------------------------------------------------------------
// Errors: a DomainError carrying a status; TrekExceptionFilter maps it to that code with an `{ error }` body.
function httpError(status: number, message: string): never {
throw new DomainError(status, message);
}
/** Re-exported under this service's historical name — the underlying type
* now lives in `_shared/collection-role.ts` (R6), the ONE shared predicate
* `isVisible`/`isOwner`/`roleOf` all resolve through. */
/** Re-exported under this service's historical name; the type lives in `_shared/collection-role.ts` (R6). */
export type EffectiveRole = CollectionRole;
type PlaceRow = CollectionPlaceRow;
@@ -110,7 +105,7 @@ export class CollectionsService {
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly notifications: NotificationsService,
private readonly storage: StorageService,
private readonly release: StorageReleaseService, // replaced and deleted images, after the write
private readonly uow: UnitOfWork,
@InjectRepository(Collections) private readonly collectionsRepo: CollectionsRepository,
@InjectRepository(CollectionMembers) private readonly members: CollectionMembersRepository,
@@ -601,13 +596,12 @@ export class CollectionsService {
return { ...col, is_owner: col.owner_id === userId };
}
/**
* Set (or clear) a list's cover image. The previous file stays where another
* row may name it; the storage reclaim job removes it once nothing does.
*/
/** Set (or clear) a list's cover image; the previous file goes unless another row still names it. */
async setCollectionCover(userId: number, id: number, coverUrl: string | null, socketId?: string): Promise<Collection> {
await this.assertCanEdit(userId, id);
const prev = (await this.collectionsRepo.coverImage(id)) ?? null;
await this.collectionsRepo.setCoverImage(id, coverUrl);
await this.release.releaseValues([prev]);
await this.notifyCollectionUsers(id, socketId, 'collections:updated');
const col = await this.getCollectionRow(id);
return { ...col, is_owner: col.owner_id === userId };
@@ -876,6 +870,7 @@ export class CollectionsService {
async updatePlace(userId: number, placeId: number, body: CollectionPlaceUpdateRequest, socketId?: string): Promise<CollectionPlace> {
const currentCollection = await this.collectionIdOfPlace(placeId);
await this.assertCanEdit(userId, currentCollection);
const prevImage = body.image_url !== undefined ? (await this.collectionPlaces.imageUrl(placeId)) ?? null : null;
const write: Parameters<CollectionPlacesRepository['updateFields']>[1] = {};
if (body.name !== undefined) write.name = body.name;
@@ -898,8 +893,7 @@ export class CollectionsService {
movedTo = body.collection_id;
}
// Field update + tag rewrite + label rewrite are one logical write — atomic
// since the post-fold quirk pass.
// Field update + tag rewrite + label rewrite are one logical write, so one transaction.
await this.uow.transactional(async () => {
await this.collectionPlaces.updateFields(placeId, write);
@@ -913,6 +907,7 @@ export class CollectionsService {
if (movedTo) await this.collectionPlaces.deleteLabelAssignments(placeId);
if (body.label_ids !== undefined) await this.setPlaceLabels(placeId, movedTo ?? currentCollection, body.label_ids);
});
await this.release.releaseValues([prevImage]); // a replaced custom thumbnail (#1136), unless a copy names it
await this.notifyCollectionUsers(currentCollection, socketId, 'collections:updated');
if (movedTo) await this.notifyCollectionUsers(movedTo, socketId, 'collections:updated');
@@ -944,34 +939,36 @@ export class CollectionsService {
return this.getPlaceById(placeId);
}
// A saved place's uploaded image is never deleted here: copy-to-trip and
// save-to-collection share it by reference, and other columns can name it
// too. The storage reclaim job removes it once no row does (#2924).
// A saved place's image goes with it unless a trip place it was copied to (or from) still names it (#2924).
async deletePlace(userId: number, placeId: number, socketId?: string): Promise<void> {
const collectionId = await this.collectionIdOfPlace(placeId);
await this.assertCanDelete(userId, collectionId);
const image = (await this.collectionPlaces.imageUrl(placeId)) ?? null;
await this.collectionPlaces.deleteById(placeId); // CASCADE drops tags. NO photo-cache reclaim.
await this.release.releaseValues([image]);
await this.notifyCollectionUsers(collectionId, socketId, 'collections:updated');
}
async deletePlacesMany(userId: number, ids: number[], socketId?: string): Promise<number[]> {
// All-or-nothing since the post-fold quirk pass: every id is resolved and
// permission-checked BEFORE any delete (the relocated legacy interleaved
// checks with deletes, so a mid-list 403/404 left earlier deletes committed),
// and the deletes then run in one transaction.
// All-or-nothing: every id is resolved and permission-checked BEFORE any delete (the legacy interleaved
// them, so a mid-list 403/404 left earlier deletes committed), then the deletes run in one transaction.
const deleted: number[] = [];
const touched = new Set<number>();
const reads: Promise<string | null | undefined>[] = [];
for (const id of ids) {
const collectionId = await this.collectionIdOfPlace(id);
await this.assertCanDelete(userId, collectionId);
touched.add(collectionId);
reads.push(this.collectionPlaces.imageUrl(id).catch(() => null)); // to release once the rows are gone
}
const images = await Promise.all(reads);
await this.uow.transactional(async () => {
for (const id of ids) {
await this.collectionPlaces.deleteById(id);
deleted.push(id);
}
});
await this.release.releaseValues(images);
for (const cid of touched) await this.notifyCollectionUsers(cid, socketId, 'collections:updated');
return deleted;
}
@@ -1059,11 +1056,13 @@ export class CollectionsService {
return this.collectionPlaces.matchingByTripSource(collectionIds, tripId, place, COORD_DEDUP_TOLERANCE);
}
/** Set (or clear) a saved place's custom thumbnail; the storage reclaim job takes the replaced upload. */
/** Set (or clear) a saved place's custom thumbnail; the replaced upload goes unless another row names it. */
async setPlaceImage(userId: number, placeId: number, imageUrl: string | null, socketId?: string): Promise<CollectionPlace> {
const collectionId = await this.collectionIdOfPlace(placeId);
await this.assertCanEdit(userId, collectionId);
const prev = (await this.collectionPlaces.imageUrl(placeId)) ?? null;
await this.collectionPlaces.setImageUrl(placeId, imageUrl);
await this.release.releaseValues([prev]);
await this.notifyCollectionUsers(collectionId, socketId, 'collections:updated');
return this.getPlaceById(placeId);
}
+22 -32
View File
@@ -23,6 +23,7 @@ import { restTripWriter, type TripWriter } from '../common/trip-writer';
import { UnitOfWork } from '../database/unit-of-work';
import { PermissionsService } from '../permissions/permissions.service';
import { RealtimeService } from '../realtime/realtime.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { StorageService } from '../storage/storage.service';
import { StorageNotFoundError, StorageInvalidKeyError, type ObjectStat } from '../storage/storage.types';
import { TripAccessService } from '../trip-membership/trip-access.service';
@@ -73,6 +74,9 @@ export interface FileLink {
*/
export const FILE_CONTENT_MAX = 10 * 1024 * 1024;
/** How `trip_files.filename` names its object: the bare name, after any legacy path (every reader basename()s it). */
const TRIP_FILE_KEYS = { bare: 'files', basename: true } as const;
/** Why a content read was refused. Each caller maps it onto its own error shape. */
export type FileContentRefusal = 'not-found' | 'too-large' | 'not-accessible';
@@ -155,6 +159,7 @@ export class FilesService {
@InjectRepository(Places) private readonly placesRepo: PlacesRepository,
@InjectRepository(DayAssignments) private readonly dayAssignmentsRepo: DayAssignmentsRepository,
@InjectRepository(BudgetItems) private readonly budgetItemsRepo: BudgetItemsRepository,
private readonly release: StorageReleaseService,
) {}
async verifyTripAccess(tripId: string | number, userId: number) {
@@ -490,46 +495,31 @@ export class FilesService {
return formatFile(restored!);
}
/**
* Drops the row, then releases its object: the bytes go unless another row
* (a trip copy's file, a doc-sync copy) still names them (#2924). A release
* never fails the delete; what it leaves is the nightly reclaim pass's.
*/
async permanentDeleteFile(file: TripFileRow): Promise<void> {
// storage.delete is idempotent on a missing object (the old rm force:true
// contract). Only drop the DB row when the delete either succeeded or the
// object was already gone — otherwise a permission / ENOSPC failure
// would orphan the bytes with no DB pointer left to clean them.
try {
await this.storage.delete('files', path.basename(file.filename));
} catch (e) {
console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e);
throw e;
}
await this.tripFilesRepo.deleteById(file.id);
await this.release.releaseValues([file.filename], TRIP_FILE_KEYS);
}
/** R2: the storage-delete loop stays outside any transaction (rule 24); only the final bulk `trip_files` delete (FL23) runs inside one. */
/**
* Every trashed file of the trip, rows first, then the release of what they
* named, as for permanentDeleteFile. The bulk `trip_files` delete (FL23) is
* one statement, so it needs no transaction (task-8-review.md L1/U1).
*/
async emptyTrash(tripId: string | number): Promise<number> {
const tripIdNum = toRowId(tripId) ?? -1;
const trashed = await this.tripFilesRepo.listTrashed(tripIdNum);
// Collect successful IDs separately so we only DELETE rows whose disk
// content was actually removed — failing unlinks keep their DB row
// and a retry via the single-file delete path can try again.
const successfullyUnlinked: number[] = [];
await Promise.all(
trashed.map(async (file) => {
try {
await this.storage.delete('files', path.basename(file.filename));
successfullyUnlinked.push(file.id);
} catch (e) {
console.error(`[files] unlink failed for ${file.filename}, keeping DB row:`, e);
}
}),
if (trashed.length === 0) return 0;
await this.tripFilesRepo.deleteMany(trashed.map((file) => file.id));
await this.release.releaseValues(
trashed.map((file) => file.filename),
TRIP_FILE_KEYS,
);
// A single statement is already atomic — no `uow.transactional` wrapper
// needed here (task-8-review.md L1/U1: the wrapper this method used to
// carry was vacuous, and FILE-SVC-062's "rollback" claim never actually
// exercised it — removing it keeps every test green).
if (successfullyUnlinked.length > 0) {
await this.tripFilesRepo.deleteMany(successfullyUnlinked);
}
return successfullyUnlinked.length;
return trashed.length;
}
// ---------------------------------------------------------------------------
@@ -14,6 +14,7 @@ import { PluginGuardsModule } from '../../nest-rpc/plugin-guards.module';
import { TrekPhotosModule } from '../photos/trek-photos.module';
import { RealtimeModule } from '../realtime/realtime.module';
import { SettingsModule } from '../settings/settings.module';
import { StorageModule } from '../storage/storage.module';
import { JourneyDomainService } from './journey-domain.service';
import { JourneyEntryStaysService } from './journey-entry-stays.service';
import { JourneyMapService } from './journey-map.service';
@@ -88,6 +89,8 @@ import { Module } from '@nestjs/common';
TrekPhotosModule,
PluginGuardsModule,
SettingsModule,
// JourneySettingsService releases a replaced cover's object after the write.
StorageModule,
MikroOrmModule.forFeature([
Journeys,
JourneyContributors,
@@ -2,6 +2,7 @@ import { Journeys } from '../../db/entities/Journeys.entity';
import type { JourneysRepository } from '../../db/repositories/Journeys.repository';
import type { Journey } from '../../types';
import { UnitOfWork } from '../database/unit-of-work';
import { StorageReleaseService } from '../storage/storage-release.service';
import { JourneyDomainService } from './journey-domain.service';
import {
journeySettingsPatch,
@@ -24,7 +25,9 @@ interface JourneySettingsUpdate {
* of the input and the same owner check), with the stored values of exactly
* the columns it wrote, read in the same transaction. That is what takes the
* change back: the assistant's undo writes those values again through the
* same tool. The broadcast goes out after the commit.
* same tool. The broadcast goes out after the commit, and so does the release
* of a replaced cover: its object goes unless another row (the trip it was
* inherited from, a gallery photo) still names it (#2924).
*/
@Injectable()
export class JourneySettingsService {
@@ -32,6 +35,7 @@ export class JourneySettingsService {
private readonly journeys: JourneyDomainService,
private readonly uow: UnitOfWork,
@InjectRepository(Journeys) private readonly journeysRepo: JourneysRepository,
private readonly release: StorageReleaseService,
) {}
/** Null for anyone but the owner, who alone reshapes the journey itself, as on the route. */
@@ -50,6 +54,7 @@ export class JourneySettingsService {
return previousSettings(before as unknown as Readonly<Record<string, unknown>> | undefined, patch);
});
if (written) await this.journeys.broadcastJourneyEvent(journeyId, 'journey:updated', {}, sid);
if (typeof previous.cover_image === 'string') await this.release.releaseValues([previous.cover_image]);
const journey = (await this.journeysRepo.findById(journeyId)) as Journey | undefined;
return journey ? { journey, previous } : null;
}
+19 -4
View File
@@ -25,6 +25,8 @@ import type { User } from '../../types';
import { RuntimeEnvService } from '../app-config/runtime-env.service';
import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { journeyThumbName } from '../memories/thumbnail.service';
import { JourneyService } from './journey.service';
import { JourneyBookService } from './journey-book.service';
import { JourneyPhotoCaptureService, type ProviderPhotoBatch, type ProviderPhotoTarget } from './journey-photo-capture.service';
@@ -175,6 +177,8 @@ export class JourneyController {
private readonly photoLinks: JourneyPhotoLinksService,
// Whether DEMO_MODE is on, read on every upload (refuseDemoUpload).
private readonly env: RuntimeEnvService,
// What a deleted photo named, once its row is gone (#2924).
private readonly release: StorageReleaseService,
) {}
/**
@@ -459,16 +463,27 @@ export class JourneyController {
@Delete('photos/:photoId')
@ResponseContract(successResponseSchema)
async deletePhoto(@CurrentUser() user: User, @Param('photoId') photoId: string) {
// The bytes stay: another journey, a trip or a cover may name the same
// object. Once the photo row is gone the storage reclaim job takes the file
// and its poster, and the thumbnail sweep its thumbnail (#2924).
const photo = await this.journey.deletePhoto(Number(photoId), user.id);
if (!photo) {
throw new HttpException({ error: 'Photo not found' }, 404);
}
await this.releasePhoto(photo);
return { success: true };
}
/**
* After a photo's row went: its file and its poster go at once unless another
* row (another journey's photo, a journey cover) still names them, and the
* thumbnail derived from a file that went goes with it (#2924).
*/
private async releasePhoto(photo: { file_path?: string | null; thumbnail_path?: string | null }): Promise<void> {
const released = await this.release.releaseValues([photo.file_path, photo.thumbnail_path], { bare: 'journey' });
const filePath = photo.file_path;
if (!filePath?.startsWith('journey/')) return;
if (!released.some((ref) => ref.category === 'journey' && `journey/${ref.key}` === filePath)) return;
await this.storage.delete('journey', journeyThumbName(filePath)).catch(() => { /* already gone */ });
}
// ── Gallery (prefix /:id/gallery — before /:id) ─────────────────────────
@Post(':id/gallery/photos')
@UseInterceptors(FilesInterceptor('photos'))
@@ -546,11 +561,11 @@ export class JourneyController {
@HttpCode(204)
@ResponseContract(emptyResponseSchema)
async deleteGalleryPhoto(@CurrentUser() user: User, @Param('journeyPhotoId') journeyPhotoId: string): Promise<void> {
// The bytes stay, as for deletePhoto above.
const photo = await this.journey.deleteGalleryPhoto(Number(journeyPhotoId), user.id);
if (!photo) {
throw new HttpException({ error: 'Photo not found or not allowed' }, 404);
}
await this.releasePhoto(photo);
}
// ── Journeys /:id ───────────────────────────────────────────────────────
+10 -2
View File
@@ -6,6 +6,7 @@ import { PhotoProviderRegistry } from '../memories/photo-provider.registry';
import { PhotoResolverService } from '../memories/photo-resolver.service';
import { JourneyDomainService } from './journey-domain.service';
import { JourneyMapService } from './journey-map.service';
import { JourneySettingsService } from './journey-settings.service';
import { JourneyShareService } from './journey-share.service';
import { Injectable } from '@nestjs/common';
@@ -28,6 +29,7 @@ export class JourneyService {
private readonly share: JourneyShareService,
private readonly immichRepo: UserImmichRepository,
private readonly map: JourneyMapService,
private readonly settings: JourneySettingsService,
) {}
async journeyAddonEnabled(): Promise<boolean> {
@@ -44,8 +46,14 @@ export class JourneyService {
getJourneyFull(id: number, userId: number) {
return this.journey.getJourneyFull(id, userId);
}
updateJourney(id: number, userId: number, data: Parameters<typeof this.journey.updateJourney>[2], sid?: string) {
return this.journey.updateJourney(id, userId, data, sid);
/** The settings update the MCP tool makes too, so a replaced cover is released the same way on both. */
async updateJourney(
id: number,
userId: number,
data: Parameters<typeof this.journey.updateJourney>[2],
sid?: string,
) {
return (await this.settings.updateWithPrevious(id, userId, data, sid))?.journey ?? null;
}
deleteJourney(id: number, userId: number, sid?: string) {
return this.journey.deleteJourney(id, userId, sid);
+14 -16
View File
@@ -25,6 +25,7 @@ import { MAX_PLACE_IMAGE_SIZE, PLACE_IMAGE_EXTENSIONS } from '../common/place-im
import { randomUUID } from 'node:crypto';
import { JourneyDomainService } from '../journey/journey-domain.service';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { AccommodationsService } from '../accommodations/accommodations.service';
import { Places } from '../../db/entities/Places.entity';
import { Tags } from '../../db/entities/Tags.entity';
@@ -42,8 +43,6 @@ import type { CategoriesRepository } from '../../db/repositories/Categories.repo
import type { TripsRepository } from '../../db/repositories/Trips.repository';
import { BudgetItems } from '../../db/entities/BudgetItems.entity';
import type { BudgetItemsRepository } from '../../db/repositories/BudgetItems.repository';
import { CollectionPlaces } from '../../db/entities/CollectionPlaces.entity';
import type { CollectionPlacesRepository } from '../../db/repositories/CollectionPlaces.repository';
/** Rows a place delete took down with the nights booked there, for the caller to announce. */
export interface CancelledStays {
@@ -157,17 +156,11 @@ export class PlacesService {
@InjectRepository(TripMembers) private readonly tripMembersRepo: TripMembersRepository,
@InjectRepository(DayAssignments) private readonly dayAssignmentsRepo: DayAssignmentsRepository,
@InjectRepository(Categories) private readonly categoriesRepo: CategoriesRepository,
// Task 9 fix wave (B-M2 / A-L3 ruling): `DatabaseService.getTripTitle`
// does not stay — it was the one method on that class this domain's own
// PL28 comment said should convert like its siblings once concurrent
// implementers stopped colliding on this file. `Trips` is already on
// `PlacesModule`'s `forFeature` list, so this needs no module change.
// Task 9 fix wave (B-M2 / A-L3): PL28's trip title, read here since `DatabaseService.getTripTitle` went.
@InjectRepository(Trips) private readonly tripsRepo: TripsRepository,
// Plan 3e Task 2 (budget) — additive, PL15/PL18/PL22 only.
@InjectRepository(BudgetItems) private readonly budgetItemsRepo: BudgetItemsRepository,
// Plan 3h Task 6 (survivors): read by nothing since the storage reclaim job
// took over the image ref-count (#2924); kept so the constructor stays put.
@InjectRepository(CollectionPlaces) private readonly collectionPlacesRepo: CollectionPlacesRepository,
private readonly release: StorageReleaseService, // the images a change or a delete left unnamed (#2924)
// Reading files and shared lists; this service only persists what it reads.
private readonly placeImport: PlaceImportService,
) {}
@@ -418,7 +411,8 @@ export class PlacesService {
body: PlaceUpdateInput,
ifMatch?: string,
): Promise<PlaceWithTags | UpdateConflict | null> {
const { result } = await this.uow.transactional(() => this.applyUpdate(tripId, placeId, body, ifMatch));
const { result, previousImage } = await this.uow.transactional(() => this.applyUpdate(tripId, placeId, body, ifMatch));
await this.release.releaseValues([previousImage]); // after the commit: kept while a copy or a cover names it
return result;
}
@@ -431,7 +425,7 @@ export class PlacesService {
placeId: string,
body: PlaceUpdateInput,
ifMatch?: string,
): Promise<{ result: PlaceWithTags | UpdateConflict | null }> {
): Promise<{ result: PlaceWithTags | UpdateConflict | null; previousImage?: string | null }> {
// PL9 — the pre-image read every `!== undefined` fallback below needs.
// `toRowId` first (Task 3 review H1, absorbed here): every write in
// this method (`updatePlace`, `tagsRepo.deleteForPlace`/`insertIgnore`)
@@ -524,9 +518,8 @@ export class PlacesService {
if (tags.length > 0) await this.tagsRepo.insertIgnore(id, await this.tagsOnTrip(tid, tags)); // PL13
}
// A replaced or cleared custom upload (#1136) stays: a copy of this place or
// a cover may name it, and the storage reclaim job removes it once none does.
return { result: await this.placesRepo.findWithTagsAndRatings(id) }; // PL14
// PL14; the image this write replaced (#1136), for the caller to release once the transaction committed.
return { result: await this.placesRepo.findWithTagsAndRatings(id), previousImage: image_url !== undefined ? existingPlace.image_url : null };
}
// -------------------------------------------------------------------------
@@ -618,6 +611,7 @@ export class PlacesService {
await this.placesRepo.deleteById(id);
});
await reclaimPhotoCache(this.photoCache, place.google_place_id, place.image_url);
await this.release.releaseValues([place.image_url]);
return { deleted: true, deletedTourPlaceIds: wasTour ? [id] : [], cancelled };
}
@@ -663,6 +657,7 @@ export class PlacesService {
for (const row of reclaimable) {
await reclaimPhotoCache(this.photoCache, row.google_place_id, row.image_url);
}
await this.release.releaseValues(reclaimable.map((row) => row.image_url));
return { deleted, deletedTourPlaceIds, cancelled };
}
@@ -690,14 +685,17 @@ export class PlacesService {
async updateMany(tripId: string, ids: number[], body: PlaceUpdateInput): Promise<PlaceWithTags[]> {
if (ids.length === 0) return [];
const updated: PlaceWithTags[] = [];
const previousImages: (string | null | undefined)[] = [];
await this.uow.transactional(async () => {
for (const id of ids) {
// Bulk update sends no If-Match, so applyUpdate() never returns a
// conflict here; the guard keeps the types honest.
const { result: place } = await this.applyUpdate(tripId, String(id), body);
const { result: place, previousImage } = await this.applyUpdate(tripId, String(id), body);
if (place && !isUpdateConflict(place)) updated.push(place);
previousImages.push(previousImage);
}
});
await this.release.releaseValues(previousImages); // one bulk value may have replaced many uploads
return updated;
}
@@ -0,0 +1,141 @@
import { logDebug, logWarn } from '../audit/audit-log.logger';
import { StorageJobsService } from './storage-jobs.service';
import { isValidKey } from './storage-keys';
import { StorageReclaimRegistryService } from './storage-reclaim-registry.service';
import { inReclaimScope, RECLAIM_LEDGER, sharedLocationsOf } from './storage-reclaim.ledger';
import {
storageKeysIn,
storageOwnerMarkerInstall,
type StorageKeyOptions,
type StorageKeyRef,
type StorageReferenceSource,
} from './storage-reclaim.types';
import { StorageRegistryService } from './storage-registry.service';
import { StorageService } from './storage.service';
import { SERVED_CATEGORIES, type ServedCategory } from './storage.types';
import { Injectable } from '@nestjs/common';
const SERVED = new Set<string>(SERVED_CATEGORIES);
/**
* Deletes what a user's change left unnamed, right after it: a replaced or
* removed cover, place image or avatar, a deleted journey photo or trip file.
* The owner calls it once its transaction committed, with the values the rows
* held, and every registered reference source (the nightly pass's sources and
* parser) is asked whether anything still names each object. Only what no row
* names goes, through `StorageService.delete` (replicas follow, a demo keeps
* its bytes), so an object a trip copy, a journey or a saved place still shows
* stays, unlike before (#2924), while a deleted photo is gone at once as it
* always was.
*
* The pass's own guards hold: only the root of a reclaimable category, never
* an owner marker, never while a restore runs, never where a required source
* is missing or the category's location overlaps another's. It never fails
* the caller's request: anything it leaves (an error, a guard) is the nightly
* pass's to find. Releases are logged at debug, not audited one by one.
*/
@Injectable()
export class StorageReleaseService {
constructor(
private readonly storage: StorageService,
private readonly registry: StorageRegistryService,
private readonly references: StorageReclaimRegistryService,
private readonly jobs: StorageJobsService,
) {}
/**
* The objects these stored values name (any spelling a column holds), each
* deleted when no row names it any more. Answers the ones that went.
*/
async releaseValues(
values: ReadonlyArray<string | null | undefined>,
options: StorageKeyOptions = {},
): Promise<StorageKeyRef[]> {
const byCategory = new Map<ServedCategory, Set<string>>();
for (const value of values) {
for (const ref of storageKeysIn(value, options)) {
if (!SERVED.has(ref.category)) continue;
const category = ref.category as ServedCategory;
const keys = byCategory.get(category) ?? new Set<string>();
keys.add(ref.key);
byCategory.set(category, keys);
}
}
const released = await Promise.all(
[...byCategory].map(async ([category, keys]) =>
(await this.releaseIfUnreferenced(category, [...keys])).map((key) => ({ category, key })),
),
);
return released.flat();
}
/** These keys of one category, each deleted when it exists and no row names it. Answers the ones that went. */
async releaseIfUnreferenced(category: ServedCategory, keys: readonly string[]): Promise<string[]> {
try {
const candidates = [...new Set(keys)].filter(
(key) => inReclaimScope(key) && isValidKey(key) && storageOwnerMarkerInstall(key) === null,
);
if (candidates.length === 0) return [];
const sources = this.sourcesFor(category);
if (sources === null) return [];
const referenced = await this.referencedIn(category, candidates, sources);
const unnamed = candidates.filter(
(key) => !referenced.has(key.toLowerCase()) && !this.storage.keepsOnDelete(category, key),
);
const released = await Promise.all(unnamed.map((key) => this.deleteIfPresent(category, key)));
return unnamed.filter((_key, i) => released[i]);
} catch (err) {
logWarn(`[storage] could not release '${category}' objects, the nightly pass will: ${messageOf(err)}`);
return [];
}
}
/** The sources to ask, or null when the category may not be released from now. */
private sourcesFor(category: ServedCategory): readonly StorageReferenceSource[] | null {
const rule = RECLAIM_LEDGER[category];
if (!rule.reclaim || this.jobs.restoreRunning) return null;
const sources = this.references.sources().filter((source) => source.claims.some((c) => c.category === category));
const whole = new Set(
sources
.filter((source) => source.claims.some((c) => c.category === category && c.prefix === ''))
.map((source) => source.id),
);
if (rule.requiredSources.some((id) => !whole.has(id))) return null;
if (sharedLocationsOf(this.registry).has(category)) return null;
return sources;
}
private async referencedIn(
category: ServedCategory,
keys: readonly string[],
sources: readonly StorageReferenceSource[],
): Promise<Set<string>> {
const answers = await Promise.all(
sources.map(async (source) => {
const asked = keys.filter((key) =>
source.claims.some((claim) => claim.category === category && key.startsWith(claim.prefix)),
);
return asked.length === 0 ? new Set<string>() : await source.referenced(category, asked);
}),
);
const referenced = new Set<string>();
for (const answer of answers) for (const key of answer) referenced.add(key.toLowerCase());
return referenced;
}
private async deleteIfPresent(category: ServedCategory, key: string): Promise<boolean> {
try {
if ((await this.storage.stat(category, key)) === null) return false;
await this.storage.delete(category, key);
logDebug(`[storage] released an unreferenced '${category}' object`);
return true;
} catch (err) {
logWarn(`[storage] could not release a '${category}' object, the nightly pass will: ${messageOf(err)}`);
return false;
}
}
}
function messageOf(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
+14 -3
View File
@@ -13,6 +13,7 @@ import { StorageReclaimController } from './storage-reclaim.controller';
import { StorageReclaimJob } from './storage-reclaim.job';
import { StorageReclaimService } from './storage-reclaim.service';
import { StorageRegistryService } from './storage-registry.service';
import { StorageReleaseService } from './storage-release.service';
import { StorageStatsService } from './storage-stats.service';
import { StorageUsageScanJob } from './storage-usage-scan.job';
import { StorageService } from './storage.service';
@@ -37,8 +38,11 @@ import { Module } from '@nestjs/common';
* and follows database reopens (DatabaseLifecycleModule); the domains whose
* rows name stored objects register their reference sources with the exported
* StorageReclaimRegistryService, so storage never imports them.
* StorageJobsService is exported for the backup restore, which marks itself
* running there and waits for a reclaim pass in flight to let go.
* StorageJobsService is exported for the backups: a restore marks itself
* running there and waits for a reclaim pass in flight to let go, a backup
* marks itself so the pass deletes nothing it still reads.
* StorageReleaseService is exported for the owners, which hand it the values a
* delete or a replace left unnamed right after their write.
*/
@Module({
imports: [
@@ -60,7 +64,14 @@ import { Module } from '@nestjs/common';
StorageReclaimRegistryService,
StorageReclaimService,
StorageReclaimJob,
StorageReleaseService,
],
exports: [
StorageService,
StorageEventsService,
StorageJobsService,
StorageReclaimRegistryService,
StorageReleaseService,
],
exports: [StorageService, StorageEventsService, StorageJobsService, StorageReclaimRegistryService],
})
export class StorageModule {}
+10 -1
View File
@@ -3,6 +3,7 @@ import { DomainError } from '../common/domain-error';
import { restTripWriter, type TripWriter } from '../common/trip-writer';
import { PermissionsService } from '../permissions/permissions.service';
import { RealtimeService } from '../realtime/realtime.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { TripsService, type DeleteTripInfo } from './trips.service';
import { Injectable } from '@nestjs/common';
@@ -36,6 +37,7 @@ export class TripWritesService {
private readonly trips: TripsService,
private readonly permissions: PermissionsService,
private readonly realtime: RealtimeService,
private readonly release: StorageReleaseService,
) {}
/** REST's writer for a trip: the room minus the tab that sent the request. */
@@ -62,10 +64,17 @@ export class TripWritesService {
return newTripId;
}
/** Deletes a trip and tells everyone in its room, once the delete is committed. */
/**
* Deletes a trip and tells everyone in its room, once the delete is
* committed. Its cover goes with it unless another row (a trip copy, a
* journey) still names it; the objects its places and files named are the
* nightly reclaim pass's, since their rows belong to other domains.
*/
async remove(tripId: string | number, writer: TripWriter): Promise<DeleteTripInfo> {
const cover = ((await this.trips.getRaw(tripId)) as { cover_image?: string | null } | undefined)?.cover_image;
const info = await this.trips.remove(tripId, writer.userId, writer.role);
writer.events.emit('trip:deleted', { id: info.tripId });
await this.release.releaseValues([cover]);
return info;
}
+4 -2
View File
@@ -8,6 +8,7 @@ import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
import { CalendarService } from '../calendar/calendar.service';
import { contentDisposition } from '../common/content-disposition';
import { isDemoWriteBlocked, DEMO_WRITE_ERROR } from '../common/demo-write';
import { StorageReleaseService } from '../storage/storage-release.service';
import { StorageService } from '../storage/storage.service';
import { TripReadModelService } from '../trip-read-model/trip-read-model.service';
import { UnsplashService } from '../unsplash/unsplash.service';
@@ -84,6 +85,7 @@ export class TripsController {
private readonly readModel: TripReadModelService,
private readonly storage: StorageService,
private readonly tripWrites: TripWritesService,
private readonly release: StorageReleaseService,
) {}
@Get()
@@ -257,11 +259,11 @@ export class TripsController {
}
// Commit the spooled upload to its final storage location (atomic
// same-volume rename) before anything references the final path.
// The replaced cover stays: a trip copy or a journey may show the same
// object, and the storage reclaim job removes it once nothing names it.
await this.storage.put('covers', file.filename, { tmpPath: file.path });
const coverUrl = `/uploads/covers/${file.filename}`;
await this.trips.updateCoverImage(id, coverUrl);
// The replaced cover goes unless a trip copy or a journey still shows it.
await this.release.releaseValues([trip.cover_image]);
return { cover_image: coverUrl };
}
+3 -2
View File
@@ -43,7 +43,7 @@ import { BudgetService, type CurrencyRebase } from '../budget/budget.service';
import { ReservationsService } from '../reservations/reservations.service';
import { VacayService } from '../vacay/vacay.service';
import { UnsplashService } from '../unsplash/unsplash.service';
import { StorageService } from '../storage/storage.service';
import { StorageReleaseService } from '../storage/storage-release.service';
import { SettingsService } from '../settings/settings.service';
import { escapeLikePattern } from '../places/places.helpers';
import { NotFoundError, ValidationError } from '../common/domain-errors';
@@ -154,7 +154,7 @@ export class TripsService {
private readonly vacay: VacayService,
private readonly realtime: RealtimeService,
private readonly unsplash: UnsplashService,
private readonly storage: StorageService,
private readonly release: StorageReleaseService, // the cover a change replaced, released after the commit
private readonly uow: UnitOfWork,
// Plan 3c Task 0b (task-0a-review-security.md F-A1): `canAccessTrip`/
// `isOwner` below resolve `TripsRepository` directly through this,
@@ -537,6 +537,7 @@ export class TripsService {
}
});
if (vacayId !== undefined) await this.vacay.notifyPlanUsers(vacayId, undefined);
if (cover_image !== undefined) await this.release.releaseValues([trip.cover_image]); // kept while any row names it
const changes: Record<string, unknown> = {};
if (title && title !== trip.title) changes.title = title;
+7 -4
View File
@@ -283,12 +283,13 @@ export async function createPluginRpcHostParts(
await createTestPlacesRepo(db),
await createTestDayAssignmentsRepo(db),
await createTestBudgetItemsRepo(db),
{ releaseValues: async () => [] } as never, // StorageReleaseService
);
const collab = new CollabService(
permissions,
realtime,
notificationsStub(),
generalStorage,
{ releaseValues: async () => [] } as never, // StorageReleaseService
new RateLimitService(),
await createTestUnitOfWork(db),
await createTestCollabMessageReactionsRepo(db),
@@ -460,7 +461,7 @@ export async function createPluginRpcHostParts(
await createTestCategoriesRepo(db),
await createTestTripsRepo(db),
await createTestBudgetItemsRepo(db),
await createTestCollectionPlacesRepo(db),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
// After places: copying saved places into a trip runs the places journey hook.
@@ -468,7 +469,7 @@ export async function createPluginRpcHostParts(
permissions,
realtime,
notificationsStub(),
generalStorage,
{ releaseValues: async () => [] } as never, // StorageReleaseService
await createTestUnitOfWork(db),
// Plan 3h Task 1, the constructor-ripple fix: collections part A's own
// repositories, first cut (`CollectionMembersRepository` extended by
@@ -527,7 +528,8 @@ export async function createPluginRpcHostParts(
vacay,
realtime,
unsplash,
generalStorage,
// StorageReleaseService: a replaced cover's release has its own suites; nothing is released here.
{ releaseValues: async () => [] } as never,
await createTestUnitOfWork(db),
(await sharedTestOrm(db)).em,
new SettingsService(await createTestUnitOfWork(db), appSettings, await createTestSettingsRepo(db), membership),
@@ -551,6 +553,7 @@ export async function createPluginRpcHostParts(
await createTestShareTokensRepo(db),
await createTestPluginsRepo(db),
await createTestPluginUserErasureQueueRepo(db),
{ releaseValues: async () => [] } as never,
),
permissions,
realtime,
+59
View File
@@ -0,0 +1,59 @@
/**
* Stand-ins for StorageReleaseService in suites that hand-build an owner
* service. The release itself (asking every reference source, the ledger's
* guards) has its own suites: storage-release.service.test.ts and the
* integration suite over the full app.
*
* - `releaseNothing()` releases nothing and records the calls;
* - `releaseDirect(storage)` deletes every root-level object the values name,
* as a suite with no second row naming them would see the real one do.
*/
import { inReclaimScope } from '../../src/nest/storage/storage-reclaim.ledger';
import {
storageKeysIn,
type StorageKeyOptions,
type StorageKeyRef,
} from '../../src/nest/storage/storage-reclaim.types';
import type { StorageReleaseService } from '../../src/nest/storage/storage-release.service';
import type { StorageService } from '../../src/nest/storage/storage.service';
import type { ServedCategory } from '../../src/nest/storage/storage.types';
import { vi } from 'vitest';
export function releaseNothing() {
const releaseValues = vi.fn(async (): Promise<StorageKeyRef[]> => []);
const releaseIfUnreferenced = vi.fn(async (): Promise<string[]> => []);
return { releaseValues, releaseIfUnreferenced } as unknown as StorageReleaseService & {
releaseValues: typeof releaseValues;
releaseIfUnreferenced: typeof releaseIfUnreferenced;
};
}
export function releaseDirect(storage: StorageService): StorageReleaseService {
const releaseIfUnreferenced = async (category: ServedCategory, keys: readonly string[]): Promise<string[]> => {
const present = await Promise.all(
[...new Set(keys)].map(async (key) =>
inReclaimScope(key) && (await storage.stat(category, key).catch(() => null)) !== null ? key : null,
),
);
const gone = present.filter((key): key is string => key !== null);
await Promise.all(gone.map((key) => storage.delete(category, key)));
return gone;
};
const releaseValues = async (
values: ReadonlyArray<string | null | undefined>,
options: StorageKeyOptions = {},
): Promise<StorageKeyRef[]> => {
const byCategory = new Map<string, Set<string>>();
for (const ref of values.flatMap((value) => storageKeysIn(value, options))) {
byCategory.set(ref.category, (byCategory.get(ref.category) ?? new Set<string>()).add(ref.key));
}
const released = await Promise.all(
[...byCategory].map(async ([category, keys]) =>
(await releaseIfUnreferenced(category as ServedCategory, [...keys])).map((key) => ({ category, key })),
),
);
return released.flat();
};
return { releaseValues, releaseIfUnreferenced } as unknown as StorageReleaseService;
}
@@ -0,0 +1,159 @@
/**
* The immediate release over the full app (#2924): what a user's replace or
* delete leaves unnamed is gone right after the request, through every owner's
* real reference source, while an object another row still names (a trip
* copy, a journey, a saved place) stays. The objects are written straight into
* the test run's uploads folder under random names and taken back afterwards.
* RELI-001 to RELI-006.
*/
import { buildApp } from '../../src/bootstrap';
import { Users } from '../../src/db/entities/Users.entity';
import { authCookie } from '../helpers/auth';
import { makeCollection, makeCollectionPlace } from '../helpers/factories/collections';
import type { FactoryOrm } from '../helpers/factories/context';
import { makeJourney } from '../helpers/factories/journeys';
import { makePlace } from '../helpers/factories/places';
import { updateRows } from '../helpers/factories/rows';
import { makeTrip } from '../helpers/factories/trips';
import { makeUser } from '../helpers/factories/users';
import { MikroORM } from '@mikro-orm/core';
import type { INestApplication } from '@nestjs/common';
import type { Application } from 'express';
import { randomUUID } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import request from 'supertest';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
const UPLOADS = path.join(__dirname, '../../uploads');
const written: string[] = [];
/** An object of `category` under a fresh random name, as an upload would leave it. */
function stored(category: 'covers' | 'places' | 'avatars', ext = '.jpg'): string {
const name = `${randomUUID()}${ext}`;
const file = path.join(UPLOADS, category, name);
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, 'bytes');
written.push(file);
return name;
}
function exists(category: string, name: string): boolean {
return fs.existsSync(path.join(UPLOADS, category, name));
}
let nestApp: INestApplication;
let app: Application;
let orm: FactoryOrm;
beforeAll(async () => {
nestApp = await buildApp();
app = nestApp.getHttpAdapter().getInstance();
orm = nestApp.get(MikroORM);
});
afterAll(async () => {
for (const file of written) fs.rmSync(file, { force: true });
await nestApp.close();
});
describe('the immediate release of what a change left unnamed', () => {
it('RELI-001: a replaced trip cover goes at once, but not while a trip copy still shows it', async () => {
const { user } = await makeUser(orm);
const old = stored('covers');
const trip = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${old}` });
const copy = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${old}` });
const next = stored('covers');
const first = await request(app)
.put(`/api/trips/${trip.id}`)
.set('Cookie', authCookie(user.id))
.send({ cover_image: `/uploads/covers/${next}` });
expect(first.status).toBe(200);
expect(exists('covers', old)).toBe(true);
const second = await request(app)
.put(`/api/trips/${copy.id}`)
.set('Cookie', authCookie(user.id))
.send({ cover_image: `/uploads/covers/${next}` });
expect(second.status).toBe(200);
expect(exists('covers', old)).toBe(false);
expect(exists('covers', next)).toBe(true);
});
it('RELI-002: a deleted trip takes its cover along unless a journey inherited it', async () => {
const { user } = await makeUser(orm);
const alone = stored('covers');
const inherited = stored('covers');
const tripA = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${alone}` });
const tripB = await makeTrip(orm, user.id, { cover_image: `/uploads/covers/${inherited}` });
await makeJourney(orm, user.id, { cover_image: `covers/${inherited}` });
const deleteTrip = (id: number) => request(app).delete(`/api/trips/${id}`).set('Cookie', authCookie(user.id));
expect((await deleteTrip(tripA.id)).status).toBe(200);
expect((await deleteTrip(tripB.id)).status).toBe(200);
expect(exists('covers', alone)).toBe(false);
expect(exists('covers', inherited)).toBe(true);
});
it('RELI-003: a replaced place image goes, one a saved place still names stays', async () => {
const { user } = await makeUser(orm);
const trip = await makeTrip(orm, user.id);
const own = stored('places');
const saved = stored('places');
const placeA = await makePlace(orm, trip.id, { image_url: `/uploads/places/${own}` });
const placeB = await makePlace(orm, trip.id, { image_url: `/uploads/places/${saved}?v=2` });
const collection = await makeCollection(orm, user.id);
await makeCollectionPlace(orm, collection.id, user.id, {
image_url: `https://trip.example.test/uploads/places/${saved}`,
});
const clearImage = (id: number) =>
request(app)
.put(`/api/trips/${trip.id}/places/${id}`)
.set('Cookie', authCookie(user.id))
.send({ image_url: null });
expect((await clearImage(placeA.id)).status).toBe(200);
expect((await clearImage(placeB.id)).status).toBe(200);
expect(exists('places', own)).toBe(false);
expect(exists('places', saved)).toBe(true);
});
it('RELI-004: a deleted place takes its image along', async () => {
const { user } = await makeUser(orm);
const trip = await makeTrip(orm, user.id);
const image = stored('places');
const place = await makePlace(orm, trip.id, { image_url: `/uploads/places/${image}` });
const res = await request(app)
.delete(`/api/trips/${trip.id}/places/${place.id}`)
.set('Cookie', authCookie(user.id));
expect(res.status).toBe(200);
expect(exists('places', image)).toBe(false);
});
it('RELI-005: a deleted avatar goes; an avatar another account still names stays', async () => {
const { user } = await makeUser(orm);
const { user: other } = await makeUser(orm);
const own = stored('avatars', '.png');
const shared = stored('avatars', '.png');
await updateRows(orm, Users, { id: user.id }, { avatar: own });
await updateRows(orm, Users, { id: other.id }, { avatar: shared });
const res = await request(app).delete('/api/auth/avatar').set('Cookie', authCookie(user.id));
expect(res.status).toBe(200);
expect(exists('avatars', own)).toBe(false);
await updateRows(orm, Users, { id: user.id }, { avatar: shared });
await request(app).delete('/api/auth/avatar').set('Cookie', authCookie(user.id));
expect(exists('avatars', shared)).toBe(true);
});
it('RELI-006: the release never fails the request, whatever the stored value holds', async () => {
const { user } = await makeUser(orm);
const trip = await makeTrip(orm, user.id, { cover_image: '/uploads/covers/../../etc/passwd' });
const res = await request(app)
.put(`/api/trips/${trip.id}`)
.set('Cookie', authCookie(user.id))
.send({ cover_image: 'https://images.example.test/cover.jpg' });
expect(res.status).toBe(200);
});
});
+2 -3
View File
@@ -1301,7 +1301,7 @@ describe('Trip cover upload parity', () => {
expect(fsMod.existsSync(pathMod.join(coversDir, diskName))).toBe(true);
});
it('TRIP-P02 — re-upload keeps the previous cover file for the storage reclaim job (#2924)', async () => {
it('TRIP-P02 — re-upload deletes the previous cover file', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
@@ -1317,8 +1317,7 @@ describe('Trip cover upload parity', () => {
.set('Cookie', authCookie(user.id))
.attach('cover', FIXTURE_IMG, 'two.jpg');
expect(second.status).toBe(201);
// A trip copy or a journey may show the same object, so the replace leaves it.
expect(fsMod.existsSync(pathMod.join(coversDir, firstName))).toBe(true);
expect(fsMod.existsSync(pathMod.join(coversDir, firstName))).toBe(false);
});
it('TRIP-P03 — no file → 400 "No image uploaded"', async () => {
@@ -173,6 +173,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
);
auth = new AuthService(
permissions,
@@ -136,6 +136,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
);
auth = new AuthService(
permissions,
@@ -159,6 +159,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
),
mailerStub,
new EphemeralTokenService(),
@@ -127,11 +127,9 @@ async function linkOfFile(fileId: number) {
// deleteGuest routes through the SAME BudgetService domain SQL
// (removeUserFromBudgetItems) under test.
//
// Plan 3c Task 0b: `dbsEm` is resolved once, at the top of the `beforeAll`
// below — `canAccessTrip`/`isOwner`/`rosterUserIds`/`getPlaceWithTags`
// resolve `TripsRepository`/`TripMembersRepository`/`PlacesRepository`
// through it. Plan 4 Task 4: `DatabaseService` itself is gone —
// `UserCleanupService` now takes this `EntityManager` directly.
// Plan 3c Task 0b: `dbsEm` is resolved once, at the top of the `beforeAll` below — `canAccessTrip`/`isOwner`/
// `rosterUserIds`/`getPlaceWithTags` resolve `TripsRepository`/`TripMembersRepository`/`PlacesRepository`
// through it. Plan 4 Task 4: `DatabaseService` is gone; `UserCleanupService` takes this `EntityManager`.
let dbsEm: import('@mikro-orm/core').EntityManager | undefined;
let budget: BudgetService;
@@ -157,7 +155,7 @@ beforeAll(async () => {
);
membersSvc = new TripMembersService(
budget,
new UserCleanupService(new MaintenanceRepository(dbsEm!), budget, await createTestUnitOfWork(testDb), await createTestUsersRepo(testDb), await createTestTripMembersRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestBudgetSettlementsRepo(testDb), await createTestJourneyShareTokensRepo(testDb), await createTestJourneysRepo(testDb), await createTestJourneyEntriesRepo(testDb), await createTestJourneyContributorsRepo(testDb), await createTestShareTokensRepo(testDb), await createTestPluginsRepo(testDb), await createTestPluginUserErasureQueueRepo(testDb)),
new UserCleanupService(new MaintenanceRepository(dbsEm!), budget, await createTestUnitOfWork(testDb), await createTestUsersRepo(testDb), await createTestTripMembersRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestBudgetSettlementsRepo(testDb), await createTestJourneyShareTokensRepo(testDb), await createTestJourneysRepo(testDb), await createTestJourneyEntriesRepo(testDb), await createTestJourneyContributorsRepo(testDb), await createTestShareTokensRepo(testDb), await createTestPluginsRepo(testDb), await createTestPluginUserErasureQueueRepo(testDb), { releaseValues: () => [] } as never),
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
new RealtimeService(),
notificationsStub(),
+29 -21
View File
@@ -29,6 +29,7 @@ import type { TripWriter } from '../../../src/nest/common/trip-writer';
import type { NotificationsService } from '../../../src/nest/notifications/notifications.service';
import { PermissionsService } from '../../../src/nest/permissions/permissions.service';
import { RealtimeService } from '../../../src/nest/realtime/realtime.service';
import type { StorageReleaseService } from '../../../src/nest/storage/storage-release.service';
import {
createTestCollabNotesRepo,
createTestCollabMessageReactionsRepo,
@@ -41,7 +42,7 @@ import { createUser, createTrip } from '../../helpers/factories';
import { countRows, findRow, findRows, insertRow, updateRows } from '../../helpers/factories/rows';
import { FakeRealtimeService } from '../../helpers/fake-realtime';
import { notificationsStub } from '../../helpers/notifications';
import { makeStorageFixture } from '../../helpers/storage-fixture';
import { releaseNothing } from '../../helpers/storage-release';
import { resetTestDb } from '../../helpers/test-db';
import type { TestOrm } from '../../helpers/test-orm';
import {
@@ -101,7 +102,8 @@ vi.mock('../../../src/utils/ssrfGuard', () => {
};
});
const collabFx = makeStorageFixture('files/');
/** Where note and chat attachments go once their rows are gone; the release has its own suites. */
const collabRelease = releaseNothing();
const rateLimit = new RateLimitService();
let svc: CollabService;
// Named bindings (not only inside `svc`), the `files.service.test.ts`
@@ -115,7 +117,7 @@ let linksRepo: CollabLinksRepository;
let messagesRepo: CollabMessagesRepository;
async function buildCollabService(
storage = collabFx.storage,
release: StorageReleaseService = collabRelease,
rl = rateLimit,
notifications = notificationsStub(),
realtime: RealtimeService = new RealtimeService(),
@@ -126,7 +128,7 @@ async function buildCollabService(
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
realtime,
notifications,
storage,
release,
rl,
await createTestUnitOfWork(testDb),
await createTestCollabMessageReactionsRepo(testDb),
@@ -158,7 +160,7 @@ beforeAll(async () => {
});
/** A CollabService with its own preview cache and budget, for the tests that fill either. */
const freshSvc = async () => buildCollabService(collabFx.storage, new RateLimitService());
const freshSvc = async () => buildCollabService(collabRelease, new RateLimitService());
beforeEach(() => {
resetTestDb(testDb);
@@ -415,7 +417,7 @@ describe('postNote / postMessage', () => {
send,
actorName: vi.fn(async () => 'owner'),
} as unknown as NotificationsService;
const service = await buildCollabService(collabFx.storage, rateLimit, notifications, realtime);
const service = await buildCollabService(collabRelease, rateLimit, notifications, realtime);
return { service, realtime, send };
}
@@ -886,7 +888,7 @@ describe('linkPreview hardening', () => {
const limiter = new RateLimitService();
const check = vi.spyOn(limiter, 'check');
const service = await buildCollabService(collabFx.storage, limiter);
const service = await buildCollabService(collabRelease, limiter);
const all = Promise.all(Array.from({ length: 20 }, () => service.linkPreview('https://example.com/same', 9)));
release!();
const results = await all;
@@ -903,7 +905,7 @@ describe('linkPreview hardening', () => {
const fetchMock = stubFetch({ ok: true, text: async () => '<title>Eigenes Budget</title>' });
const limiter = new RateLimitService();
for (let i = 0; i < 60; i++) await limiter.check('collab_link_preview', '42', 60, 60_000, Date.now());
const service = await buildCollabService(collabFx.storage, limiter);
const service = await buildCollabService(collabRelease, limiter);
// 42 is first in line and out of budget; 43 arrives while 42's check runs.
const [spent, fresh] = await Promise.all([
@@ -985,24 +987,30 @@ describe('hardening', () => {
expect(await countRows(orm, CollabNotes, { id: note.id })).toBe(1);
});
it('COLLAB-SVC-036: a failing storage delete is swallowed — note + file deletes still succeed', async () => {
it('COLLAB-SVC-036a: a note file, a note and a message release their attachments only once their rows are gone', async () => {
const { user1, trip } = setup();
const failingStorage = { delete: vi.fn().mockRejectedValue(new Error('EACCES')) };
const failing = await buildCollabService(
failingStorage as unknown as import('../../../src/nest/storage/storage.service').StorageService,
);
const note = await failing.createNote(trip.id, user1.id, { title: 'Sticky file' });
const note = await svc.createNote(trip.id, user1.id, { title: 'With files' });
await attachToNote(trip.id, note.id, 'stuck.pdf', 'stuck.pdf');
const attachment = await findRow(orm, TripFiles, { note: note.id });
if (!attachment) throw new Error('the note attachment was not written');
const fileId = attachment.id;
collabRelease.releaseValues.mockClear();
collabRelease.releaseValues.mockImplementationOnce(async () => {
expect(await countRows(orm, TripFiles, { id: attachment.id })).toBe(0);
return [];
});
expect(await svc.deleteNoteFile(trip.id, note.id, attachment.id)).toBe(true);
expect(collabRelease.releaseValues).toHaveBeenLastCalledWith(['stuck.pdf'], { bare: 'files', basename: true });
expect(await failing.deleteNoteFile(trip.id, note.id, fileId)).toBe(true);
expect(await countRows(orm, TripFiles, { id: fileId })).toBe(0);
await attachToNote(trip.id, note.id, 'stuck2.pdf', 'stuck2.pdf');
expect(await failing.deleteNote(trip.id, note.id)).toBe(true);
expect(await countRows(orm, CollabNotes, { id: note.id })).toBe(0);
await attachToNote(trip.id, note.id, 'files/legacy.pdf', 'legacy.pdf');
collabRelease.releaseValues.mockImplementationOnce(async () => {
expect(await countRows(orm, CollabNotes, { id: note.id })).toBe(0);
return [];
});
expect(await svc.deleteNote(trip.id, note.id)).toBe(true);
expect(collabRelease.releaseValues).toHaveBeenLastCalledWith(['files/legacy.pdf'], {
bare: 'files',
basename: true,
});
});
it('COLLAB-SVC-036: getFormattedNoteById is trip-scoped and null-safe', async () => {
@@ -33,6 +33,7 @@ import { PermissionsService } from '../../../src/nest/permissions/permissions.se
import { CollectionsService } from '../../../src/nest/collections/collections.service';
import { buildPlaceImportService } from '../../helpers/place-import';
import { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
import { releaseDirect } from '../../helpers/storage-release';
import { makeStorageFixture } from '../../helpers/storage-fixture';
import { notificationsStub } from '../../helpers/notifications';
import {
@@ -125,7 +126,7 @@ beforeAll(async () => {
permissionsForSpy = new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb));
svc = new CollectionsService(
permissionsForSpy,
realtime, notificationsStub(notifSend), storageFx.storage, await createTestUnitOfWork(testDb),
realtime, notificationsStub(notifSend), releaseDirect(storageFx.storage), await createTestUnitOfWork(testDb),
await createTestCollectionsRepo(testDb), await createTestCollectionMembersRepo(testDb),
await createTestCollectionLabelsRepo(testDb), await createTestCategoriesRepo(testDb),
await createTestCollectionPlacesRepo(testDb), await createTestCollectionPlaceRatingsRepo(testDb),
@@ -846,7 +847,7 @@ describe('custom saved-place image', () => {
expect(updated.image_url).toBe('/uploads/places/col-set.jpg');
});
it('COLLECTIONS-SVC-061: setPlaceImage stores the url and leaves a replaced upload to the reclaim job', async () => {
it('COLLECTIONS-SVC-061: setPlaceImage stores the url and reclaims a replaced upload', async () => {
const u = createUser(testDb).user;
const col = await svc.createCollection(u.id, { name: 'Photos' });
const p = (await svc.savePlace(u.id, { collection_id: col.id, name: 'Pic' })).place!;
@@ -856,12 +857,10 @@ describe('custom saved-place image', () => {
const res = await svc.setPlaceImage(u.id, p.id, '/uploads/places/col-replace-b.jpg');
expect(res.image_url).toBe('/uploads/places/col-replace-b.jpg');
// A trip copy may still show it; the storage reclaim job decides (#2924).
expect(fs.existsSync(fileA)).toBe(true);
fs.unlinkSync(fileA);
expect(fs.existsSync(fileA)).toBe(false);
});
it('COLLECTIONS-SVC-062: deletePlace leaves the uploaded image to the reclaim job', async () => {
it('COLLECTIONS-SVC-062: deletePlace reclaims the uploaded image when unreferenced', async () => {
const u = createUser(testDb).user;
const col = await svc.createCollection(u.id, { name: 'Photos' });
const p = (await svc.savePlace(u.id, { collection_id: col.id, name: 'Pic', image_url: '/uploads/places/col-delete.jpg' })).place!;
@@ -869,8 +868,7 @@ describe('custom saved-place image', () => {
expect(fs.existsSync(fileA)).toBe(true);
await svc.deletePlace(u.id, p.id);
expect(fs.existsSync(fileA)).toBe(true);
fs.unlinkSync(fileA);
expect(fs.existsSync(fileA)).toBe(false);
});
});
@@ -165,7 +165,7 @@ beforeAll(async () => {
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
});
+31 -29
View File
@@ -46,6 +46,7 @@ import {
import { makeCollabMessage } from '../../helpers/factories/collab';
import { countRows, findRow, insertRow } from '../../helpers/factories/rows';
import { createTestTripFilesRepo, createTestFileLinksRepo, createTestBudgetItemsRepo } from '../../helpers/files-repos';
import { releaseNothing } from '../../helpers/storage-release';
import { resetTestDb } from '../../helpers/test-db';
import type { TestOrm } from '../../helpers/test-orm';
import {
@@ -93,6 +94,8 @@ const { consumeEphemeralToken } = vi.hoisted(() => ({ consumeEphemeralToken: vi.
vi.mock('../../../src/nest/auth-core/ephemeral-tokens', () => ({ consumeEphemeralToken }));
const storageDelete = vi.fn();
/** What a permanent delete hands on: the release asks every reference source, in its own suites. */
const release = releaseNothing();
const storagePut = vi.fn(async () => undefined);
const storageStub = {
delete: storageDelete,
@@ -143,6 +146,7 @@ beforeAll(async () => {
await createTestPlacesRepo(testDb),
await createTestDayAssignmentsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
release,
);
});
@@ -551,66 +555,64 @@ describe('toggleStarred / softDeleteFile / restoreFile', () => {
});
});
// ── permanentDeleteFile / emptyTrash (unlink-first semantics) ─────────────────
// ── permanentDeleteFile / emptyTrash (row first, then the release) ────────────
const FILE_KEYS = { bare: 'files', basename: true };
describe('permanentDeleteFile', () => {
it('FILE-SVC-022: deletes the storage object (idempotent on missing), then the DB row', async () => {
it('FILE-SVC-022: deletes the DB row, then hands its filename to the release, never to storage itself', async () => {
const { user, trip } = seedTrip();
const file = await makeFile(trip.id, user.id, { filename: 'on-disk.pdf' });
storageDelete.mockResolvedValue(undefined);
release.releaseValues.mockImplementationOnce(async () => {
expect(await svc.getFileById(file.id, trip.id)).toBeUndefined();
return [];
});
await svc.permanentDeleteFile((await svc.getFileById(file.id, trip.id))!);
expect(storageDelete).toHaveBeenCalledWith('files', 'on-disk.pdf');
expect(await svc.getFileById(file.id, trip.id)).toBeUndefined();
expect(release.releaseValues).toHaveBeenCalledWith(['on-disk.pdf'], FILE_KEYS);
expect(storageDelete).not.toHaveBeenCalled();
});
it('FILE-SVC-023: a delete failure logs [files], rethrows and keeps the DB row', async () => {
it('FILE-SVC-023: a row delete that fails releases nothing and keeps the row', async () => {
const { user, trip } = seedTrip();
const file = await makeFile(trip.id, user.id, { filename: 'stuck.pdf' });
const boom = new Error('EACCES');
storageDelete.mockRejectedValue(boom);
const err = vi.spyOn(console, 'error').mockImplementation(() => {});
await expect(svc.permanentDeleteFile((await svc.getFileById(file.id, trip.id))!)).rejects.toThrow('EACCES');
expect(err).toHaveBeenCalledWith('[files] unlink failed for stuck.pdf, keeping DB row:', boom);
const spy = vi.spyOn(tripFilesRepo, 'deleteById').mockRejectedValueOnce(new Error('SQLITE_BUSY'));
await expect(svc.permanentDeleteFile((await svc.getFileById(file.id, trip.id))!)).rejects.toThrow('SQLITE_BUSY');
expect(release.releaseValues).not.toHaveBeenCalled();
expect(await svc.getFileById(file.id, trip.id)).toBeDefined();
spy.mockRestore();
});
});
describe('emptyTrash', () => {
it('FILE-SVC-024: unlinks every trashed file and deletes their rows, returning the count', async () => {
it('FILE-SVC-024: deletes every trashed row, then releases their names, returning the count', async () => {
const { user, trip } = seedTrip();
const a = await makeFile(trip.id, user.id, { filename: 'a.pdf' });
const b = await makeFile(trip.id, user.id, { filename: 'b.pdf' });
const kept = await makeFile(trip.id, user.id, { filename: 'kept.pdf' });
await svc.softDeleteFile(a.id);
await svc.softDeleteFile(b.id);
storageDelete.mockResolvedValue(undefined);
await expect(svc.emptyTrash(trip.id)).resolves.toBe(2);
expect(await svc.listFiles(trip.id, true)).toEqual([]);
expect(await svc.getFileById(kept.id, trip.id)).toBeDefined();
const [names, options] = release.releaseValues.mock.calls[0] as unknown as [string[], unknown];
expect([...names].sort()).toEqual(['a.pdf', 'b.pdf']);
expect(options).toEqual(FILE_KEYS);
expect(storageDelete).not.toHaveBeenCalled();
});
it('FILE-SVC-025: a partial unlink failure keeps the failed row, swallows the error and counts only successes', async () => {
it('FILE-SVC-025: a legacy prefixed filename reaches the release as stored, the basename reading is its own', async () => {
const { user, trip } = seedTrip();
const good = await makeFile(trip.id, user.id, { filename: 'good.pdf' });
const bad = await makeFile(trip.id, user.id, { filename: 'bad.pdf' });
await svc.softDeleteFile(good.id);
await svc.softDeleteFile(bad.id);
const boom = new Error('EBUSY');
storageDelete.mockImplementation((_category: string, name: string) =>
name.includes('bad.pdf') ? Promise.reject(boom) : Promise.resolve(),
);
const err = vi.spyOn(console, 'error').mockImplementation(() => {});
const legacy = await makeFile(trip.id, user.id, { filename: 'uploads/files/legacy.pdf' });
await svc.softDeleteFile(legacy.id);
await expect(svc.emptyTrash(trip.id)).resolves.toBe(1);
expect(err).toHaveBeenCalledWith('[files] unlink failed for bad.pdf, keeping DB row:', boom);
expect(await svc.getFileById(good.id, trip.id)).toBeUndefined();
expect(await svc.getDeletedFile(bad.id, trip.id)).toBeDefined();
expect(release.releaseValues).toHaveBeenCalledWith(['uploads/files/legacy.pdf'], FILE_KEYS);
});
it('FILE-SVC-026: an empty trash resolves to 0 without touching storage', async () => {
it('FILE-SVC-026: an empty trash resolves to 0 without a release', async () => {
const { trip } = seedTrip();
await expect(svc.emptyTrash(trip.id)).resolves.toBe(0);
expect(release.releaseValues).not.toHaveBeenCalled();
expect(storageDelete).not.toHaveBeenCalled();
});
@@ -627,10 +629,10 @@ describe('emptyTrash', () => {
const { user, trip } = seedTrip();
const a = await makeFile(trip.id, user.id, { filename: 'a.pdf' });
await svc.softDeleteFile(a.id);
storageDelete.mockResolvedValue(undefined);
const spy = vi.spyOn(tripFilesRepo, 'deleteMany').mockRejectedValueOnce(new Error('boom'));
await expect(svc.emptyTrash(trip.id)).rejects.toThrow('boom');
expect(await countRows(orm, TripFiles, { id: a.id })).toBe(1);
expect(release.releaseValues).not.toHaveBeenCalled();
spy.mockRestore();
});
});
@@ -8,6 +8,9 @@ import type { JourneyPhotoLinksService } from '../../../src/nest/journey/journey
import { JourneyPublicController } from '../../../src/nest/journey/journey-public.controller';
import { JourneyController } from '../../../src/nest/journey/journey.controller';
import type { JourneyService } from '../../../src/nest/journey/journey.service';
import { journeyThumbName } from '../../../src/nest/memories/thumbnail.service';
import type { StorageKeyRef } from '../../../src/nest/storage/storage-reclaim.types';
import type { StorageReleaseService } from '../../../src/nest/storage/storage-release.service';
import type { StorageService } from '../../../src/nest/storage/storage.service';
import type { User } from '../../../src/types';
import { HttpException } from '@nestjs/common';
@@ -43,6 +46,18 @@ const captureBackfillStub = {
scheduleUpload,
} as unknown as JourneyPhotoCaptureService;
/**
* What a deleted photo hands on. By default every `journey/<name>` it names
* goes, as the real release answers when no other row names them; a case
* about a shared object answers what it keeps.
*/
const releaseValues = vi.fn(async (values: ReadonlyArray<string | null | undefined>): Promise<StorageKeyRef[]> =>
values
.filter((value): value is string => typeof value === 'string' && value.startsWith('journey/'))
.map((value) => ({ category: 'journey', key: value.slice('journey/'.length) })),
);
const releaseStub = { releaseValues } as unknown as StorageReleaseService;
/**
* Build the controller.
*
@@ -66,6 +81,7 @@ function ctl(
photoLinks as JourneyPhotoLinksService,
// The live env: a case about demo mode stubs DEMO_MODE, every other runs outside it.
new RuntimeEnvService(),
releaseStub,
);
}
@@ -705,14 +721,25 @@ describe('JourneyController', () => {
expect(updatePhoto).toHaveBeenCalledWith(7, 1, { caption: 'x' }, 'sock-1');
});
it('DELETE photo leaves the bytes to the storage reclaim job (#2924)', async () => {
it('DELETE photo releases its file and poster once the row is gone, and the thumb of a file that went', async () => {
(storageStub.delete as ReturnType<typeof vi.fn>).mockClear();
releaseValues.mockClear();
const deletePhoto = vi.fn().mockReturnValue({ id: 7, file_path: 'journey/a.jpg', thumbnail_path: 'journey/p.jpg' });
expect(await ctl(svc({ deletePhoto } as Partial<JourneyService>)).deletePhoto(user, '7')).toEqual({
success: true,
});
expect(deletePhoto).toHaveBeenCalledWith(7, 1);
// Another row may name the same object: the photo row going is what frees it.
expect(releaseValues).toHaveBeenCalledWith(['journey/a.jpg', 'journey/p.jpg'], { bare: 'journey' });
expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/a.jpg'));
});
it('DELETE photo keeps the thumb of a file another row still names, and of a path outside journey/', async () => {
(storageStub.delete as ReturnType<typeof vi.fn>).mockClear();
releaseValues.mockResolvedValueOnce([]);
const shared = vi.fn().mockReturnValue({ id: 8, file_path: 'journey/shared.jpg', thumbnail_path: null });
await ctl(svc({ deletePhoto: shared } as Partial<JourneyService>)).deletePhoto(user, '8');
const outside = vi.fn().mockReturnValue({ id: 9, file_path: 'elsewhere/x.jpg' });
await ctl(svc({ deletePhoto: outside } as Partial<JourneyService>)).deletePhoto(user, '9');
expect(storageStub.delete).not.toHaveBeenCalled();
});
@@ -743,7 +770,7 @@ describe('JourneyController', () => {
expect(await ctl(svc()).galleryProviderPhotos(user, '9', { provider: 'immich', asset_id: 'a' })).toEqual({ id: 3 });
});
it('DELETE gallery photo: 404, then leaves the bytes and the poster to the reclaim job', async () => {
it('DELETE gallery photo: 404, then releases the video, its poster and its thumb', async () => {
expect(
await thrownAsync(() =>
ctl(svc({ deleteGalleryPhoto: vi.fn().mockReturnValue(null) } as Partial<JourneyService>)).deleteGalleryPhoto(
@@ -753,6 +780,7 @@ describe('JourneyController', () => {
),
).toEqual({ status: 404, body: { error: 'Photo not found or not allowed' } });
(storageStub.delete as ReturnType<typeof vi.fn>).mockClear();
releaseValues.mockClear();
const deleteGalleryPhoto = vi
.fn()
.mockReturnValue({ id: 10, file_path: 'journey/clip.mp4', thumbnail_path: 'journey/poster.jpg' });
@@ -760,7 +788,13 @@ describe('JourneyController', () => {
await ctl(svc({ deleteGalleryPhoto } as Partial<JourneyService>)).deleteGalleryPhoto(user, '10'),
).toBeUndefined();
expect(deleteGalleryPhoto).toHaveBeenCalledWith(10, 1);
expect(storageStub.delete).not.toHaveBeenCalled();
expect(releaseValues).toHaveBeenCalledWith(['journey/clip.mp4', 'journey/poster.jpg'], { bare: 'journey' });
expect(storageStub.delete).toHaveBeenCalledWith('journey', journeyThumbName('journey/clip.mp4'));
// A thumb delete that finds nothing is swallowed.
(storageStub.delete as ReturnType<typeof vi.fn>).mockRejectedValueOnce(new Error('ENOENT'));
await expect(
ctl(svc({ deleteGalleryPhoto } as Partial<JourneyService>)).deleteGalleryPhoto(user, '10'),
).resolves.toBeUndefined();
});
it('PATCH /:id returns the updated journey on success', async () => {
@@ -186,6 +186,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
),
mailerStub,
new EphemeralTokenService(),
@@ -162,6 +162,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
),
mailerStub,
new EphemeralTokenService(),
@@ -48,7 +48,6 @@ import {
createTestTripsRepo,
sharedTestOrm,
} from '../../helpers/test-uow';
import { createTestCollectionPlacesRepo } from '../../helpers/test-uow';
import { describe, it, expect, vi, beforeAll, beforeEach } from 'vitest';
@@ -150,7 +149,7 @@ async function svc(searchNominatim: OsmClient['searchNominatim']): Promise<Place
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
}
+37 -53
View File
@@ -66,8 +66,6 @@ import { TrekPhotoRegistrationService } from '../../../src/nest/photos/trek-phot
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
import { TripPhotos } from '../../../src/db/entities/TripPhotos.entity';
import { BudgetItems } from '../../../src/db/entities/BudgetItems.entity';
import { CollectionPlaces } from '../../../src/db/entities/CollectionPlaces.entity';
import { Collections } from '../../../src/db/entities/Collections.entity';
import { DayAccommodations } from '../../../src/db/entities/DayAccommodations.entity';
import { DayAssignments } from '../../../src/db/entities/DayAssignments.entity';
import { FileLinks } from '../../../src/db/entities/FileLinks.entity';
@@ -85,7 +83,7 @@ import { makeStorageFixture } from '../../helpers/storage-fixture';
import { noGoogleQuota } from '../../helpers/google-quota';
import { createTestUnitOfWork, createTestAppSettingsRepo, createTestUsersRepo, createTestTagsRepo, createTestPlaceRatingsRepo, createTestAssignmentParticipantsRepo, createTestPlacesRepo, createTestTripMembersRepo, createTestDayAssignmentsRepo, createTestCategoriesRepo, createTestTripsRepo, sharedTestOrm } from '../../helpers/test-uow';
import { createTestBudgetItemsRepo } from '../../helpers/files-repos';
import { createTestCollectionPlacesRepo } from '../../helpers/test-uow';
import { releaseNothing } from '../../helpers/storage-release';
import {
createTestJourneysRepo, createTestJourneyContributorsRepo, createTestJourneyTripsRepo, createTestJourneyEntriesRepo,
createTestJourneyPhotosRepo, createTestJourneyEntryPhotosRepo,
@@ -129,6 +127,8 @@ const KML_FIXTURE = path.join(__dirname, '../../fixtures/test.kml');
* `maps` is a parameter because the enrichment cases hand in their own provider.
*/
const placesStorageFx = makeStorageFixture('');
/** What the place writes hand to the storage release, which has its own suites. */
const placesRelease = releaseNothing();
async function makePlacesService(
maps: MapsService = buildMapsService(photoCacheStub, noAppSettings, noUsers, {} as never, {} as never, noGoogleQuota),
@@ -157,7 +157,7 @@ async function makePlacesService(
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
placesRelease,
buildPlaceImportService(),
);
}
@@ -1498,68 +1498,52 @@ describe('importKmlPlaces — full stored row and atomicity (M2/L2)', () => {
});
});
// ── Custom place images stay for the storage reclaim job (#1136, #2924) ──────────────────────────────────────────
// ── Replaced and deleted place images go to the storage release (#1136, #2924) ─────────────
describe('custom place images', () => {
function writePlaceImage(name: string): string {
const filePath = path.join(placesStorageFx.root, name);
fs.writeFileSync(filePath, 'jpeg-bytes');
return filePath;
beforeEach(() => placesRelease.releaseValues.mockClear());
async function placeWithImage(image: string) {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const place = createPlace(testDb, trip.id, { name: 'Photo' }) as { id: number };
await updateRows(orm, Places, { id: place.id }, { image_url: image });
return { trip, place };
}
it('PLACE-SVC-046 — replacing image_url keeps the previous upload for the reclaim job', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any;
const fileA = writePlaceImage('svc-replace-a.jpg');
await updateRows(orm, Places, { id: place.id }, { image_url: '/uploads/places/svc-replace-a.jpg' });
expect(fs.existsSync(fileA)).toBe(true);
it('PLACE-SVC-046 — replacing image_url hands the previous value to the release once it is written', async () => {
const { trip, place } = await placeWithImage('/uploads/places/svc-replace-a.jpg');
await svc.update(String(trip.id), String(place.id), { image_url: '/uploads/places/svc-replace-b.jpg' });
expect(fs.existsSync(fileA)).toBe(true);
expect(placesRelease.releaseValues).toHaveBeenCalledWith(['/uploads/places/svc-replace-a.jpg']);
expect((await findRow(orm, Places, { id: place.id }))?.image_url).toBe('/uploads/places/svc-replace-b.jpg');
});
it('PLACE-SVC-047 — clearing image_url to null keeps the previous upload', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any;
const fileA = writePlaceImage('svc-clear.jpg');
await updateRows(orm, Places, { id: place.id }, { image_url: '/uploads/places/svc-clear.jpg' });
expect(fs.existsSync(fileA)).toBe(true);
it('PLACE-SVC-047 — clearing image_url releases the previous upload; a write leaving it alone releases nothing', async () => {
const { trip, place } = await placeWithImage('/uploads/places/svc-clear.jpg');
await svc.update(String(trip.id), String(place.id), { image_url: null } as any);
expect(fs.existsSync(fileA)).toBe(true);
expect(placesRelease.releaseValues).toHaveBeenLastCalledWith(['/uploads/places/svc-clear.jpg']);
await svc.update(String(trip.id), String(place.id), { name: 'Renamed' });
expect(placesRelease.releaseValues).toHaveBeenLastCalledWith([null]);
});
it('PLACE-SVC-048 — remove keeps the uploaded image', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const place = createPlace(testDb, trip.id, { name: 'Photo' }) as any;
const fileA = writePlaceImage('svc-delete.jpg');
await updateRows(orm, Places, { id: place.id }, { image_url: '/uploads/places/svc-delete.jpg' });
expect(fs.existsSync(fileA)).toBe(true);
it('PLACE-SVC-048 — remove and removeMany release the images of the places they deleted', async () => {
const { trip, place } = await placeWithImage('/uploads/places/svc-delete.jpg');
await svc.remove(String(trip.id), String(place.id));
expect(fs.existsSync(fileA)).toBe(true);
expect(placesRelease.releaseValues).toHaveBeenLastCalledWith(['/uploads/places/svc-delete.jpg']);
const second = await placeWithImage('/uploads/places/svc-many.jpg');
await svc.removeMany(String(second.trip.id), [second.place.id]);
expect(placesRelease.releaseValues).toHaveBeenLastCalledWith(['/uploads/places/svc-many.jpg']);
});
it('PLACE-SVC-049 — a collection_places reference keeps the file when the trip place is deleted', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const place = createPlace(testDb, trip.id, { name: 'Shared Photo' }) as any;
const fileA = writePlaceImage('svc-shared.jpg');
await updateRows(orm, Places, { id: place.id }, { image_url: '/uploads/places/svc-shared.jpg' });
// A saved-place in a collection holds the same uploaded file — the ref-count guard must protect it.
const col = await insertRow(orm, Collections, { owner: user.id, name: 'Saved' });
await insertRow(orm, CollectionPlaces, { collection: col, owner: user.id, name: 'Shared Photo', image_url: '/uploads/places/svc-shared.jpg' });
expect(fs.existsSync(fileA)).toBe(true);
await svc.remove(String(trip.id), String(place.id));
expect(fs.existsSync(fileA)).toBe(true);
// resetTestDb does not clear collections; drop what this test inserted and its file.
testDb.exec('DELETE FROM collection_places; DELETE FROM collections;');
fs.unlinkSync(fileA);
it('PLACE-SVC-049 — updateMany releases every image the bulk value replaced, after its transaction', async () => {
const { trip, place } = await placeWithImage('/uploads/places/svc-bulk-a.jpg');
const other = createPlace(testDb, trip.id, { name: 'Other' }) as { id: number };
await updateRows(orm, Places, { id: other.id }, { image_url: '/uploads/places/svc-bulk-b.jpg' });
await svc.updateMany(String(trip.id), [place.id, other.id], { image_url: '/uploads/places/svc-bulk-c.jpg' });
expect(placesRelease.releaseValues).toHaveBeenCalledWith([
'/uploads/places/svc-bulk-a.jpg',
'/uploads/places/svc-bulk-b.jpg',
]);
});
// ── Collaborative ratings (#1435) ──────────────────────────────────────────
@@ -0,0 +1,250 @@
/**
* StorageReleaseService (#2924): what a user's delete or replace left unnamed
* goes right after it, through the same sources and parser as the nightly
* pass, over a real StorageModule (a LocalDriver on a temp folder) with fake
* sources standing in for the owner domains. An object any row still names
* stays; the pass's guards hold; a failure never reaches the caller.
* RELS-001 to RELS-010.
*/
import { AppSettings } from '../../../../src/db/entities/AppSettings.entity';
import { logWarn } from '../../../../src/nest/audit/audit-log.logger';
import { StorageJobsService } from '../../../../src/nest/storage/storage-jobs.service';
import { StorageReclaimRegistryService } from '../../../../src/nest/storage/storage-reclaim-registry.service';
import {
STORAGE_RECLAIMABLE_CATEGORIES,
STORAGE_REFERENCE_SOURCE_IDS,
storageOwnerMarkerKey,
type StorageReferenceClaim,
type StorageReferenceSource,
type StorageReferenceSourceId,
} from '../../../../src/nest/storage/storage-reclaim.types';
import { StorageReleaseService } from '../../../../src/nest/storage/storage-release.service';
import { StorageModule } from '../../../../src/nest/storage/storage.module';
import { StorageService } from '../../../../src/nest/storage/storage.service';
import type { ServedCategory } from '../../../../src/nest/storage/storage.types';
import { createSnapshotTestDb } from '../../../helpers/db-mock';
import { deleteRows } from '../../../helpers/factories/rows';
import { setAppSetting } from '../../../helpers/factories/settings';
import { createTestModule, type TestModule } from '../../../helpers/test-module';
import { createTestOrm, type TestOrm } from '../../../helpers/test-orm';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { Readable } from 'node:stream';
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
vi.mock('../../../../src/nest/audit/audit-log.logger', async (importOriginal) => ({
...(await importOriginal<typeof import('../../../../src/nest/audit/audit-log.logger')>()),
logInfo: vi.fn(),
logDebug: vi.fn(),
logError: vi.fn(),
logWarn: vi.fn(),
}));
const db = createSnapshotTestDb();
let t: TestModule;
let orm: TestOrm;
let uploadsRoot: string;
let release: StorageReleaseService;
let storage: StorageService;
let references: StorageReclaimRegistryService;
/** What the fake sources still name, per category, lowercased. */
let named: Map<ServedCategory, Set<string>>;
let asked: Array<{ id: StorageReferenceSourceId; category: ServedCategory; keys: string[] }>;
let failWith: Error | null;
function fakeSource(id: StorageReferenceSourceId): StorageReferenceSource {
const categories: readonly ServedCategory[] =
id === 'auth' ? ['avatars'] : id === 'files' ? ['files'] : STORAGE_RECLAIMABLE_CATEGORIES;
return {
id,
claims: categories.map((category): StorageReferenceClaim => ({ category, prefix: '' })),
namedKeys: async () => {
throw new Error('a release never reads a whole category');
},
async referenced(category, keys) {
asked.push({ id, category, keys: [...keys] });
if (failWith) throw failWith;
const set = named.get(category) ?? new Set<string>();
return new Set(keys.filter((key) => set.has(key.toLowerCase())));
},
};
}
async function put(category: ServedCategory, key: string): Promise<void> {
await storage.put(category, key, Readable.from('bytes'));
}
function exists(category: string, key: string): boolean {
return fs.existsSync(path.join(uploadsRoot, category, key));
}
beforeAll(async () => {
uploadsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'trek-release-uploads-'));
orm = await createTestOrm(db);
await setAppSetting(
orm,
'storage.backends',
JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }]),
);
t = await createTestModule({ db, imports: [StorageModule], init: true });
release = t.get(StorageReleaseService);
storage = t.get(StorageService);
references = t.get(StorageReclaimRegistryService);
await storage.reloadConfig();
for (const id of STORAGE_REFERENCE_SOURCE_IDS) references.register(fakeSource(id));
});
beforeEach(() => {
vi.restoreAllMocks();
vi.mocked(logWarn).mockClear();
named = new Map();
asked = [];
failWith = null;
});
afterAll(async () => {
await t.close();
await deleteRows(orm, AppSettings, { key: 'storage.backends' });
await orm.close();
db.close();
fs.rmSync(uploadsRoot, { recursive: true, force: true });
});
const UUID = '6f1e2d3c-4b5a-4968-8776-655443322110';
describe('StorageReleaseService', () => {
it('RELS-001: deletes what no row names, keeps what one still does, and answers what went', async () => {
await put('covers', `${UUID}-old.jpg`);
await put('covers', `${UUID}-shared.jpg`);
named.set('covers', new Set([`${UUID}-shared.jpg`]));
const released = await release.releaseValues([`/uploads/covers/${UUID}-old.jpg`, `covers/${UUID}-shared.jpg`]);
expect(released).toEqual([{ category: 'covers', key: `${UUID}-old.jpg` }]);
expect(exists('covers', `${UUID}-old.jpg`)).toBe(false);
expect(exists('covers', `${UUID}-shared.jpg`)).toBe(true);
});
it('RELS-002: reads every spelling a column holds, and a bare name with its column options', async () => {
await put('places', `${UUID}.jpg`);
await put('avatars', `${UUID}.png`);
await put('files', `${UUID}. #4711`);
const released = await release.releaseValues([`https://trip.example.test/uploads/places/${UUID}.jpg?v=2`]);
expect(released).toEqual([{ category: 'places', key: `${UUID}.jpg` }]);
expect(await release.releaseValues([`${UUID}.png`], { bare: 'avatars' })).toEqual([
{ category: 'avatars', key: `${UUID}.png` },
]);
expect(await release.releaseValues([`legacy/${UUID}. #4711`], { bare: 'files', basename: true })).toEqual([
{ category: 'files', key: `${UUID}. #4711` },
]);
// Every source that claims the category was asked about the keys, narrowed, never about the whole category.
expect(
asked
.filter((question) => question.category === 'files')
.map((question) => question.id)
.sort(),
).toEqual(['collections', 'files', 'journey', 'photos', 'place-contrib', 'places', 'trips'].sort());
});
it('RELS-003: never touches a nested key, an owner marker or a category the ledger keeps', async () => {
await put('journey', `thumbs/${UUID}.jpg`);
await put('covers', storageOwnerMarkerKey(UUID));
await put('photos', `${UUID}.jpg`);
expect(
await release.releaseValues([
`/uploads/journey/thumbs/${UUID}.jpg`,
`/uploads/covers/${storageOwnerMarkerKey(UUID)}`,
`/uploads/photos/${UUID}.jpg`,
'https://elsewhere.example.test/x.jpg',
null,
]),
).toEqual([]);
expect(exists('journey', `thumbs/${UUID}.jpg`)).toBe(true);
expect(exists('covers', storageOwnerMarkerKey(UUID))).toBe(true);
expect(exists('photos', `${UUID}.jpg`)).toBe(true);
expect(asked).toEqual([]);
});
it('RELS-004: a category missing a required source is left to the nightly pass', async () => {
await put('avatars', `${UUID}-a.png`);
vi.spyOn(references, 'sources').mockReturnValue(references.sources().filter((source) => source.id !== 'auth'));
expect(await release.releaseIfUnreferenced('avatars', [`${UUID}-a.png`])).toEqual([]);
expect(exists('avatars', `${UUID}-a.png`)).toBe(true);
});
it('RELS-005: nothing goes while a restore runs', async () => {
await put('covers', `${UUID}-r.jpg`);
const jobs = t.get(StorageJobsService);
await jobs.beginRestore();
try {
expect(await release.releaseIfUnreferenced('covers', [`${UUID}-r.jpg`])).toEqual([]);
} finally {
jobs.endRestore();
}
expect(exists('covers', `${UUID}-r.jpg`)).toBe(true);
});
it('RELS-006: a category whose location overlaps another is left alone', async () => {
await put('covers', `${UUID}-s.jpg`);
await setAppSetting(
orm,
'storage.backends',
JSON.stringify([
{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } },
{ name: 'nested', type: 'local', options: { root: path.join(uploadsRoot, 'covers') } },
]),
);
await setAppSetting(orm, 'storage.categories', JSON.stringify({ 'photos-trek': 'nested' }));
await storage.reloadConfig();
try {
expect(await release.releaseIfUnreferenced('covers', [`${UUID}-s.jpg`])).toEqual([]);
expect(exists('covers', `${UUID}-s.jpg`)).toBe(true);
} finally {
await setAppSetting(
orm,
'storage.backends',
JSON.stringify([{ name: 'uploads-local', type: 'local', options: { root: uploadsRoot } }]),
);
await deleteRows(orm, AppSettings, { key: 'storage.categories' });
await storage.reloadConfig();
}
});
it('RELS-007: a source that fails keeps everything and never throws', async () => {
await put('covers', `${UUID}-f.jpg`);
failWith = new Error('places table locked');
await expect(release.releaseValues([`/uploads/covers/${UUID}-f.jpg`])).resolves.toEqual([]);
expect(exists('covers', `${UUID}-f.jpg`)).toBe(true);
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('places table locked'));
});
it('RELS-008: a delete that fails is logged and answered as kept, the others still go', async () => {
await put('covers', `${UUID}-d1.jpg`);
await put('covers', `${UUID}-d2.jpg`);
const realDelete = storage.delete.bind(storage);
vi.spyOn(storage, 'delete').mockImplementation(async (category, name) => {
if (name.endsWith('-d1.jpg')) throw new Error('permission denied');
await realDelete(category, name);
});
const released = await release.releaseIfUnreferenced('covers', [`${UUID}-d1.jpg`, `${UUID}-d2.jpg`]);
expect(released).toEqual([`${UUID}-d2.jpg`]);
expect(exists('covers', `${UUID}-d1.jpg`)).toBe(true);
expect(logWarn).toHaveBeenCalledWith(expect.stringContaining('permission denied'));
});
it('RELS-009: an object the facade keeps on delete (a demo) and one that is not there are not reported', async () => {
await put('covers', `${UUID}-k.jpg`);
const keeps = vi.spyOn(storage, 'keepsOnDelete').mockReturnValue(true);
expect(await release.releaseIfUnreferenced('covers', [`${UUID}-k.jpg`])).toEqual([]);
expect(exists('covers', `${UUID}-k.jpg`)).toBe(true);
keeps.mockRestore();
expect(await release.releaseIfUnreferenced('covers', [`${UUID}-missing.jpg`])).toEqual([]);
});
it('RELS-010: a key named in another spelling, case included, keeps its object', async () => {
await put('covers', `${UUID}-case.jpg`);
named.set('covers', new Set([`${UUID}-case.jpg`]));
expect(await release.releaseValues([`/uploads/covers/${UUID.toUpperCase()}-CASE.JPG`])).toEqual([]);
expect(exists('covers', `${UUID}-case.jpg`)).toBe(true);
});
});
@@ -57,7 +57,6 @@ import {
createTestDayAssignmentsRepo,
createTestCategoriesRepo,
createTestTripsRepo,
createTestCollectionPlacesRepo,
sharedTestOrm,
} from '../../helpers/test-uow';
import {
@@ -139,7 +138,7 @@ async function makePlacesService(): Promise<PlacesService> {
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
}
@@ -139,6 +139,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
),
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
realtime,
@@ -234,7 +234,7 @@ beforeAll(async () => {
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
membersSvc = new TripMembersService(
@@ -254,6 +254,7 @@ beforeAll(async () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
{ releaseValues: async () => [] } as never,
),
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
realtime,
@@ -319,7 +320,7 @@ const buildReadModel = async (tripsRepo: TripsRepository, roster: TripMembersSer
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
realtime,
notificationsStub(),
makeStorageFixture('').storage,
{ releaseValues: async () => [] } as never, // StorageReleaseService
new RateLimitService(),
await createTestUnitOfWork(testDb),
await createTestCollabMessageReactionsRepo(testDb),
@@ -360,6 +361,7 @@ const buildReadModel = async (tripsRepo: TripsRepository, roster: TripMembersSer
await createTestPlacesRepo(testDb),
await createTestDayAssignmentsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
),
);
@@ -3,6 +3,7 @@ import type { AuditService } from '../../../src/nest/audit/audit.service';
import type { CalendarService } from '../../../src/nest/calendar/calendar.service';
import { DomainError } from '../../../src/nest/common/domain-error';
import type { TripWriter } from '../../../src/nest/common/trip-writer';
import type { StorageReleaseService } from '../../../src/nest/storage/storage-release.service';
import type { TripReadModelService } from '../../../src/nest/trip-read-model/trip-read-model.service';
import type { TripListWriter, TripWritesService } from '../../../src/nest/trips/trip-writes.service';
import { TripsController, MAX_COVER_SIZE, TRIP_COVER_FILE_FILTER } from '../../../src/nest/trips/trips.controller';
@@ -54,6 +55,9 @@ const audit = { writeAudit } as unknown as AuditService;
const storageStub = {
put: vi.fn().mockResolvedValue(undefined),
} as unknown as import('../../../src/nest/storage/storage.service').StorageService;
// What a cover upload hands back for release; the release itself has its own suite.
const releaseValues = vi.fn().mockResolvedValue([]);
const releaseStub = { releaseValues } as unknown as StorageReleaseService;
/**
* TripWritesService over the stubbed TripsService: create, copy and delete go
* through it, so it hands each on to the stub and records the writer the route
@@ -87,6 +91,7 @@ const tc = (
(readModel ?? {}) as TripReadModelService,
storageStub,
writes as unknown as TripWritesService,
releaseStub,
);
function svc(o: Partial<TripsService> = {}): TripsService {
@@ -596,9 +601,13 @@ describe('TripsController (parity with the legacy /api/trips route)', async () =
} as Partial<TripsService>);
expect(await tc(s).cover(user, '9', file)).toEqual({ cover_image: '/uploads/covers/abc.jpg' });
expect(storageStub.put).toHaveBeenCalledWith('covers', 'abc.jpg', { tmpPath: undefined });
// The replaced cover is left to the storage reclaim job: a copy may share it (#2924).
expect(storageStub.put).toHaveBeenCalledTimes(1);
expect(updateCoverImage).toHaveBeenCalledWith('9', '/uploads/covers/abc.jpg');
// The replaced cover is released after the new one is stored: it goes unless a copy still names it (#2924).
expect(releaseValues).toHaveBeenCalledWith(['/old.jpg']);
expect(releaseValues.mock.invocationCallOrder[0]).toBeGreaterThan(
updateCoverImage.mock.invocationCallOrder[0] ?? 0,
);
});
it('403 in demo mode for a demo account', async () => {
+6 -6
View File
@@ -276,7 +276,7 @@ beforeAll(async () => {
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
await createTestCollectionPlacesRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
// SettingsService reads the trips a planner view may keep (#2639) through TripMembershipService,
@@ -298,12 +298,12 @@ beforeAll(async () => {
),
realtime,
undefined as never, // unsplash — not exercised here
coversFx.storage,
{ releaseValues: async () => [] } as never, // StorageReleaseService: the cover release has its own suite
await createTestUnitOfWork(testDb),
(await sharedTestOrm(testDb)).em,
new SettingsService(await createTestUnitOfWork(testDb), await createTestAppSettingsRepo(testDb), await createTestSettingsRepo(testDb), membership.get(TripMembershipService)),
);
membersSvc = new TripMembersService(budgetSvc, new UserCleanupService(new MaintenanceRepository(dbsEm!), budgetSvc, await createTestUnitOfWork(testDb), await createTestUsersRepo(testDb), await createTestTripMembersRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestBudgetSettlementsRepo(testDb), await createTestJourneyShareTokensRepo(testDb), await createTestJourneysRepo(testDb), await createTestJourneyEntriesRepo(testDb), await createTestJourneyContributorsRepo(testDb), await createTestShareTokensRepo(testDb), await createTestPluginsRepo(testDb), await createTestPluginUserErasureQueueRepo(testDb)), new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), realtime, notificationsStub(), await createTestUnitOfWork(testDb), await createTestTripsRepo(testDb), await createTestTripMembersRepo(testDb), await createTestUsersRepo(testDb));
membersSvc = new TripMembersService(budgetSvc, new UserCleanupService(new MaintenanceRepository(dbsEm!), budgetSvc, await createTestUnitOfWork(testDb), await createTestUsersRepo(testDb), await createTestTripMembersRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestBudgetSettlementsRepo(testDb), await createTestJourneyShareTokensRepo(testDb), await createTestJourneysRepo(testDb), await createTestJourneyEntriesRepo(testDb), await createTestJourneyContributorsRepo(testDb), await createTestShareTokensRepo(testDb), await createTestPluginsRepo(testDb), await createTestPluginUserErasureQueueRepo(testDb), { releaseValues: async () => [] } as never), new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), realtime, notificationsStub(), await createTestUnitOfWork(testDb), await createTestTripsRepo(testDb), await createTestTripMembersRepo(testDb), await createTestUsersRepo(testDb));
readModelSvc = new TripReadModelService(
await createTestTripsRepo(testDb), membersSvc, daysSvc, accommodationsSvc, budgetSvc,
new PackingService(
@@ -316,7 +316,7 @@ beforeAll(async () => {
new CollabService(
// Plan 4 Task 2 — CollabService's own DatabaseService param is gone:
// canAccessTrip now reads through the TripsRepository at the end.
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), realtime, notificationsStub(), coversFx.storage, new RateLimitService(), await createTestUnitOfWork(testDb),
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), realtime, notificationsStub(), { releaseValues: async () => [] } as never, new RateLimitService(), await createTestUnitOfWork(testDb),
await createTestCollabMessageReactionsRepo(testDb), await createTestCollabNotesRepo(testDb), await createTestCollabPollsRepo(testDb),
await createTestCollabPollVotesRepo(testDb), await createTestCollabLinksRepo(testDb), await createTestCollabMessagesRepo(testDb),
await createTestTripsRepo(testDb),
@@ -329,8 +329,7 @@ beforeAll(async () => {
await createTestTripMembersRepo(testDb),
),
new FilesService(
// Plan 4 Task 2 — FilesService's own canAccessTrip delegate is now
// TripsRepository.findAccessible, in the same constructor slot.
// Plan 4 Task 2 — canAccessTrip is TripsRepository.findAccessible, in the same constructor slot.
new TripAccessService(await createTestTripsRepo(testDb)),
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
realtime,
@@ -344,6 +343,7 @@ beforeAll(async () => {
await createTestPlacesRepo(testDb),
await createTestDayAssignmentsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
),
);
});
@@ -54,6 +54,7 @@ import {
createTestPluginsRepo,
createTestPluginUserErasureQueueRepo,
} from '../../helpers/share-repos';
import { releaseNothing } from '../../helpers/storage-release';
import { resetTestDb } from '../../helpers/test-db';
import type { TestOrm } from '../../helpers/test-orm';
import {
@@ -93,6 +94,8 @@ vi.mock('../../../src/nest/plugins/paths', () => ({ pluginsDataRoot: () => dataR
let em: EntityManager;
let budget: BudgetService;
let svc: UserCleanupService;
/** What a deletion hands on: the avatar, released by its own suites. */
const cleanupRelease = releaseNothing();
beforeAll(async () => {
// Plan 4 Task 4: UserCleanupService's own DatabaseService param is gone.
// UC1 goes through MaintenanceRepository, which Nest injects (built here
@@ -125,6 +128,7 @@ beforeAll(async () => {
// Plan 4 Task 8a constructor-ripple: UC2/UC3's repositories.
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
cleanupRelease,
);
});
@@ -159,6 +163,7 @@ const addEntry = (journeyId: number, authorId: number, type: string, title: stri
beforeEach(async () => {
resetTestDb(testDb);
cleanupRelease.releaseValues.mockClear();
// The plugin tables are not user data, so resetTestDb leaves them alone —
// these tests own them and must not leak rows into each other.
await deleteRows(orm, PluginUserErasureQueue);
@@ -261,6 +266,7 @@ describe('erasePluginUserData', () => {
// constructor-time failure.
await createTestPluginsRepo(slim),
await createTestPluginUserErasureQueueRepo(slim),
{ releaseValues: async () => [] } as never,
);
await expect(slimSvc.erasePluginUserData(1)).resolves.toBeUndefined();
@@ -431,13 +437,26 @@ describe('deleteUserCompletely', () => {
await createTestShareTokensRepo(testDb),
await createTestPluginsRepo(testDb),
await createTestPluginUserErasureQueueRepo(testDb),
cleanupRelease,
).deleteUserCompletely(victim.id),
).rejects.toThrow('boom');
expect(await findRow(orm, Users, { id: victim.id })).not.toBeNull();
expect((await findRow(orm, TripMembers, { user: owner.id }))?.invited_by).toBe(victim.id);
expect(cleanupRelease.releaseValues).not.toHaveBeenCalled();
} finally {
deleteByIdSpy.mockRestore();
}
});
it('USER-CLEANUP-010: hands the uploaded avatar to the release once the user row is gone', async () => {
const { user } = createUser(testDb, { username: 'with-avatar' });
await updateRows(orm, Users, { id: user.id }, { avatar: 'face.png' });
cleanupRelease.releaseValues.mockImplementationOnce(async () => {
expect(await findRow(orm, Users, { id: user.id })).toBeNull();
return [];
});
await svc.deleteUserCompletely(user.id);
expect(cleanupRelease.releaseValues).toHaveBeenCalledWith(['face.png'], { bare: 'avatars' });
});
});
@@ -12,6 +12,7 @@ import { updateRows } from '../../helpers/factories/rows';
import { readAppSetting, setAppSetting } from '../../helpers/factories/settings';
import { readUser } from '../../helpers/factories/users';
import { makeStorageFixture } from '../../helpers/storage-fixture';
import { releaseDirect } from '../../helpers/storage-release';
import { resetTestDb } from '../../helpers/test-db';
import {
createTestUnitOfWork,
@@ -64,7 +65,7 @@ let profile: UserProfileService;
beforeAll(async () => {
profile = new UserProfileService(
avatarsFx.storage,
releaseDirect(avatarsFx.storage),
await createTestUnitOfWork(testDb),
await createTestAppSettingsRepo(testDb),
await createTestUsersRepo(testDb),