mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
685506d067
The release queue narrows each owner's rows with LIKE '%run%' before it parses them, and relied on SQLite's LIKE ignoring ASCII case. On Postgres LIKE is exact, so a row naming an object in uppercase was not found and the release deleted an object that row still names. likeAny now matches LOWER(column) against the lowercase run, which reads alike on both engines. RELI-010 fails on Postgres without it.
268 lines
15 KiB
TypeScript
268 lines
15 KiB
TypeScript
/**
|
|
* The reference sources the owner domains hand the storage reclaim pass
|
|
* (#2924), each against the migrated schema: which of a page of keys its rows
|
|
* still name, in every spelling those columns hold. One object kept too many
|
|
* costs a few bytes; one called unreferenced while a row names it is lost, so
|
|
* every case here is a spelling a real row can carry. A source answers twice:
|
|
* `namedKeys`, every key of a category its rows name (one read, for a pass),
|
|
* and `referenced`, a few keys asked fresh and narrowed to the rows holding
|
|
* their plain runs (the re-check before a delete, and a release).
|
|
* RSRC-001 to RSRC-017.
|
|
*/
|
|
import { PlaceContribOutbox } from '../../../src/db/entities/PlaceContribOutbox.entity';
|
|
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
|
|
import { Users } from '../../../src/db/entities/Users.entity';
|
|
import { AuthStorageRefsService } from '../../../src/nest/auth/auth-storage-refs.service';
|
|
import { CollectionsStorageRefsService } from '../../../src/nest/collections/collections-storage-refs.service';
|
|
import { FilesStorageRefsService } from '../../../src/nest/files/files-storage-refs.service';
|
|
import { JourneyStorageRefsService } from '../../../src/nest/journey/journey-storage-refs.service';
|
|
import { PhotosStorageRefsService } from '../../../src/nest/photos/photos-storage-refs.service';
|
|
import { PlaceContribStorageRefsService } from '../../../src/nest/place-contrib/place-contrib-storage-refs.service';
|
|
import { PlacesStorageRefsService } from '../../../src/nest/places/places-storage-refs.service';
|
|
import { StorageReclaimRegistryService } from '../../../src/nest/storage/storage-reclaim-registry.service';
|
|
import { likeAny, MAX_LIKE_TERMS } from '../../../src/nest/storage/storage-reclaim-source.service';
|
|
import {
|
|
STORAGE_RECLAIMABLE_CATEGORIES,
|
|
STORAGE_REFERENCE_SOURCE_IDS,
|
|
type StorageReferenceSource,
|
|
} from '../../../src/nest/storage/storage-reclaim.types';
|
|
import type { ServedCategory } from '../../../src/nest/storage/storage.types';
|
|
import { TripsStorageRefsService } from '../../../src/nest/trips/trips-storage-refs.service';
|
|
import { createSnapshotTestDb } from '../../helpers/db-mock';
|
|
import { makeCollection, makeCollectionPlace } from '../../helpers/factories/collections';
|
|
import { makeTripFile } from '../../helpers/factories/files';
|
|
import { makeJourney } from '../../helpers/factories/journeys';
|
|
import { makePlace } from '../../helpers/factories/places';
|
|
import { insertRow, updateRows } from '../../helpers/factories/rows';
|
|
import { makeTrip } from '../../helpers/factories/trips';
|
|
import { makeUser } from '../../helpers/factories/users';
|
|
import { createTestModule, type TestModule } from '../../helpers/test-module';
|
|
import { createTestOrm, type TestOrm } from '../../helpers/test-orm';
|
|
import { SqlitePlatform } from '@mikro-orm/sql';
|
|
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
const SOURCES = [
|
|
TripsStorageRefsService,
|
|
CollectionsStorageRefsService,
|
|
PlacesStorageRefsService,
|
|
PlaceContribStorageRefsService,
|
|
JourneyStorageRefsService,
|
|
PhotosStorageRefsService,
|
|
FilesStorageRefsService,
|
|
AuthStorageRefsService,
|
|
];
|
|
|
|
const db = createSnapshotTestDb();
|
|
let orm: TestOrm;
|
|
let t: TestModule;
|
|
let userId: number;
|
|
let tripId: number;
|
|
|
|
beforeAll(async () => {
|
|
orm = await createTestOrm(db);
|
|
t = await createTestModule({ db, imports: [], providers: [StorageReclaimRegistryService, ...SOURCES], init: true });
|
|
userId = (await makeUser(orm)).user.id;
|
|
tripId = (await makeTrip(orm, userId)).id;
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await t.close();
|
|
await orm.close();
|
|
db.close();
|
|
});
|
|
|
|
/** The sorted keys a source still references. */
|
|
async function asked(source: StorageReferenceSource, category: ServedCategory, keys: string[]): Promise<string[]> {
|
|
return [...(await source.referenced(category, keys))].sort();
|
|
}
|
|
|
|
describe('reference sources', () => {
|
|
it('RSRC-001: every source registers itself once the app boots, under the ids the ledger knows', () => {
|
|
const registry = t.get(StorageReclaimRegistryService);
|
|
expect([...registry.ids()].sort()).toEqual([...STORAGE_REFERENCE_SOURCE_IDS].sort());
|
|
for (const source of registry.sources()) {
|
|
const claimed = source.claims.map((claim) => `${claim.category}:${claim.prefix}`).sort();
|
|
if (source.id === 'files') expect(claimed).toEqual(['files:']);
|
|
else if (source.id === 'auth') expect(claimed).toEqual(['avatars:']);
|
|
else expect(claimed).toEqual(STORAGE_RECLAIMABLE_CATEGORIES.map((category) => `${category}:`).sort());
|
|
}
|
|
});
|
|
|
|
it('RSRC-002: a trip cover keeps its object in every spelling, and in whatever category it points into', async () => {
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-a.jpg' });
|
|
await makeTrip(orm, userId, { cover_image: 'https://trip.example.test/uploads/places/rsrc-b.jpg?v=2' });
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Mixed.JPG' });
|
|
await makeTrip(orm, userId, { cover_image: 'https://images.unsplash.com/rsrc-z.jpg' });
|
|
const trips = t.get(TripsStorageRefsService);
|
|
expect(await asked(trips, 'covers', ['rsrc-a.jpg', 'rsrc-mixed.jpg', 'rsrc-z.jpg', 'rsrc-b.jpg'])).toEqual([
|
|
'rsrc-a.jpg',
|
|
'rsrc-mixed.jpg',
|
|
]);
|
|
expect(await asked(trips, 'places', ['rsrc-b.jpg', 'rsrc-a.jpg'])).toEqual(['rsrc-b.jpg']);
|
|
expect(await asked(trips, 'covers', [])).toEqual([]);
|
|
});
|
|
|
|
it('RSRC-003: a list cover and a saved place image both keep their objects', async () => {
|
|
const collection = await makeCollection(orm, userId, { cover_image: '/uploads/covers/rsrc-c.jpg' });
|
|
await makeCollectionPlace(orm, collection.id, userId, { image_url: '/uploads/places/rsrc-d.jpg' });
|
|
await makeCollectionPlace(orm, collection.id, userId, { image_url: 'https://example.test/rsrc-y.jpg' });
|
|
const collections = t.get(CollectionsStorageRefsService);
|
|
expect(await asked(collections, 'covers', ['rsrc-c.jpg', 'rsrc-d.jpg'])).toEqual(['rsrc-c.jpg']);
|
|
expect(await asked(collections, 'places', ['rsrc-d.jpg', 'rsrc-y.jpg'])).toEqual(['rsrc-d.jpg']);
|
|
});
|
|
|
|
it('RSRC-004: a place image keeps its object; a photo-proxy path or an inline image names none', async () => {
|
|
await makePlace(orm, tripId, { image_url: '/uploads/places/rsrc-e.jpg' });
|
|
await makePlace(orm, tripId, { image_url: '/uploads/covers/rsrc-e2.jpg' });
|
|
await makePlace(orm, tripId, { image_url: '/api/maps/place-photo/rsrc-x/bytes' });
|
|
await makePlace(orm, tripId, { image_url: 'data:image/png;base64,AAAA' });
|
|
const places = t.get(PlacesStorageRefsService);
|
|
expect(await asked(places, 'places', ['rsrc-e.jpg', 'rsrc-x', 'bytes'])).toEqual(['rsrc-e.jpg']);
|
|
expect(await asked(places, 'covers', ['rsrc-e2.jpg'])).toEqual(['rsrc-e2.jpg']);
|
|
});
|
|
|
|
it('RSRC-005: a queued contribution keeps the image it has yet to send', async () => {
|
|
const place = await makePlace(orm, tripId, { image_url: null });
|
|
await insertRow(orm, PlaceContribOutbox, {
|
|
contrib_key: 'osm:node/2924',
|
|
field: 'image',
|
|
value: '/uploads/places/rsrc-f.jpg',
|
|
place: place.id,
|
|
place_name: 'Queued',
|
|
cid: 'cid-rsrc-0001',
|
|
eligible_at: '2026-01-01 00:00:00',
|
|
next_attempt_at: '2026-01-01 00:00:00',
|
|
});
|
|
const outbox = t.get(PlaceContribStorageRefsService);
|
|
expect(await asked(outbox, 'places', ['rsrc-f.jpg', 'rsrc-f2.jpg'])).toEqual(['rsrc-f.jpg']);
|
|
});
|
|
|
|
it('RSRC-006: a journey cover keeps a shared trip cover and a gallery photo by their relative spelling', async () => {
|
|
await makeJourney(orm, userId, { cover_image: 'covers/rsrc-g.jpg' });
|
|
await makeJourney(orm, userId, { cover_image: 'journey/rsrc-h.jpg' });
|
|
await makeJourney(orm, userId, { cover_image: '/uploads/covers/rsrc-g2.jpg' });
|
|
const journeys = t.get(JourneyStorageRefsService);
|
|
expect(await asked(journeys, 'covers', ['rsrc-g.jpg', 'rsrc-g2.jpg', 'rsrc-h.jpg'])).toEqual([
|
|
'rsrc-g.jpg',
|
|
'rsrc-g2.jpg',
|
|
]);
|
|
expect(await asked(journeys, 'journey', ['rsrc-h.jpg', 'rsrc-g.jpg'])).toEqual(['rsrc-h.jpg']);
|
|
});
|
|
|
|
it('RSRC-007: every photo row with a path keeps it, whatever its provider and links', async () => {
|
|
// A local photo retargeted to a remote provider is still served from its bytes.
|
|
await insertRow(orm, TrekPhotos, {
|
|
provider: 'immich',
|
|
asset_id: 'rsrc-asset',
|
|
owner: userId,
|
|
file_path: 'journey/rsrc-i.jpg',
|
|
thumbnail_path: 'journey/rsrc-i-poster.jpg',
|
|
});
|
|
await insertRow(orm, TrekPhotos, { provider: 'local', owner: userId, file_path: 'legacy/rsrc-j.jpg' });
|
|
await insertRow(orm, TrekPhotos, { provider: 'immich', asset_id: 'rsrc-remote', owner: userId });
|
|
const photos = t.get(PhotosStorageRefsService);
|
|
expect(
|
|
await asked(photos, 'journey', ['rsrc-i.jpg', 'rsrc-i-poster.jpg', 'rsrc-j.jpg', 'rsrc-remote', 'rsrc-k.jpg']),
|
|
).toEqual(['rsrc-i-poster.jpg', 'rsrc-i.jpg', 'rsrc-j.jpg']);
|
|
expect(await asked(photos, 'covers', ['rsrc-i.jpg'])).toEqual([]);
|
|
});
|
|
|
|
it('RSRC-008: a trip file keeps its object, trashed or carrying a legacy path', async () => {
|
|
await makeTripFile(orm, tripId, { filename: 'rsrc-k.pdf' });
|
|
await makeTripFile(orm, tripId, { filename: 'rsrc-l.pdf', deleted_at: '2026-01-01 00:00:00' });
|
|
await makeTripFile(orm, tripId, { filename: 'files/rsrc-m.pdf' });
|
|
await makeTripFile(orm, tripId, { filename: '/uploads/files/rsrc-n.pdf' });
|
|
await makeTripFile(orm, tripId, { filename: 'uploads\\files\\rsrc-p.pdf' });
|
|
const files = t.get(FilesStorageRefsService);
|
|
expect(
|
|
await asked(files, 'files', ['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf', 'rsrc-o.pdf']),
|
|
).toEqual(['rsrc-k.pdf', 'rsrc-l.pdf', 'rsrc-m.pdf', 'rsrc-n.pdf', 'rsrc-p.pdf']);
|
|
});
|
|
|
|
it('RSRC-009: a source asked about a category it does not claim answers nothing', async () => {
|
|
await makeTripFile(orm, tripId, { filename: 'rsrc-q.jpg' });
|
|
expect(await asked(t.get(FilesStorageRefsService), 'covers', ['rsrc-q.jpg'])).toEqual([]);
|
|
expect(await asked(t.get(FilesStorageRefsService), 'files', [])).toEqual([]);
|
|
expect(await asked(t.get(AuthStorageRefsService), 'files', ['rsrc-q.jpg'])).toEqual([]);
|
|
});
|
|
|
|
it('RSRC-010: an uploaded avatar keeps its object; an OIDC picture names none', async () => {
|
|
const uploaded = (await makeUser(orm)).user;
|
|
const oidc = (await makeUser(orm)).user;
|
|
await updateRows(orm, Users, { id: uploaded.id }, { avatar: 'rsrc-r.png' });
|
|
await updateRows(orm, Users, { id: oidc.id }, { avatar: 'https://idp.example.test/pictures/rsrc-s.png' });
|
|
const auth = t.get(AuthStorageRefsService);
|
|
expect(await asked(auth, 'avatars', ['rsrc-r.png', 'rsrc-s.png'])).toEqual(['rsrc-r.png']);
|
|
expect(await asked(auth, 'avatars', [])).toEqual([]);
|
|
});
|
|
|
|
it('RSRC-011: every source asked with no keys answers nothing', async () => {
|
|
const answers = await Promise.all(
|
|
SOURCES.map((Source) => asked(t.get<StorageReferenceSource>(Source), 'covers', [])),
|
|
);
|
|
expect(answers).toEqual(SOURCES.map(() => []));
|
|
});
|
|
|
|
it('RSRC-012: a narrowed question still finds a uuid-named object in every spelling a row holds', async () => {
|
|
const id = '7b1f0c2e-5a4d-4e3f-8a9b-0c1d2e3f4a5b';
|
|
await makeTrip(orm, userId, {
|
|
cover_image: `https://trip.example.test/uploads/covers/${id.toUpperCase()}.JPG?v=3`,
|
|
});
|
|
await makePlace(orm, tripId, { image_url: `/uploads/places/${id}-p.jpg#frag` });
|
|
const trips = t.get(TripsStorageRefsService);
|
|
expect(await asked(trips, 'covers', [`${id}.jpg`, `${id}-other.jpg`])).toEqual([`${id}.jpg`]);
|
|
expect(await asked(t.get(PlacesStorageRefsService), 'places', [`${id}-p.jpg`])).toEqual([`${id}-p.jpg`]);
|
|
});
|
|
|
|
it('RSRC-013: namedKeys answers every key of the category its rows name, lowercased, in one read', async () => {
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/covers/RSRC-Named-A.jpg' });
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-named-b.jpg' });
|
|
const trips = t.get(TripsStorageRefsService);
|
|
const covers = await trips.namedKeys('covers');
|
|
expect(covers.has('rsrc-named-a.jpg')).toBe(true);
|
|
expect(covers.has('rsrc-named-b.jpg')).toBe(false);
|
|
expect((await trips.namedKeys('places')).has('rsrc-named-b.jpg')).toBe(true);
|
|
expect(await t.get(FilesStorageRefsService).namedKeys('covers')).toEqual(new Set());
|
|
});
|
|
|
|
it('RSRC-014: a trip file whose name holds a #, a ? or a trailing blank keeps its object both ways', async () => {
|
|
const id = '3c9d8e7f-6a5b-4c4d-9e3f-2a1b0c9d8e7f';
|
|
const names = [`${id}. #4711`, `${id}.c#`, `${id}.v2?final`, `${id}.pdf `];
|
|
await Promise.all(names.map((filename) => makeTripFile(orm, tripId, { filename })));
|
|
const files = t.get(FilesStorageRefsService);
|
|
expect(await asked(files, 'files', names)).toEqual([...names].sort());
|
|
const named = await files.namedKeys('files');
|
|
for (const name of names) expect(named.has(name.toLowerCase())).toBe(true);
|
|
});
|
|
|
|
it('RSRC-015: a question about many keys is split into reads of at most MAX_LIKE_TERMS terms each', async () => {
|
|
const ids = Array.from({ length: 60 }, (_unused, i) => `9d8c7b6a-5f4e-4d3c-8b2a-${String(i).padStart(12, '0')}`);
|
|
await Promise.all(ids.slice(0, 30).map((id) => makePlace(orm, tripId, { image_url: `/uploads/places/${id}.jpg` })));
|
|
const keys = ids.map((id) => `${id}.jpg`);
|
|
expect(await asked(t.get(PlacesStorageRefsService), 'places', keys)).toEqual(keys.slice(0, 30).sort());
|
|
// The photos source reads two columns, still within the bound.
|
|
expect(await asked(t.get(PhotosStorageRefsService), 'journey', keys)).toEqual([]);
|
|
});
|
|
|
|
it('RSRC-016: a filter past MAX_LIKE_TERMS terms is refused before it reaches SQLite', () => {
|
|
const runs = Array.from({ length: MAX_LIKE_TERMS + 1 }, (_unused, i) => `run-${String(i).padStart(8, '0')}`);
|
|
expect(() => likeAny<{ a: string }>(new SqlitePlatform(), ['a'], runs)).toThrow(/at most/);
|
|
expect(() => likeAny<{ a: string; b: string }>(new SqlitePlatform(), ['a', 'b'], runs.slice(0, 25))).not.toThrow();
|
|
expect(() => likeAny<{ a: string; b: string }>(new SqlitePlatform(), ['a', 'b'], runs.slice(0, 26))).toThrow(
|
|
/at most/,
|
|
);
|
|
});
|
|
|
|
it('RSRC-017: namedKeysByCategory reads once and answers every category the source claims, none it does not', async () => {
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/covers/rsrc-once-a.jpg' });
|
|
await makeTrip(orm, userId, { cover_image: '/uploads/places/rsrc-once-b.jpg' });
|
|
const answer = await t.get(TripsStorageRefsService).namedKeysByCategory(['covers', 'places', 'photos']);
|
|
expect(answer.get('covers')?.has('rsrc-once-a.jpg')).toBe(true);
|
|
expect(answer.get('covers')?.has('rsrc-once-b.jpg')).toBe(false);
|
|
expect(answer.get('places')?.has('rsrc-once-b.jpg')).toBe(true);
|
|
expect(answer.has('photos')).toBe(false);
|
|
expect(await t.get(FilesStorageRefsService).namedKeysByCategory(['covers'])).toEqual(new Map());
|
|
});
|
|
});
|