Files
TREK/server/tests/unit/nest/tours.gpx.atomic.test.ts
T
Maurice d17457de56 fix(storage): delete what a delete or a replace left unnamed right away again
Leaving every replaced cover, place image and deleted journey photo to the
nightly pass kept them publicly fetchable by URL for a week at least, and
for good while deletion was off. Owners now hand the values a delete or a
replace left behind to StorageReleaseService once their write committed.
It asks every registered reference source, with the same parser as the
nightly pass, and deletes at once what no row names any more: an object a
trip copy, a journey or a saved place still shows stays, which the old
inline deletes got wrong.

The release holds the pass's guards (reclaimable categories at their root
only, never an owner marker, never while a restore runs, never with a
required source missing or an overlapping location), never fails the
request and logs at debug. It runs for trip covers (update, upload, trip
delete), collection covers and saved-place images, place images (update,
bulk update, delete), journey covers and journey or gallery photos
(their thumbnail goes with a file that went), avatars (change, delete,
account deletion), trip files (permanent delete, empty trash) and collab
note and chat attachments. A trip file row now goes before its bytes, so a
failing backend no longer fails the delete. The objects of a deleted trip's
places and files are left to the nightly pass: their rows are other
domains'.

PlacesService drops the collection places repository nothing read any
more. Size, warning and boundary baselines follow the shrunk files.
2026-11-01 16:18:21 +01:00

503 lines
22 KiB
TypeScript

/**
* The tours-mode GPX import end to end through the real PlacesService and
* ToursService over the migrated per-worker DB: the places, their track
* colours and their tours facets commit in one transaction (PlacesService's
* importPreparedGpx nests in ToursService's as a savepoint), any failure on
* the way rolls all of it back, and the realtime events go out only after the
* commit. Failures are injected with SQLite triggers on the real tables.
*/
import { db as testDb } from '../../../src/db/database';
import { Places } from '../../../src/db/entities/Places.entity';
import { TourWaypoints } from '../../../src/db/entities/TourWaypoints.entity';
import { Tours } from '../../../src/db/entities/Tours.entity';
import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity';
import { TripPhotos } from '../../../src/db/entities/TripPhotos.entity';
import type { AppSettingsRepository } from '../../../src/db/repositories/AppSettings.repository';
import type { UsersRepository } from '../../../src/db/repositories/Users.repository';
import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.service';
import type { UnitOfWork } from '../../../src/nest/database/unit-of-work';
import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.service';
import { PermissionsService } from '../../../src/nest/permissions/permissions.service';
import { TrekPhotoRegistrationService } from '../../../src/nest/photos/trek-photo-registration.service';
import { gpxGeometryParser } from '../../../src/nest/place-import/gpx.codec';
import type { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service';
import { PlacesService } from '../../../src/nest/places/places.service';
import { QueryHelpersService } from '../../../src/nest/query-helpers/query-helpers.service';
import { RealtimeService } from '../../../src/nest/realtime/realtime.service';
import { ToursService } from '../../../src/nest/tours/tours.service';
import { UnsplashService } from '../../../src/nest/unsplash/unsplash.service';
import { accommodationsOver } from '../../helpers/accommodations-service';
import { createPlace, createTrip, createUser } from '../../helpers/factories';
import { countRows, findRows, updateRows } from '../../helpers/factories/rows';
import { noPaddleChecks } from '../../helpers/fake-paddle-checks';
import { tourStageDays } from '../../helpers/fake-tour-stage-days';
import { createTestBudgetItemsRepo } from '../../helpers/files-repos';
import { noGoogleQuota } from '../../helpers/google-quota';
import {
createTestJourneysRepo,
createTestJourneyContributorsRepo,
createTestJourneyTripsRepo,
createTestJourneyEntriesRepo,
createTestJourneyPhotosRepo,
createTestJourneyEntryPhotosRepo,
} from '../../helpers/journey-repos';
import { buildMapsService } from '../../helpers/maps-service';
import { buildPlaceImportService } from '../../helpers/place-import';
import { makeStorageFixture } from '../../helpers/storage-fixture';
import { resetTestDb } from '../../helpers/test-db';
import {
createTestUnitOfWork,
createTestAppSettingsRepo,
createTestUsersRepo,
createTestTagsRepo,
createTestPlaceRatingsRepo,
createTestAssignmentParticipantsRepo,
createTestPlacesRepo,
createTestTripMembersRepo,
createTestDayAssignmentsRepo,
createTestCategoriesRepo,
createTestTripsRepo,
sharedTestOrm,
} from '../../helpers/test-uow';
import {
createTestTourActivitiesRepo,
createTestToursRepo,
createTestTourStationVisitsRepo,
createTestTourTypesRepo,
createTestTourWaypointsRepo,
} from '../../helpers/tours-repos';
import type { EntityClass } from '@mikro-orm/core';
import { Logger } from '@nestjs/common';
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest';
vi.mock('../../../src/db/database', async () => {
const { createSnapshotTestDb, buildDbMock } = await import('../../helpers/db-mock');
return buildDbMock(createSnapshotTestDb());
});
const mixedGpx = Buffer.from(`<gpx>
<wpt lat="1" lon="2"><name>Ignored POI</name></wpt>
<rte><desc>Route description</desc><rtept lat="48" lon="11"/><rtept lat="48.01" lon="11.01"/></rte>
<trk><name>Ridge</name><desc>Track description</desc><trkseg>
<trkpt lat="49" lon="12"><ele>100</ele></trkpt>
<trkpt lat="49.01" lon="12.01"><ele>150</ele></trkpt>
</trkseg><trkseg><trkpt lat="49.02" lon="12.02"><ele>120</ele></trkpt></trkseg></trk>
</gpx>`);
const TRIGGERS = ['fail_facet', 'fail_place', 'fail_color', 'fail_adopt'];
/** The collaborator set the container hands PlacesService (the places.service.test.ts builder). */
async function makePlacesService(): Promise<PlacesService> {
const photoCacheStub = { removeIfUnreferenced: vi.fn() } as unknown as PlacePhotoCacheService;
const noAppSettings = { getValue: async () => null } as unknown as AppSettingsRepository;
const noUsers = { getApiKeyColumn: async () => null } as unknown as UsersRepository;
const storage = makeStorageFixture('').storage;
const orm = await sharedTestOrm(testDb);
return new PlacesService(
new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)),
new RealtimeService(),
buildMapsService(photoCacheStub, noAppSettings, noUsers, {} as never, {} as never, noGoogleQuota),
new QueryHelpersService(
await createTestTagsRepo(testDb),
await createTestPlaceRatingsRepo(testDb),
await createTestAssignmentParticipantsRepo(testDb),
),
new UnsplashService(
await createTestAppSettingsRepo(testDb),
await createTestUsersRepo(testDb),
new RuntimeEnvService(),
storage,
),
photoCacheStub,
new JourneyDomainService(
new RealtimeService(),
new TrekPhotoRegistrationService(
orm.repo(TrekPhotos),
orm.repo(TripPhotos),
await createTestJourneyPhotosRepo(testDb),
),
await createTestUnitOfWork(testDb),
await createTestJourneysRepo(testDb),
await createTestJourneyContributorsRepo(testDb),
await createTestJourneyTripsRepo(testDb),
await createTestJourneyEntriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestJourneyPhotosRepo(testDb),
await createTestJourneyEntryPhotosRepo(testDb),
await createTestPlacesRepo(testDb),
),
storage,
await accommodationsOver(testDb),
await createTestUnitOfWork(testDb),
await createTestPlacesRepo(testDb),
await createTestTagsRepo(testDb),
await createTestPlaceRatingsRepo(testDb),
await createTestTripMembersRepo(testDb),
await createTestDayAssignmentsRepo(testDb),
await createTestCategoriesRepo(testDb),
await createTestTripsRepo(testDb),
await createTestBudgetItemsRepo(testDb),
{ releaseValues: async () => [] } as never, // StorageReleaseService
buildPlaceImportService(),
);
}
let uow: UnitOfWork;
let places: PlacesService;
let tours: ToursService;
let tripId: string;
let broadcast: MockInstance<PlacesService['broadcast']>;
const countedTables = { places: Places, tours: Tours, tour_waypoints: TourWaypoints } as const;
const count = async (table: keyof typeof countedTables) =>
countRows(await sharedTestOrm(testDb), countedTables[table] as EntityClass<object>);
/**
* The broadcast probe reads mid-request, while the import may still hold the
* connection inside its transaction: it has to see the handle's own state
* synchronously, past the ORM's connection queue, which a raw read on the
* shared handle does.
*/
const countNow = (table: keyof typeof countedTables) =>
// test-sql-allow: a synchronous read on the shared handle mid-transaction is what this probe measures.
(testDb.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n;
beforeAll(async () => {
uow = await createTestUnitOfWork(testDb);
places = await makePlacesService();
tours = new ToursService(
uow,
places,
await createTestToursRepo(testDb),
await createTestTourTypesRepo(testDb),
await createTestTourWaypointsRepo(testDb),
await createTestTourStationVisitsRepo(testDb),
await createTestPlacesRepo(testDb),
await createTestTourActivitiesRepo(testDb),
buildPlaceImportService(),
noPaddleChecks(),
tourStageDays(),
);
});
beforeEach(async () => {
resetTestDb(testDb);
(await sharedTestOrm(testDb)).clear();
const { user } = createUser(testDb);
tripId = String(createTrip(testDb, user.id).id);
broadcast = vi.spyOn(places, 'broadcast').mockImplementation(() => {});
});
afterEach(() => {
for (const name of TRIGGERS) testDb.exec(`DROP TRIGGER IF EXISTS ${name}`);
vi.restoreAllMocks();
});
afterAll(() => {
testDb.close();
});
describe('Tours GPX atomic persistence and postcommit publication', () => {
async function expectEmpty() {
expect(await Promise.all((['places', 'tours', 'tour_waypoints'] as const).map((table) => count(table)))).toEqual([
0, 0, 0,
]);
expect(broadcast).not.toHaveBeenCalled();
}
it('commits routes, tracks, colors and facets once before publishing, without Planner waypoints', async () => {
const transactional = vi.spyOn(uow, 'transactional');
const seen: { inTransaction: boolean; tours: number }[] = [];
broadcast.mockImplementation(() => {
seen.push({ inTransaction: testDb.inTransaction, tours: countNow('tours') });
});
const result = (await tours.importGpxAsTour(tripId, mixedGpx, 'walk.gpx', 'socket'))!;
// The tours transaction, with importPreparedGpx and its colouring nested inside as savepoints.
expect(transactional.mock.calls.length).toBeGreaterThanOrEqual(2);
expect(result.tours).toHaveLength(2);
expect(result.skipped).toBe(0);
expect(result.caution).toBe(true);
expect(result.tours[0]).toMatchObject({
name: 'walk',
tour_type: 'hike',
max_hiking_difficulty: null,
planned: false,
has_waypoints: false,
caution: true,
match_confidence: 0.3,
duration: null,
planned_duration_minutes: null,
break_additional_minutes: null,
});
expect(result.tours[1]).toMatchObject({
name: 'Ridge',
elevation_gain: 50,
elevation_loss: 30,
caution: false,
match_confidence: 1,
});
const rows = (await findRows(await sharedTestOrm(testDb), Places, {}, { id: 'asc' })) as Array<{
id: number;
trip_id: number;
description: string;
route_geometry: string;
route_color: string;
}>;
expect(rows.map((row) => row.trip_id)).toEqual([Number(tripId), Number(tripId)]);
expect(rows.map((row) => row.description)).toEqual(['Route description', 'Track description']);
expect(JSON.parse(rows[0].route_geometry)).toEqual([
[48, 11],
[48.01, 11.01],
]);
expect(JSON.parse(rows[1].route_geometry)).toEqual([
[49, 12, 100],
[49.01, 12.01, 150],
[49.02, 12.02, 120],
]);
expect(new Set(rows.map((row) => row.route_color)).size).toBe(2);
expect(
(await findRows(await sharedTestOrm(testDb), Tours, {}, { place: 'asc' })).map((tour) => ({
place_id: tour.place_id,
tour_type: tour.tour_type,
duration: tour.duration,
planned_duration_minutes: tour.planned_duration_minutes,
break_additional_minutes: tour.break_additional_minutes,
})),
).toEqual(
rows.map((row) => ({
place_id: row.id,
tour_type: 'hike',
duration: null,
planned_duration_minutes: null,
break_additional_minutes: null,
})),
);
expect(await count('tour_waypoints')).toBe(0);
expect(seen).toEqual([
{ inTransaction: false, tours: 2 },
{ inTransaction: false, tours: 2 },
{ inTransaction: false, tours: 2 },
]);
expect(broadcast.mock.calls.map((call) => call[1])).toEqual(['tours:changed', 'place:created', 'place:created']);
expect(broadcast.mock.calls[0]).toEqual([
tripId,
'tours:changed',
{ placeIds: rows.map((row) => row.id) },
'socket',
]);
for (const call of broadcast.mock.calls.slice(1)) {
expect(call[3]).toBe('socket');
expect((call[2] as { place: { route_color: string } }).place.route_color).toBeTruthy();
}
expect(broadcast.mock.calls.slice(1).map((call) => (call[2] as { place: { id: number } }).place.id)).toEqual(
result.tours.map((tour) => tour.place_id),
);
});
it.each([1, 2])('rolls back every carrier and facet when facet %i fails', async (ordinal) => {
testDb.exec(`CREATE TRIGGER fail_facet BEFORE INSERT ON tours
WHEN (SELECT COUNT(*) FROM tours) = ${ordinal - 1}
BEGIN SELECT RAISE(ABORT, 'facet failure'); END;`);
await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('facet failure');
await expectEmpty();
});
it('rolls back earlier carriers when the later Place insert fails', async () => {
testDb.exec(`CREATE TRIGGER fail_place BEFORE INSERT ON places WHEN NEW.name = 'Ridge'
BEGIN SELECT RAISE(ABORT, 'place failure'); END;`);
await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('place failure');
await expectEmpty();
});
it('rolls the new places back too when taking over a track from before Tours fails (#2810)', async () => {
const legacy = createPlace(testDb, Number(tripId), { name: 'Ridge' }).id;
await updateRows(
await sharedTestOrm(testDb),
Places,
{ id: legacy },
{ route_geometry: '[[49,12,100],[49.01,12.01,150]]' },
);
testDb.exec(`CREATE TRIGGER fail_adopt BEFORE INSERT ON tours WHEN NEW.place_id = ${legacy}
BEGIN SELECT RAISE(ABORT, 'adopt failure'); END;`);
await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('adopt failure');
expect(await Promise.all([count('places'), count('tours')])).toEqual([1, 0]);
expect(broadcast).not.toHaveBeenCalled();
});
it('rolls back carriers and earlier colors when later coloring fails', async () => {
testDb.exec(`CREATE TRIGGER fail_color BEFORE UPDATE OF route_color ON places WHEN NEW.name = 'Ridge'
BEGIN SELECT RAISE(ABORT, 'color failure'); END;`);
await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('color failure');
await expectEmpty();
});
it.each([
'<not-gpx/>',
'<gpx/>',
'<gpx><wpt lat="1" lon="2"/></gpx>',
'<gpx><trk><trkseg><trkpt/></trkseg></trk></gpx>',
])('does not write or publish unusable input %s', async (xml) => {
const transactional = vi.spyOn(uow, 'transactional');
expect(await tours.importGpxAsTour(tripId, Buffer.from(xml))).toBeNull();
expect(transactional).not.toHaveBeenCalled();
await expectEmpty();
});
it('propagates parser failures before entering the transaction', async () => {
const transactional = vi.spyOn(uow, 'transactional');
vi.spyOn(gpxGeometryParser, 'parse').mockImplementationOnce(() => {
throw new Error('parser failure');
});
await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('parser failure');
expect(transactional).not.toHaveBeenCalled();
await expectEmpty();
});
it('keeps a committed response and attempts remaining events if publication throws', async () => {
const warn = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
broadcast.mockImplementationOnce(() => {
throw new Error('transport unavailable');
});
expect((await tours.importGpxAsTour(tripId, mixedGpx))?.tours).toHaveLength(2);
expect(await count('tours')).toBe(2);
expect(broadcast).toHaveBeenCalledTimes(3);
expect(warn).toHaveBeenCalledOnce();
});
it('keeps committed rows and success response if a place event fails', async () => {
const warn = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
broadcast.mockImplementation((_tripId, event) => {
if (event === 'place:created') throw new Error('transport unavailable');
});
expect((await tours.importGpxAsTour(tripId, mixedGpx))?.tours).toHaveLength(2);
expect(await count('places')).toBe(2);
expect(warn).toHaveBeenCalledTimes(2);
});
it('keeps committed rows and success response if the Tours invalidation event fails', async () => {
const warn = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => {});
broadcast.mockImplementation((_tripId, event) => {
if (event === 'tours:changed') throw new Error('invalidation unavailable');
});
const result = await tours.importGpxAsTour(tripId, mixedGpx);
expect(result?.tours).toHaveLength(2);
expect(await count('places')).toBe(2);
expect(await count('tours')).toBe(2);
expect(broadcast.mock.calls.map((call) => call[1])).toEqual(['tours:changed', 'place:created', 'place:created']);
expect(warn).toHaveBeenCalledOnce();
});
it('returns a successful skipped result for duplicate-only imports without publishing events', async () => {
expect((await tours.importGpxAsTour(tripId, mixedGpx))?.tours).toHaveLength(2);
const placeRows = async () =>
(await findRows(await sharedTestOrm(testDb), Places, {}, { id: 'asc' })).map((row) => ({
id: row.id,
trip_id: row.trip_id,
name: row.name,
}));
const tourFacets = async () =>
(await findRows(await sharedTestOrm(testDb), Tours, {}, { place: 'asc' })).map((tour) => ({
place_id: tour.place_id,
}));
const rowsBefore = await placeRows();
const facetsBefore = await tourFacets();
broadcast.mockClear();
expect(await tours.importGpxAsTour(tripId, mixedGpx)).toEqual({ tours: [], caution: false, skipped: 2 });
expect(await placeRows()).toEqual(rowsBefore);
expect(await tourFacets()).toEqual(facetsBefore);
expect(broadcast).not.toHaveBeenCalled();
});
it('imports only the new tracks and publishes only committed rows in a mixed result', async () => {
// The same Ridge as in the mixed file, line and all: a track is a duplicate only of one that runs alike (#2870).
const existing = Buffer.from(`<gpx><trk><name>Ridge</name><trkseg>
<trkpt lat="49" lon="12"><ele>100</ele></trkpt><trkpt lat="49.01" lon="12.01"><ele>150</ele></trkpt>
</trkseg><trkseg><trkpt lat="49.02" lon="12.02"><ele>120</ele></trkpt></trkseg></trk></gpx>`);
expect((await tours.importGpxAsTour(tripId, existing))?.tours).toHaveLength(1);
broadcast.mockClear();
const result = await tours.importGpxAsTour(tripId, mixedGpx, 'walk.gpx', 'socket');
expect(result?.tours).toHaveLength(1);
expect(result?.tours[0].name).toBe('walk');
expect(result?.skipped).toBe(1);
expect(await count('places')).toBe(2);
expect(await count('tours')).toBe(2);
const walkId = result!.tours[0].place_id;
expect(broadcast.mock.calls.map((call) => [call[1], call[2]])).toEqual([
['tours:changed', { placeIds: [walkId] }],
['place:created', expect.objectContaining({ place: expect.objectContaining({ id: walkId, name: 'walk' }) })],
]);
});
it('preserves single-track response and trip scoping', async () => {
const { user } = createUser(testDb);
const otherTripId = String(createTrip(testDb, user.id).id);
const single = Buffer.from(
'<gpx><trk><trkseg><trkpt lat="48" lon="11"/><trkpt lat="48.01" lon="11.01"/></trkseg></trk></gpx>',
);
const result = (await tours.importGpxAsTour(otherTripId, single, 'single.gpx'))!;
expect(result.tours).toHaveLength(1);
expect(result.tours[0].name).toBe('single');
expect(result.caution).toBe(true);
expect(
(await findRows(await sharedTestOrm(testDb), Places, {}, { id: 'asc' })).map((row) => ({ trip_id: row.trip_id })),
).toEqual([{ trip_id: Number(otherTripId) }]);
expect((await tours.listTours(otherTripId)).map((t) => t.place_id)).toEqual([result.tours[0].place_id]);
expect(await tours.listTours(tripId)).toEqual([]);
expect(broadcast.mock.calls.map((call) => call[1])).toEqual(['tours:changed', 'place:created']);
});
});
describe('Tours KML import through the same persistence (IO-02)', () => {
const twoLines = Buffer.from(`<kml><Document>
<Placemark><name>Coast</name><LineString><coordinates>11,48,0 11.01,48.01,0</coordinates></LineString></Placemark>
<Placemark><name>Ridge</name><LineString><altitudeMode>absolute</altitudeMode>
<coordinates>12,49,100 12.01,49.01,150</coordinates></LineString></Placemark>
<Placemark><name>Hut</name><Point><coordinates>12,49</coordinates></Point></Placemark>
</Document></kml>`);
it('stores each line as a tour of its own, heights only where the file has them', async () => {
const result = await tours.importGpxAsTour(tripId, twoLines, 'alps.kml', 'socket');
expect(result.tours.map((tour) => [tour.name, tour.origin, tour.elevation_gain])).toEqual([
['Coast', 'gpx', null],
['Ridge', 'gpx', 50],
]);
expect(result.caution).toBe(true);
expect(await count('places')).toBe(2);
expect(await count('tours')).toBe(2);
});
it('rolls every line back when a later one fails, and stores nothing', async () => {
testDb.exec(`CREATE TRIGGER fail_place BEFORE INSERT ON places WHEN NEW.name = 'Ridge'
BEGIN SELECT RAISE(ABORT, 'place failure'); END;`);
await expect(tours.importGpxAsTour(tripId, twoLines, 'alps.kml')).rejects.toThrow('place failure');
expect(await Promise.all((['places', 'tours', 'tour_waypoints'] as const).map((table) => count(table)))).toEqual([
0, 0, 0,
]);
expect(broadcast).not.toHaveBeenCalled();
});
it('refuses a broken file before any transaction', async () => {
const transactional = vi.spyOn(uow, 'transactional');
const broken = Buffer.from('<kml><Placemark></kml>');
await expect(tours.importGpxAsTour(tripId, broken, 'broken.kml')).rejects.toMatchObject({
code: 'tour_file_unreadable',
message: 'Malformed KML: invalid XML structure',
});
expect(transactional).not.toHaveBeenCalled();
expect(await count('places')).toBe(0);
});
});