feat(collections): a list as a file, so it can leave the instance (#2198)

Sharing a list works between people on the same TREK. This is the other
case: giving one to somebody who runs their own, or keeping a copy. Export
sits beside Edit and Share in the list header and downloads the open list;
Import sits beside New list in the rail, reads a file, shows what is in it
and makes a new list of it. Desktop only, which the route already decides.

What travels is what somebody wrote down: the list's name, colour and
description, its labels, and each place with its address, coordinates,
notes, status, contact details, category name and labels. A file is dull
JSON on purpose, so it can be read and edited by a person.

What does not travel is everything that only means something on the instance
it came from. Ids of every kind, since the receiver issues its own and a
category crosses over by name. Star ratings and members, because a rating is
an opinion somebody gave the people on that list. Provenance into trips,
which points at rows the receiver cannot see. And an image path under
/uploads, which resolves on the sender's server: only an absolute https URL
survives.

A file is a stranger's JSON and is read as one. It is parsed in the browser
that chose it and never uploaded; the parsed list goes through the same
contract on both sides; and a link that is not http(s) arrives as nothing
rather than as an anchor. Strict where it must refuse, forgiving where it
can drop: a file that does not say what it is never reaches the importer,
while one unreadable place inside it is skipped and counted, and one bad
field on a good place costs that field alone.

An import is always a new list. Importing into an existing one would mean
deciding what happens to a place that is already there, and every answer to
that is a merge nobody asked for.
This commit is contained in:
Maurice
2026-09-11 14:06:05 +02:00
committed by Maurice
parent a8e4c45576
commit 2c06c9904b
43 changed files with 1803 additions and 16 deletions
@@ -51,6 +51,7 @@ import {
CollectionLabelCreateDto,
CollectionLabelUpdateDto,
CollectionLabelAssignDto,
CollectionImportDto,
} from './collections.dto';
import { PlaceRatingDto } from '../places/places.dto';
@@ -199,6 +200,17 @@ export class CollectionsController {
}
// ── Copy to trip ────────────────────────────────────────────────────────────
/**
* Read a list file back as a new list of the caller's own (#2198).
*
* 201, like POST / above, because it creates a list. The other POSTs here
* are @HttpCode(200) because they act on one that already exists.
*/
@Post('import')
importCollection(@CurrentUser() user: User, @Body() body: CollectionImportDto) {
return this.collections.importCollection(user.id, body);
}
@Post('copy-to-trip')
@HttpCode(200)
copyToTrip(@CurrentUser() user: User, @Body() body: CollectionCopyToTripDto) {
@@ -370,6 +382,16 @@ export class CollectionsController {
/** Preview for the bulk trip import: the trip's places plus, per place, the same
* duplicate verdict the import itself applies. Read-only, so the dialog can grey out
* what would be skipped instead of reporting it afterwards. */
/**
* The list as a file (#2198). A plain JSON body: the browser turns it into a
* download, so there is no Content-Disposition to get wrong here and no
* filename decided by the server.
*/
@Get(':id/export')
exportCollection(@CurrentUser() user: User, @Param('id') id: string) {
return this.collections.exportCollection(user.id, Number(id));
}
@Get(':id/importable/:tripId')
importable(@CurrentUser() user: User, @Param('id') id: string, @Param('tripId') tripId: string) {
return this.collections.importablePlaces(user.id, Number(id), Number(tripId));
@@ -20,6 +20,7 @@ import {
collectionLabelCreateRequestSchema,
collectionLabelUpdateRequestSchema,
collectionLabelAssignRequestSchema,
collectionImportRequestSchema,
} from '@trek/shared';
/**
@@ -49,3 +50,4 @@ export class CollectionSetMemberRoleDto extends createZodDto(collectionSetMember
export class CollectionLabelCreateDto extends createZodDto(collectionLabelCreateRequestSchema) {}
export class CollectionLabelUpdateDto extends createZodDto(collectionLabelUpdateRequestSchema) {}
export class CollectionLabelAssignDto extends createZodDto(collectionLabelAssignRequestSchema) {}
export class CollectionImportDto extends createZodDto(collectionImportRequestSchema) {}
@@ -12,7 +12,19 @@ import {
trackInsertedInDedupSet,
type DedupSet,
} from '../places/places.helpers';
import { placeMatchStrategies, type PlaceMatchCandidate } from '@trek/shared';
import {
placeMatchStrategies,
collectionFilePlaceSchema,
COLLECTION_FILE_FORMAT,
COLLECTION_FILE_VERSION,
MAX_COLLECTION_FILE_LABELS,
type PlaceMatchCandidate,
type CollectionFile,
type CollectionFileLabel,
type CollectionFilePlace,
type CollectionImportRequest,
type CollectionImportResult,
} from '@trek/shared';
import type {
Collection,
CollectionDetailResponse,
@@ -339,6 +351,170 @@ export class CollectionsService {
};
}
// -------------------------------------------------------------------------
// Export / import as a file (#2198)
// -------------------------------------------------------------------------
/**
* A list as a portable file.
*
* Built here rather than in the browser from what the page happens to hold,
* for two reasons: the read is access-checked like every other read, and the
* decision about what may leave the instance is one decision in one place
* instead of whatever the client forgot to strip. The contract in
* `collection-file.schema.ts` says what travels and why.
*
* Any member may export. A list is shared with somebody so they can use it,
* and a viewer who can read all of this on screen loses nothing by having it
* as a file; what a viewer must not do is write, which no export does.
*/
exportCollection(userId: number, id: number): CollectionFile {
this.assertAccess(userId, id);
const collection = this.getCollectionRow(id);
const labels = this.loadLabelsByCollection(id);
const labelNameById = new Map(labels.map(l => [l.id, l.name]));
const rows = this.db.all<PlaceRow>(`
SELECT cp.*, c.name AS category_name, c.color AS category_color, c.icon AS category_icon
FROM collection_places cp
LEFT JOIN categories c ON cp.category_id = c.id
WHERE cp.collection_id = ?
ORDER BY cp.sort_order, cp.created_at
`, id);
const labelIdsByPlace = this.loadLabelIdsByPlaceIds(rows.map(r => r.id));
const places: CollectionFilePlace[] = rows.map(row => ({
name: row.name,
description: row.description ?? null,
lat: row.lat ?? null,
lng: row.lng ?? null,
address: row.address ?? null,
notes: row.notes ?? null,
price: row.price ?? null,
currency: row.currency ?? null,
website: row.website ?? null,
phone: row.phone ?? null,
// Only an absolute https URL survives: a /uploads path resolves on this
// server, not the reader's. See the note in the file contract.
image_url: typeof row.image_url === 'string' && /^https:\/\//i.test(row.image_url) ? row.image_url : null,
google_place_id: row.google_place_id ?? null,
google_ftid: row.google_ftid ?? null,
osm_id: row.osm_id ?? null,
status: row.status,
links: parseLinks((row as { links?: unknown }).links),
category: row.category_name ?? null,
labels: (labelIdsByPlace[row.id] || []).map(lid => labelNameById.get(lid)).filter((n): n is string => !!n),
}));
return {
format: COLLECTION_FILE_FORMAT,
version: COLLECTION_FILE_VERSION,
name: collection.name,
description: collection.description ?? null,
color: collection.color ?? null,
icon: collection.icon ?? null,
exported_at: new Date().toISOString(),
labels: labels.map(l => ({ name: l.name, color: l.color ?? null })),
places,
};
}
/**
* Read a file back as a new list of the caller's own.
*
* Always a new list. Importing into an existing one would mean deciding what
* happens to a place that is already there, and every answer to that is a
* merge somebody did not ask for; a fresh list is a thing they can look at
* and then move places out of, which the list UI already does well.
*
* The places go in through the same INSERT the rest of the service uses,
* inside one transaction, so a file that fails halfway leaves nothing
* behind. Each place is re-validated against the file contract on the way
* in — the body was validated once at the pipe, and this is the second pass
* that lets one bad row be dropped instead of failing the whole import.
*/
importCollection(userId: number, body: CollectionImportRequest): CollectionImportResult {
const file = body.file;
const name = (body.name ?? file.name).trim().slice(0, 120) || file.name;
// The palette is instance-wide and read-only here: a file names a category,
// it does not get to create one.
const categoryIdByName = new Map<string, number>();
for (const c of this.db.all<{ id: number; name: string }>('SELECT id, name FROM categories')) {
categoryIdByName.set(c.name.trim().toLowerCase(), c.id);
}
const result = this.db.transaction(() => {
const collection = this.createCollection(userId, {
name,
description: file.description ?? null,
color: file.color ?? undefined,
icon: file.icon ?? undefined,
});
const labelIdByName = new Map<string, number>();
for (const label of (file.labels ?? []).slice(0, MAX_COLLECTION_FILE_LABELS)) {
const key = label.name.trim().toLowerCase();
if (!key || labelIdByName.has(key)) continue;
labelIdByName.set(key, this.insertImportedLabel(collection.id, label, labelIdByName.size));
}
const insertPlace = this.db.prepare(`
INSERT INTO collection_places (
collection_id, owner_id, saved_by, name, description, lat, lng, address,
category_id, price, currency, notes, image_url, google_place_id, google_ftid,
osm_id, website, phone, status, links, sort_order
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
const assignLabel = this.db.prepare('INSERT OR IGNORE INTO collection_place_labels (collection_place_id, label_id) VALUES (?, ?)');
let imported = 0;
let skipped = 0;
for (const raw of file.places) {
const parsed = collectionFilePlaceSchema.safeParse(raw);
if (!parsed.success) { skipped += 1; continue; }
const place = parsed.data;
const res = insertPlace.run(
collection.id, userId, userId,
place.name, place.description ?? null, place.lat ?? null, place.lng ?? null, place.address ?? null,
place.category ? (categoryIdByName.get(place.category.trim().toLowerCase()) ?? null) : null,
place.price ?? null, place.currency ?? null, place.notes ?? null,
place.image_url ?? null, place.google_place_id ?? null, place.google_ftid ?? null,
place.osm_id ?? null, place.website ?? null, place.phone ?? null,
place.status ?? 'idea', serializeLinks(place.links), imported,
);
const placeId = Number(res.lastInsertRowid);
for (const labelName of place.labels ?? []) {
const labelId = labelIdByName.get(labelName.trim().toLowerCase());
if (labelId) assignLabel.run(placeId, labelId);
}
imported += 1;
}
return { collectionId: collection.id, imported, skipped };
});
const collection = this.getCollectionRow(result.collectionId);
return {
collection: { ...collection, is_owner: true, labels: this.loadLabelsByCollection(result.collectionId) },
imported: result.imported,
skipped: result.skipped,
};
}
/**
* A label straight from a file, without createLabel's duplicate check.
*
* The caller already de-duplicates by name, the list is empty, and
* createLabel would notify the (nonexistent) members once per label.
*/
private insertImportedLabel(collectionId: number, label: CollectionFileLabel, sortOrder: number): number {
const res = this.db.run(
'INSERT INTO collection_labels (collection_id, name, color, sort_order) VALUES (?, ?, ?, ?)',
collectionId, label.name.trim(), label.color ?? '#6366f1', sortOrder,
);
return Number(res.lastInsertRowid);
}
createCollection(userId: number, body: CollectionCreateRequest): Collection {
const max = this.db.get<{ m: number }>('SELECT COALESCE(MAX(sort_order), -1) AS m FROM collections WHERE owner_id = ?', userId)!.m;
const result = this.db.run(`
@@ -1050,3 +1050,236 @@ describe('bulk status', () => {
expect(svc.findMembership(owner.id, { google_place_id: 'gp-9' }).lists[0].can_edit).toBe(true);
});
});
// ── Export / import as a file (#2198) ────────────────────────────────────────
describe('exportCollection / importCollection (#2198)', () => {
/** A list with two labels, a category and one place carrying both. */
function seedList(ownerId: number) {
const cat = createCategory(testDb, { name: 'Restaurant' });
const col = svc.createCollection(ownerId, { name: 'Lisbon', description: 'Three days', color: '#ef4444' });
const must = svc.createLabel(ownerId, col.id, 'Must see', '#ff0000');
const rain = svc.createLabel(ownerId, col.id, 'Rainy day', '#00ff00');
const market = svc.savePlace(ownerId, {
collection_id: col.id, name: 'Time Out Market', description: 'Market hall',
lat: 38.7071, lng: -9.1459, address: 'Av. 24 de Julho 49', notes: 'Before noon',
website: 'https://timeoutmarket.com/', phone: '+351 210 606 040', osm_id: 'node/1',
status: 'want', category_id: cat.id, price: 12, currency: 'EUR',
}).place!;
const view = svc.savePlace(ownerId, { collection_id: col.id, name: 'Miradouro', lat: 38.7195, lng: -9.1327 }).place!;
svc.assignLabels(ownerId, [must.id], [market.id], false);
svc.assignLabels(ownerId, [rain.id], [view.id], false);
return { col, cat, must, rain, market, view };
}
it('COLLECTIONS-SVC-100: a file carries the list, its labels and each place with its label names', () => {
const owner = createUser(testDb).user;
const { col, market } = seedList(owner.id);
const file = svc.exportCollection(owner.id, col.id);
expect(file.format).toBe('trek.collection');
expect(file.version).toBe(1);
expect(file).toMatchObject({ name: 'Lisbon', description: 'Three days', color: '#ef4444' });
expect(file.labels).toEqual([
{ name: 'Must see', color: '#ff0000' },
{ name: 'Rainy day', color: '#00ff00' },
]);
expect(file.places).toHaveLength(2);
const first = file.places[0] as Record<string, unknown>;
expect(first).toMatchObject({
name: 'Time Out Market', description: 'Market hall', lat: 38.7071, lng: -9.1459,
address: 'Av. 24 de Julho 49', notes: 'Before noon', website: 'https://timeoutmarket.com/',
phone: '+351 210 606 040', osm_id: 'node/1', status: 'want', price: 12, currency: 'EUR',
category: 'Restaurant', labels: ['Must see'],
});
expect(market.id).toBeGreaterThan(0);
});
it('COLLECTIONS-SVC-101: a file carries no ids, no members and no ratings', () => {
const owner = createUser(testDb).user;
const member = createUser(testDb).user;
const { col, market } = seedList(owner.id);
addMember(col.id, member.id, 'editor');
svc.setRating(member.id, market.id, 5);
const file = svc.exportCollection(owner.id, col.id);
const serialised = JSON.stringify(file);
for (const forbidden of ['"id"', 'collection_id', 'owner_id', 'saved_by', 'source_trip_id', 'source_place_id', 'rating', 'user_id', 'category_id']) {
expect(serialised, forbidden).not.toContain(forbidden);
}
});
it('COLLECTIONS-SVC-102: an instance-local image path does not leave the instance', () => {
const owner = createUser(testDb).user;
const col = svc.createCollection(owner.id, { name: 'Pictures' });
svc.savePlace(owner.id, { collection_id: col.id, name: 'Uploaded', image_url: '/uploads/places/secret.jpg' });
svc.savePlace(owner.id, { collection_id: col.id, name: 'Proxied', image_url: '/api/maps/place-photo/abc' });
svc.savePlace(owner.id, { collection_id: col.id, name: 'Remote', image_url: 'https://example.com/photo.jpg' });
const images = svc.exportCollection(owner.id, col.id).places.map(p => (p as { image_url?: string | null }).image_url);
expect(images).toEqual([null, null, 'https://example.com/photo.jpg']);
});
it('COLLECTIONS-SVC-103: a member may export, a stranger may not', () => {
const owner = createUser(testDb).user;
const viewer = createUser(testDb).user;
const stranger = createUser(testDb).user;
const { col } = seedList(owner.id);
addMember(col.id, viewer.id, 'viewer');
expect(svc.exportCollection(viewer.id, col.id).places).toHaveLength(2);
try {
svc.exportCollection(stranger.id, col.id);
throw new Error('should have thrown');
} catch (e) {
expect((e as { status: number }).status).toBe(404);
}
});
it('COLLECTIONS-SVC-104: a file round-trips into an equivalent list of the importer', () => {
const owner = createUser(testDb).user;
const other = createUser(testDb).user;
const { col } = seedList(owner.id);
const file = svc.exportCollection(owner.id, col.id);
const result = svc.importCollection(other.id, { file });
const created = result.collection as { id: number; owner_id: number; name: string };
expect(result).toMatchObject({ imported: 2, skipped: 0 });
expect(created.owner_id).toBe(other.id);
expect(created.name).toBe('Lisbon');
// Re-exporting the copy gives the same file, bar the timestamp.
const again = svc.exportCollection(other.id, created.id);
expect({ ...again, exported_at: undefined }).toEqual({ ...file, exported_at: undefined });
});
it('COLLECTIONS-SVC-105: the importer may rename the list on the way in', () => {
const owner = createUser(testDb).user;
const { col } = seedList(owner.id);
const file = svc.exportCollection(owner.id, col.id);
const created = svc.importCollection(owner.id, { file, name: 'Lisbon (from Ana)' }).collection as { id: number; name: string };
expect(created.name).toBe('Lisbon (from Ana)');
// And the original is untouched.
expect(svc.getCollection(owner.id, col.id).collection.name).toBe('Lisbon');
});
it('COLLECTIONS-SVC-106: a category arrives by name, matched case-insensitively, unknown names drop', () => {
const owner = createUser(testDb).user;
createCategory(testDb, { name: 'Restaurant' });
const created = svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Categories', places: [
{ name: 'Exact', category: 'Restaurant' },
{ name: 'Sloppy', category: ' rEsTaUrAnT ' },
{ name: 'Unknown', category: 'Not a category here' },
{ name: 'None' },
],
} }).collection as { id: number };
const places = svc.getCollection(owner.id, created.id).places;
expect(places.map(p => p.category?.name ?? null)).toEqual(['Restaurant', 'Restaurant', null, null]);
});
it('COLLECTIONS-SVC-107: labels are recreated and only names the file defines are assigned', () => {
const owner = createUser(testDb).user;
const created = svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Labelled',
labels: [{ name: 'Must see', color: '#ff0000' }, { name: 'Must see', color: '#00ff00' }],
places: [
{ name: 'Tagged', labels: ['Must see'] },
{ name: 'Sloppy case', labels: [' MUST SEE '] },
{ name: 'Ghost label', labels: ['Never defined'] },
],
} }).collection as { id: number };
const detail = svc.getCollection(owner.id, created.id);
// The duplicate name collapses to one label.
expect(detail.collection.labels).toHaveLength(1);
const labelId = detail.collection.labels![0].id;
expect(detail.places.map(p => p.label_ids)).toEqual([[labelId], [labelId], []]);
});
it('COLLECTIONS-SVC-108: a place the contract refuses is skipped, the rest still arrive', () => {
const owner = createUser(testDb).user;
const result = svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Mixed', places: [
{ name: 'Good' },
{ noName: true },
{ name: '' },
{ name: 'Also good' },
null,
],
} });
expect(result).toMatchObject({ imported: 2, skipped: 3 });
const created = result.collection as { id: number };
expect(svc.getCollection(owner.id, created.id).places.map(p => p.name)).toEqual(['Good', 'Also good']);
});
it('COLLECTIONS-SVC-109: a file cannot hand the importer ids, provenance or other people', () => {
const owner = createUser(testDb).user;
const victim = createUser(testDb).user;
const trip = createTrip(testDb, victim.id);
const place = createPlace(testDb, trip.id, { name: 'Private place' });
const created = svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Hostile', places: [{
name: 'Claims things',
id: 9999, collection_id: 1, owner_id: victim.id, saved_by: victim.id,
source_trip_id: trip.id, source_place_id: place.id,
} as never],
} }).collection as { id: number };
const row = testDb.prepare('SELECT * FROM collection_places WHERE collection_id = ?').get(created.id) as Record<string, unknown>;
expect(row.owner_id).toBe(owner.id);
expect(row.saved_by).toBe(owner.id);
expect(row.source_trip_id).toBeNull();
expect(row.source_place_id).toBeNull();
expect(row.id).not.toBe(9999);
});
it('COLLECTIONS-SVC-110: a link the browser must not follow arrives as nothing, the place still does', () => {
const owner = createUser(testDb).user;
const created = svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Links', places: [
{ name: 'Script', website: 'javascript:alert(1)' as never },
{ name: 'Local image', image_url: '/uploads/x.jpg' as never },
{ name: 'Fine', website: 'https://example.com', image_url: 'https://example.com/p.jpg' },
],
} }).collection as { id: number };
const places = svc.getCollection(owner.id, created.id).places;
expect(places.map(p => p.name)).toEqual(['Script', 'Local image', 'Fine']);
expect(places[0].website).toBeNull();
expect(places[1].image_url).toBeNull();
expect(places[2]).toMatchObject({ website: 'https://example.com', image_url: 'https://example.com/p.jpg' });
});
it('COLLECTIONS-SVC-111: an import writes nothing at all when one of its writes fails', () => {
const owner = createUser(testDb).user;
const before = svc.listCollections(owner.id).collections.length;
const insert = testDb.prepare.bind(testDb);
const spy = vi.spyOn(testDb, 'prepare').mockImplementation((sql: string) => {
if (sql.includes('INSERT INTO collection_places')) throw new Error('disk is full');
return insert(sql);
});
try {
expect(() => svc.importCollection(owner.id, { file: {
format: 'trek.collection', version: 1, name: 'Doomed', places: [{ name: 'Never arrives' }],
} })).toThrow('disk is full');
} finally {
spy.mockRestore();
}
expect(svc.listCollections(owner.id).collections).toHaveLength(before);
});
});