From 52714630645a688d8cd7ede10220f78ab593ce66 Mon Sep 17 00:00:00 2001 From: Yannis Biasutti Date: Mon, 6 Apr 2026 18:31:47 +0200 Subject: [PATCH] feat(server): add KML and KMZ place import pipeline --- server/src/routes/places.ts | 54 ++++++ .../services/placeImport/kmlImportUtils.ts | 175 ++++++++++++++++++ server/src/services/placeService.ts | 122 +++++++++++- server/tests/fixtures/test-malformed.kml | 8 + server/tests/fixtures/test-nested.kml | 26 +++ server/tests/fixtures/test.kml | 21 +++ server/tests/fixtures/test.kmz | Bin 0 -> 408 bytes server/tests/integration/places.test.ts | 123 ++++++++++++ .../unit/services/kmlImportUtils.test.ts | 78 ++++++++ 9 files changed, 606 insertions(+), 1 deletion(-) create mode 100644 server/src/services/placeImport/kmlImportUtils.ts create mode 100644 server/tests/fixtures/test-malformed.kml create mode 100644 server/tests/fixtures/test-nested.kml create mode 100644 server/tests/fixtures/test.kml create mode 100644 server/tests/fixtures/test.kmz create mode 100644 server/tests/unit/services/kmlImportUtils.test.ts diff --git a/server/src/routes/places.ts b/server/src/routes/places.ts index 642c95a4..59f7b8fe 100644 --- a/server/src/routes/places.ts +++ b/server/src/routes/places.ts @@ -13,6 +13,8 @@ import { updatePlace, deletePlace, importGpx, + importKmlPlaces, + importKmzPlaces, importGoogleList, searchPlaceImage, } from '../services/placeService'; @@ -72,6 +74,58 @@ router.post('/import/gpx', authenticate, requireTripAccess, gpxUpload.single('fi } }); +router.post('/import/kml', authenticate, requireTripAccess, gpxUpload.single('file'), (req: Request, res: Response) => { + const authReq = req as AuthRequest; + if (!checkPermission('place_edit', authReq.user.role, authReq.trip!.user_id, authReq.user.id, authReq.trip!.user_id !== authReq.user.id)) { + return res.status(403).json({ error: 'No permission' }); + } + + const { tripId } = req.params; + const file = (req as any).file; + if (!file) return res.status(400).json({ error: 'No file uploaded' }); + + try { + const result = importKmlPlaces(tripId, file.buffer); + if (result.count === 0) { + return res.status(400).json({ error: 'No valid Placemarks found in KML file', summary: result.summary }); + } + + res.status(201).json(result); + for (const place of result.places) { + broadcast(tripId, 'place:created', { place }, req.headers['x-socket-id'] as string); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : 'Failed to import KML file'; + res.status(400).json({ error: message }); + } +}); + +router.post('/import/kmz', authenticate, requireTripAccess, gpxUpload.single('file'), async (req: Request, res: Response) => { + const authReq = req as AuthRequest; + if (!checkPermission('place_edit', authReq.user.role, authReq.trip!.user_id, authReq.user.id, authReq.trip!.user_id !== authReq.user.id)) { + return res.status(403).json({ error: 'No permission' }); + } + + const { tripId } = req.params; + const file = (req as any).file; + if (!file) return res.status(400).json({ error: 'No file uploaded' }); + + try { + const result = await importKmzPlaces(tripId, file.buffer); + if (result.count === 0) { + return res.status(400).json({ error: 'No valid Placemarks found in KMZ file', summary: result.summary }); + } + + res.status(201).json(result); + for (const place of result.places) { + broadcast(tripId, 'place:created', { place }, req.headers['x-socket-id'] as string); + } + } catch (err: unknown) { + const message = err instanceof Error ? err.message : 'Failed to import KMZ file'; + res.status(400).json({ error: message }); + } +}); + // Import places from a shared Google Maps list URL router.post('/import/google-list', authenticate, requireTripAccess, async (req: Request, res: Response) => { const authReq = req as AuthRequest; diff --git a/server/src/services/placeImport/kmlImportUtils.ts b/server/src/services/placeImport/kmlImportUtils.ts new file mode 100644 index 00000000..4489d54b --- /dev/null +++ b/server/src/services/placeImport/kmlImportUtils.ts @@ -0,0 +1,175 @@ +import { TextDecoder } from 'util'; + +export interface ParsedKmlPlacemark { + name: string | null; + description: string | null; + lat: number | null; + lng: number | null; + folderName: string | null; +} + +export interface KmlPlacemarkNode { + placemark: any; + folderName: string | null; +} + +export interface KmlImportSummary { + totalPlacemarks: number; + createdCount: number; + skippedCount: number; + warnings: string[]; + errors: string[]; +} + +const UTF8_DECODER_FATAL = new TextDecoder('utf-8', { fatal: true }); +const UTF8_DECODER_LOOSE = new TextDecoder('utf-8'); + +const ENTITY_MAP: Record = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + ''': "'", + ' ': ' ', +}; + +function asArray(value: T | T[] | null | undefined): T[] { + if (value == null) return []; + return Array.isArray(value) ? value : [value]; +} + +function asTrimmedString(value: unknown): string | null { + if (value == null) return null; + const text = String(value).trim(); + return text.length > 0 ? text : null; +} + +function decodeHtmlEntities(value: string): string { + const withNamedEntities = value.replace(/&(amp|lt|gt|quot|#39|nbsp);/g, (m) => ENTITY_MAP[m] || m); + + return withNamedEntities + .replace(/&#(\d+);/g, (_, dec) => { + const code = Number(dec); + return Number.isFinite(code) ? String.fromCharCode(code) : _; + }) + .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => { + const code = Number.parseInt(hex, 16); + return Number.isFinite(code) ? String.fromCharCode(code) : _; + }); +} + +export function stripXmlNamespaces(xml: string): string { + // KML exports vary heavily; stripping namespace declarations/prefixes makes parsing resilient. + return xml + .replace(/\sxmlns(:\w+)?="[^"]*"/g, '') + .replace(/\sxmlns(:\w+)?='[^']*'/g, '') + .replace(/<(\/?)\w+:/g, '<$1'); +} + +export function decodeUtf8WithWarning(fileBuffer: Buffer): { text: string; warning: string | null } { + try { + return { text: UTF8_DECODER_FATAL.decode(fileBuffer), warning: null }; + } catch { + return { + text: UTF8_DECODER_LOOSE.decode(fileBuffer), + warning: 'The uploaded file is not valid UTF-8. Some characters may be shown incorrectly.', + }; + } +} + +export function sanitizeKmlDescription(value: unknown): string | null { + const raw = asTrimmedString(value); + if (!raw) return null; + + const withLineBreaks = raw.replace(//gi, '\n'); + const stripped = withLineBreaks.replace(/<[^>]+>/g, ''); + const decoded = decodeHtmlEntities(stripped) + .replace(/\r\n/g, '\n') + .replace(/\r/g, '\n') + .replace(/[\t\f\v]+/g, ' ') + .replace(/\n{3,}/g, '\n\n') + .trim(); + + return decoded || null; +} + +export function parseKmlPointCoordinates(value: unknown): { lat: number; lng: number } | null { + const coordinates = asTrimmedString(value); + if (!coordinates) return null; + + const firstCoordinate = coordinates.split(/\s+/)[0]; + const [lngRaw, latRaw] = firstCoordinate.split(','); + if (lngRaw == null || latRaw == null) return null; + + const lng = Number.parseFloat(lngRaw); + const lat = Number.parseFloat(latRaw); + + if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null; + return { lat, lng }; +} + +export function createKmlImportSummary(totalPlacemarks: number): KmlImportSummary { + return { + totalPlacemarks, + createdCount: 0, + skippedCount: 0, + warnings: [], + errors: [], + }; +} + +export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map { + const lookup = new Map(); + for (const category of categories) { + const normalizedName = category.name.trim().toLowerCase(); + if (!normalizedName) continue; + if (!lookup.has(normalizedName)) { + lookup.set(normalizedName, category.id); + } + } + return lookup; +} + +export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map): number | null { + if (!folderName) return null; + const normalizedFolder = folderName.trim().toLowerCase(); + if (!normalizedFolder) return null; + return lookup.get(normalizedFolder) ?? null; +} + +export function extractKmlPlacemarkNodes(kmlRoot: any): KmlPlacemarkNode[] { + const nodes: KmlPlacemarkNode[] = []; + + const visitNode = (node: any, currentFolderName: string | null): void => { + if (!node || typeof node !== 'object') return; + + for (const placemark of asArray(node.Placemark)) { + nodes.push({ placemark, folderName: currentFolderName }); + } + + for (const folder of asArray(node.Folder)) { + // Nested folders inherit/override folder context used for category matching. + const folderName = asTrimmedString(folder?.name) || currentFolderName; + visitNode(folder, folderName); + } + + for (const childDocument of asArray(node.Document)) { + visitNode(childDocument, currentFolderName); + } + }; + + visitNode(kmlRoot, null); + return nodes; +} + +export function parsePlacemarkNode(node: KmlPlacemarkNode): ParsedKmlPlacemark { + const coordinates = parseKmlPointCoordinates(node.placemark?.Point?.coordinates); + + return { + name: asTrimmedString(node.placemark?.name), + description: sanitizeKmlDescription(node.placemark?.description), + lat: coordinates?.lat ?? null, + lng: coordinates?.lng ?? null, + folderName: node.folderName, + }; +} diff --git a/server/src/services/placeService.ts b/server/src/services/placeService.ts index 911f5ae4..b602bd0c 100644 --- a/server/src/services/placeService.ts +++ b/server/src/services/placeService.ts @@ -1,7 +1,18 @@ -import { XMLParser } from 'fast-xml-parser'; +import { XMLParser, XMLValidator } from 'fast-xml-parser'; +import unzipper from 'unzipper'; import { db, getPlaceWithTags } from '../db/database'; import { loadTagsByPlaceIds } from './queryHelpers'; import { Place } from '../types'; +import { + buildCategoryNameLookup, + createKmlImportSummary, + decodeUtf8WithWarning, + extractKmlPlacemarkNodes, + parsePlacemarkNode, + resolveCategoryIdForFolder, + stripXmlNamespaces, + type KmlImportSummary, +} from './placeImport/kmlImportUtils'; interface PlaceWithCategory extends Place { category_name: string | null; @@ -14,6 +25,12 @@ interface UnsplashSearchResponse { errors?: string[]; } +export interface PlaceImportResult { + places: any[]; + count: number; + summary: KmlImportSummary; +} + // --------------------------------------------------------------------------- // List places // --------------------------------------------------------------------------- @@ -223,6 +240,12 @@ const gpxParser = new XMLParser({ isArray: (name) => ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'].includes(name), }); +const kmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name), +}); + export function importGpx(tripId: string, fileBuffer: Buffer) { const parsed = gpxParser.parse(fileBuffer.toString('utf-8')); const gpx = parsed?.gpx; @@ -291,6 +314,103 @@ export function importGpx(tripId: string, fileBuffer: Buffer) { return created; } +export function importKmlPlaces(tripId: string, fileBuffer: Buffer): PlaceImportResult { + const decoded = decodeUtf8WithWarning(fileBuffer); + const xmlWithoutNamespaces = stripXmlNamespaces(decoded.text); + + const validationResult = XMLValidator.validate(xmlWithoutNamespaces); + if (validationResult !== true) { + throw new Error('Malformed KML: invalid XML structure'); + } + + const parsed = kmlParser.parse(xmlWithoutNamespaces); + const kmlRoot = parsed?.kml ?? parsed; + + if (!kmlRoot || typeof kmlRoot !== 'object') { + throw new Error('Malformed KML: could not parse XML'); + } + + const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot); + const summary = createKmlImportSummary(placemarkNodes.length); + + if (decoded.warning) { + summary.warnings.push(decoded.warning); + } + + const categories = db.prepare('SELECT id, name FROM categories').all() as { id: number; name: string }[]; + const categoryLookup = buildCategoryNameLookup(categories); + const created: any[] = []; + + const insertStmt = db.prepare(` + INSERT INTO places (trip_id, name, description, lat, lng, category_id, transport_mode) + VALUES (?, ?, ?, ?, ?, ?, 'walking') + `); + + const insertAll = db.transaction(() => { + let fallbackIndex = 1; + for (const node of placemarkNodes) { + const parsedPlacemark = parsePlacemarkNode(node); + + // KML geometry support is intentionally limited to coordinates. + if (parsedPlacemark.lat === null || parsedPlacemark.lng === null) { + summary.skippedCount += 1; + summary.errors.push(`Skipped Placemark ${fallbackIndex}: missing Point coordinates.`); + fallbackIndex += 1; + continue; + } + + const fallbackName = `Placemark ${fallbackIndex}`; + const name = parsedPlacemark.name || fallbackName; + const categoryId = resolveCategoryIdForFolder(parsedPlacemark.folderName, categoryLookup); + + const result = insertStmt.run( + tripId, + name, + parsedPlacemark.description, + parsedPlacemark.lat, + parsedPlacemark.lng, + categoryId, + ); + + const place = getPlaceWithTags(Number(result.lastInsertRowid)); + created.push(place); + summary.createdCount += 1; + fallbackIndex += 1; + } + }); + + insertAll(); + summary.skippedCount = summary.totalPlacemarks - summary.createdCount; + + if (summary.totalPlacemarks === 0) { + summary.errors.push('No Placemarks found in KML file.'); + } + + return { places: created, count: created.length, summary }; +} + +export async function unpackKmzToKml(kmzBuffer: Buffer): Promise { + let zip; + try { + zip = await unzipper.Open.buffer(kmzBuffer); + } catch { + throw new Error('Invalid KMZ archive.'); + } + + const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml')); + if (kmlEntries.length === 0) { + throw new Error('KMZ archive does not contain a KML file.'); + } + + const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0]; + return preferredEntry.buffer(); +} + +export async function importKmzPlaces(tripId: string, kmzBuffer: Buffer): Promise { + const kmlBuffer = await unpackKmzToKml(kmzBuffer); + return importKmlPlaces(tripId, kmlBuffer); +} + // --------------------------------------------------------------------------- // Import Google Maps list // --------------------------------------------------------------------------- diff --git a/server/tests/fixtures/test-malformed.kml b/server/tests/fixtures/test-malformed.kml new file mode 100644 index 00000000..6d227c24 --- /dev/null +++ b/server/tests/fixtures/test-malformed.kml @@ -0,0 +1,8 @@ + + + + + Broken Placemark + 2.1,48.1,0 + + diff --git a/server/tests/fixtures/test-nested.kml b/server/tests/fixtures/test-nested.kml new file mode 100644 index 00000000..e29b7cb8 --- /dev/null +++ b/server/tests/fixtures/test-nested.kml @@ -0,0 +1,26 @@ + + + + + Food + + Parks + + Nested Place + Nested folder placemark
line 2 + + 13.4050,52.5200,15 + + + + + Empty Placemark + + + + 13.4010,52.5210,0 + + + + + diff --git a/server/tests/fixtures/test.kml b/server/tests/fixtures/test.kml new file mode 100644 index 00000000..5501fd26 --- /dev/null +++ b/server/tests/fixtures/test.kml @@ -0,0 +1,21 @@ + + + + + Museums + + Eiffel Tower View + for photos and skyline.]]> + + 2.2945,48.8584,0 + + + + Coordinates only placemark + + 2.3333,48.8600,0 + + + + + diff --git a/server/tests/fixtures/test.kmz b/server/tests/fixtures/test.kmz new file mode 100644 index 0000000000000000000000000000000000000000..347373022fa054eb75dde83d1512bb3992fd4ef8 GIT binary patch literal 408 zcmWIWW@Zs#U|`^2xID2f#^CLN7&S%)hAJkI2m?b&YH^8Pc5cqnAYXqbLxH`sf5^X@ zSGZeg##J6wmLron**`d0=cIFs3Z>o2cZucx1I+GLs3^=`9!*ZG6Ye1(I1)7Hl3T<}giQFfh2VFtwX=Tg{vU0}{=eMwll*^$W_EVTKWsmKXw>Xy4)A7V5@Em{n!o@>0 { createTables(testDb); @@ -528,3 +532,122 @@ describe('GPX Import', () => { expect(res.status).toBe(400); }); }); + +// ───────────────────────────────────────────────────────────────────────────── +// KML / KMZ Import +// ───────────────────────────────────────────────────────────────────────────── + +describe('KML/KMZ Import', () => { + it('PLACE-020 — POST /import/kml with valid KML creates places and returns summary', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + testDb.prepare('INSERT INTO categories (name, color, icon, user_id) VALUES (?, ?, ?, ?)') + .run('Museums', '#3b82f6', 'Landmark', user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kml`) + .set('Cookie', authCookie(user.id)) + .attach('file', KML_FIXTURE); + + expect(res.status).toBe(201); + expect(res.body.count).toBe(2); + expect(res.body.summary).toBeDefined(); + expect(res.body.summary.totalPlacemarks).toBe(2); + expect(res.body.summary.createdCount).toBe(2); + + const first = res.body.places.find((p: any) => p.name === 'Eiffel Tower View'); + expect(first).toBeDefined(); + expect(first.description).toContain('Great spot'); + expect(first.description).toContain('\n'); + expect(first.description).not.toContain(''); + expect(first.category?.name).toBe('Museums'); + }); + + it('PLACE-021 — nested folders, empty placemark, and coordinates-only placemark are handled', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + testDb.prepare('INSERT INTO categories (name, color, icon, user_id) VALUES (?, ?, ?, ?)') + .run('Parks', '#22c55e', 'Trees', user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kml`) + .set('Cookie', authCookie(user.id)) + .attach('file', KML_NESTED_FIXTURE); + + expect(res.status).toBe(201); + expect(res.body.count).toBe(2); + expect(res.body.summary.totalPlacemarks).toBe(3); + expect(res.body.summary.skippedCount).toBe(1); + expect(Array.isArray(res.body.summary.errors)).toBe(true); + expect(res.body.summary.errors.join(' ')).toContain('missing Point coordinates'); + + const nested = res.body.places.find((p: any) => p.name === 'Nested Place'); + expect(nested).toBeDefined(); + expect(nested.category?.name).toBe('Parks'); + + const fallback = res.body.places.find((p: any) => String(p.name).startsWith('Placemark')); + expect(fallback).toBeDefined(); + }); + + it('PLACE-022 — malformed KML returns 400', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kml`) + .set('Cookie', authCookie(user.id)) + .attach('file', KML_MALFORMED_FIXTURE); + + expect(res.status).toBe(400); + expect(res.body.error).toBeDefined(); + }); + + it('PLACE-023 — non-UTF8 KML continues with warning', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const prefix = Buffer.from('Caf'); + const invalidByte = Buffer.from([0xe9]); // invalid UTF-8 sequence when used standalone + const suffix = Buffer.from('2.1,48.1,0'); + const nonUtf8Kml = Buffer.concat([prefix, invalidByte, suffix]); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kml`) + .set('Cookie', authCookie(user.id)) + .attach('file', nonUtf8Kml, 'non-utf8.kml'); + + expect(res.status).toBe(201); + expect(res.body.count).toBe(1); + expect(Array.isArray(res.body.summary.warnings)).toBe(true); + expect(res.body.summary.warnings.join(' ')).toContain('not valid UTF-8'); + }); + + it('PLACE-024 — POST /import/kmz with valid KMZ creates places', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kmz`) + .set('Cookie', authCookie(user.id)) + .attach('file', KMZ_FIXTURE); + + expect(res.status).toBe(201); + expect(res.body.count).toBeGreaterThan(0); + expect(res.body.summary).toBeDefined(); + }); + + it('PLACE-025 — invalid KMZ returns 400', async () => { + const { user } = createUser(testDb); + const trip = createTrip(testDb, user.id); + + const res = await request(app) + .post(`/api/trips/${trip.id}/places/import/kmz`) + .set('Cookie', authCookie(user.id)) + .attach('file', Buffer.from('not-a-zip-archive'), 'invalid.kmz'); + + expect(res.status).toBe(400); + expect(String(res.body.error || '')).toContain('KMZ'); + }); +}); diff --git a/server/tests/unit/services/kmlImportUtils.test.ts b/server/tests/unit/services/kmlImportUtils.test.ts new file mode 100644 index 00000000..f250b696 --- /dev/null +++ b/server/tests/unit/services/kmlImportUtils.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest'; +import { + buildCategoryNameLookup, + decodeUtf8WithWarning, + extractKmlPlacemarkNodes, + parseKmlPointCoordinates, + parsePlacemarkNode, + resolveCategoryIdForFolder, + sanitizeKmlDescription, + stripXmlNamespaces, +} from '../../../src/services/placeImport/kmlImportUtils'; + +describe('kmlImportUtils', () => { + it('strips KML namespaces and prefixes', () => { + const xml = ''; + const stripped = stripXmlNamespaces(xml); + expect(stripped).not.toContain('xmlns'); + expect(stripped).toContain(''); + expect(stripped).toContain(' { + const input = 'Line 1
Line 2 & more'; + const output = sanitizeKmlDescription(input); + expect(output).toBe('Line 1\nLine 2 & more'); + }); + + it('parses KML coordinate order lng,lat,alt', () => { + const parsed = parseKmlPointCoordinates('13.4050,52.5200,15'); + expect(parsed).toEqual({ lat: 52.52, lng: 13.405 }); + }); + + it('extracts placemarks from nested folders', () => { + const root = { + Document: { + Folder: { + name: 'Parent', + Folder: { + name: 'Child', + Placemark: { name: 'Nested', Point: { coordinates: '13.4,52.5,0' } }, + }, + }, + }, + }; + + const nodes = extractKmlPlacemarkNodes(root); + expect(nodes).toHaveLength(1); + expect(nodes[0].folderName).toBe('Child'); + + const parsed = parsePlacemarkNode(nodes[0]); + expect(parsed.name).toBe('Nested'); + expect(parsed.lat).toBe(52.5); + expect(parsed.lng).toBe(13.4); + }); + + it('builds exact case-insensitive category lookup', () => { + const lookup = buildCategoryNameLookup([ + { id: 3, name: 'Museums' }, + { id: 4, name: 'Parks' }, + ]); + + expect(resolveCategoryIdForFolder('museums', lookup)).toBe(3); + expect(resolveCategoryIdForFolder('Museum', lookup)).toBeNull(); + expect(resolveCategoryIdForFolder('parks', lookup)).toBe(4); + }); + + it('returns warning for non-UTF8 payload', () => { + const buffer = Buffer.concat([ + Buffer.from('Caf'), + Buffer.from([0xe9]), + Buffer.from(''), + ]); + + const decoded = decodeUtf8WithWarning(buffer); + expect(decoded.warning).toContain('not valid UTF-8'); + expect(decoded.text).toContain(''); + }); +});