feat(server): add KML and KMZ place import pipeline

This commit is contained in:
Yannis Biasutti
2026-04-06 18:31:47 +02:00
parent 96080e8a03
commit 5271463064
9 changed files with 606 additions and 1 deletions
+54
View File
@@ -13,6 +13,8 @@ import {
updatePlace,
deletePlace,
importGpx,
importKmlPlaces,
importKmzPlaces,
importGoogleList,
searchPlaceImage,
} from '../services/placeService';
@@ -72,6 +74,58 @@ router.post('/import/gpx', authenticate, requireTripAccess, gpxUpload.single('fi
}
});
router.post('/import/kml', authenticate, requireTripAccess, gpxUpload.single('file'), (req: Request, res: Response) => {
const authReq = req as AuthRequest;
if (!checkPermission('place_edit', authReq.user.role, authReq.trip!.user_id, authReq.user.id, authReq.trip!.user_id !== authReq.user.id)) {
return res.status(403).json({ error: 'No permission' });
}
const { tripId } = req.params;
const file = (req as any).file;
if (!file) return res.status(400).json({ error: 'No file uploaded' });
try {
const result = importKmlPlaces(tripId, file.buffer);
if (result.count === 0) {
return res.status(400).json({ error: 'No valid Placemarks found in KML file', summary: result.summary });
}
res.status(201).json(result);
for (const place of result.places) {
broadcast(tripId, 'place:created', { place }, req.headers['x-socket-id'] as string);
}
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Failed to import KML file';
res.status(400).json({ error: message });
}
});
router.post('/import/kmz', authenticate, requireTripAccess, gpxUpload.single('file'), async (req: Request, res: Response) => {
const authReq = req as AuthRequest;
if (!checkPermission('place_edit', authReq.user.role, authReq.trip!.user_id, authReq.user.id, authReq.trip!.user_id !== authReq.user.id)) {
return res.status(403).json({ error: 'No permission' });
}
const { tripId } = req.params;
const file = (req as any).file;
if (!file) return res.status(400).json({ error: 'No file uploaded' });
try {
const result = await importKmzPlaces(tripId, file.buffer);
if (result.count === 0) {
return res.status(400).json({ error: 'No valid Placemarks found in KMZ file', summary: result.summary });
}
res.status(201).json(result);
for (const place of result.places) {
broadcast(tripId, 'place:created', { place }, req.headers['x-socket-id'] as string);
}
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Failed to import KMZ file';
res.status(400).json({ error: message });
}
});
// Import places from a shared Google Maps list URL
router.post('/import/google-list', authenticate, requireTripAccess, async (req: Request, res: Response) => {
const authReq = req as AuthRequest;
@@ -0,0 +1,175 @@
import { TextDecoder } from 'util';
export interface ParsedKmlPlacemark {
name: string | null;
description: string | null;
lat: number | null;
lng: number | null;
folderName: string | null;
}
export interface KmlPlacemarkNode {
placemark: any;
folderName: string | null;
}
export interface KmlImportSummary {
totalPlacemarks: number;
createdCount: number;
skippedCount: number;
warnings: string[];
errors: string[];
}
const UTF8_DECODER_FATAL = new TextDecoder('utf-8', { fatal: true });
const UTF8_DECODER_LOOSE = new TextDecoder('utf-8');
const ENTITY_MAP: Record<string, string> = {
'&amp;': '&',
'&lt;': '<',
'&gt;': '>',
'&quot;': '"',
'&#39;': "'",
'&nbsp;': ' ',
};
function asArray<T>(value: T | T[] | null | undefined): T[] {
if (value == null) return [];
return Array.isArray(value) ? value : [value];
}
function asTrimmedString(value: unknown): string | null {
if (value == null) return null;
const text = String(value).trim();
return text.length > 0 ? text : null;
}
function decodeHtmlEntities(value: string): string {
const withNamedEntities = value.replace(/&(amp|lt|gt|quot|#39|nbsp);/g, (m) => ENTITY_MAP[m] || m);
return withNamedEntities
.replace(/&#(\d+);/g, (_, dec) => {
const code = Number(dec);
return Number.isFinite(code) ? String.fromCharCode(code) : _;
})
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => {
const code = Number.parseInt(hex, 16);
return Number.isFinite(code) ? String.fromCharCode(code) : _;
});
}
export function stripXmlNamespaces(xml: string): string {
// KML exports vary heavily; stripping namespace declarations/prefixes makes parsing resilient.
return xml
.replace(/\sxmlns(:\w+)?="[^"]*"/g, '')
.replace(/\sxmlns(:\w+)?='[^']*'/g, '')
.replace(/<(\/?)\w+:/g, '<$1');
}
export function decodeUtf8WithWarning(fileBuffer: Buffer): { text: string; warning: string | null } {
try {
return { text: UTF8_DECODER_FATAL.decode(fileBuffer), warning: null };
} catch {
return {
text: UTF8_DECODER_LOOSE.decode(fileBuffer),
warning: 'The uploaded file is not valid UTF-8. Some characters may be shown incorrectly.',
};
}
}
export function sanitizeKmlDescription(value: unknown): string | null {
const raw = asTrimmedString(value);
if (!raw) return null;
const withLineBreaks = raw.replace(/<br\s*\/?>/gi, '\n');
const stripped = withLineBreaks.replace(/<[^>]+>/g, '');
const decoded = decodeHtmlEntities(stripped)
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n')
.replace(/[\t\f\v]+/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
return decoded || null;
}
export function parseKmlPointCoordinates(value: unknown): { lat: number; lng: number } | null {
const coordinates = asTrimmedString(value);
if (!coordinates) return null;
const firstCoordinate = coordinates.split(/\s+/)[0];
const [lngRaw, latRaw] = firstCoordinate.split(',');
if (lngRaw == null || latRaw == null) return null;
const lng = Number.parseFloat(lngRaw);
const lat = Number.parseFloat(latRaw);
if (!Number.isFinite(lat) || !Number.isFinite(lng)) return null;
return { lat, lng };
}
export function createKmlImportSummary(totalPlacemarks: number): KmlImportSummary {
return {
totalPlacemarks,
createdCount: 0,
skippedCount: 0,
warnings: [],
errors: [],
};
}
export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map<string, number> {
const lookup = new Map<string, number>();
for (const category of categories) {
const normalizedName = category.name.trim().toLowerCase();
if (!normalizedName) continue;
if (!lookup.has(normalizedName)) {
lookup.set(normalizedName, category.id);
}
}
return lookup;
}
export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map<string, number>): number | null {
if (!folderName) return null;
const normalizedFolder = folderName.trim().toLowerCase();
if (!normalizedFolder) return null;
return lookup.get(normalizedFolder) ?? null;
}
export function extractKmlPlacemarkNodes(kmlRoot: any): KmlPlacemarkNode[] {
const nodes: KmlPlacemarkNode[] = [];
const visitNode = (node: any, currentFolderName: string | null): void => {
if (!node || typeof node !== 'object') return;
for (const placemark of asArray(node.Placemark)) {
nodes.push({ placemark, folderName: currentFolderName });
}
for (const folder of asArray(node.Folder)) {
// Nested folders inherit/override folder context used for category matching.
const folderName = asTrimmedString(folder?.name) || currentFolderName;
visitNode(folder, folderName);
}
for (const childDocument of asArray(node.Document)) {
visitNode(childDocument, currentFolderName);
}
};
visitNode(kmlRoot, null);
return nodes;
}
export function parsePlacemarkNode(node: KmlPlacemarkNode): ParsedKmlPlacemark {
const coordinates = parseKmlPointCoordinates(node.placemark?.Point?.coordinates);
return {
name: asTrimmedString(node.placemark?.name),
description: sanitizeKmlDescription(node.placemark?.description),
lat: coordinates?.lat ?? null,
lng: coordinates?.lng ?? null,
folderName: node.folderName,
};
}
+121 -1
View File
@@ -1,7 +1,18 @@
import { XMLParser } from 'fast-xml-parser';
import { XMLParser, XMLValidator } from 'fast-xml-parser';
import unzipper from 'unzipper';
import { db, getPlaceWithTags } from '../db/database';
import { loadTagsByPlaceIds } from './queryHelpers';
import { Place } from '../types';
import {
buildCategoryNameLookup,
createKmlImportSummary,
decodeUtf8WithWarning,
extractKmlPlacemarkNodes,
parsePlacemarkNode,
resolveCategoryIdForFolder,
stripXmlNamespaces,
type KmlImportSummary,
} from './placeImport/kmlImportUtils';
interface PlaceWithCategory extends Place {
category_name: string | null;
@@ -14,6 +25,12 @@ interface UnsplashSearchResponse {
errors?: string[];
}
export interface PlaceImportResult {
places: any[];
count: number;
summary: KmlImportSummary;
}
// ---------------------------------------------------------------------------
// List places
// ---------------------------------------------------------------------------
@@ -223,6 +240,12 @@ const gpxParser = new XMLParser({
isArray: (name) => ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'].includes(name),
});
const kmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name),
});
export function importGpx(tripId: string, fileBuffer: Buffer) {
const parsed = gpxParser.parse(fileBuffer.toString('utf-8'));
const gpx = parsed?.gpx;
@@ -291,6 +314,103 @@ export function importGpx(tripId: string, fileBuffer: Buffer) {
return created;
}
export function importKmlPlaces(tripId: string, fileBuffer: Buffer): PlaceImportResult {
const decoded = decodeUtf8WithWarning(fileBuffer);
const xmlWithoutNamespaces = stripXmlNamespaces(decoded.text);
const validationResult = XMLValidator.validate(xmlWithoutNamespaces);
if (validationResult !== true) {
throw new Error('Malformed KML: invalid XML structure');
}
const parsed = kmlParser.parse(xmlWithoutNamespaces);
const kmlRoot = parsed?.kml ?? parsed;
if (!kmlRoot || typeof kmlRoot !== 'object') {
throw new Error('Malformed KML: could not parse XML');
}
const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot);
const summary = createKmlImportSummary(placemarkNodes.length);
if (decoded.warning) {
summary.warnings.push(decoded.warning);
}
const categories = db.prepare('SELECT id, name FROM categories').all() as { id: number; name: string }[];
const categoryLookup = buildCategoryNameLookup(categories);
const created: any[] = [];
const insertStmt = db.prepare(`
INSERT INTO places (trip_id, name, description, lat, lng, category_id, transport_mode)
VALUES (?, ?, ?, ?, ?, ?, 'walking')
`);
const insertAll = db.transaction(() => {
let fallbackIndex = 1;
for (const node of placemarkNodes) {
const parsedPlacemark = parsePlacemarkNode(node);
// KML geometry support is intentionally limited to <Placemark><Point> coordinates.
if (parsedPlacemark.lat === null || parsedPlacemark.lng === null) {
summary.skippedCount += 1;
summary.errors.push(`Skipped Placemark ${fallbackIndex}: missing Point coordinates.`);
fallbackIndex += 1;
continue;
}
const fallbackName = `Placemark ${fallbackIndex}`;
const name = parsedPlacemark.name || fallbackName;
const categoryId = resolveCategoryIdForFolder(parsedPlacemark.folderName, categoryLookup);
const result = insertStmt.run(
tripId,
name,
parsedPlacemark.description,
parsedPlacemark.lat,
parsedPlacemark.lng,
categoryId,
);
const place = getPlaceWithTags(Number(result.lastInsertRowid));
created.push(place);
summary.createdCount += 1;
fallbackIndex += 1;
}
});
insertAll();
summary.skippedCount = summary.totalPlacemarks - summary.createdCount;
if (summary.totalPlacemarks === 0) {
summary.errors.push('No Placemarks found in KML file.');
}
return { places: created, count: created.length, summary };
}
export async function unpackKmzToKml(kmzBuffer: Buffer): Promise<Buffer> {
let zip;
try {
zip = await unzipper.Open.buffer(kmzBuffer);
} catch {
throw new Error('Invalid KMZ archive.');
}
const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml'));
if (kmlEntries.length === 0) {
throw new Error('KMZ archive does not contain a KML file.');
}
const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0];
return preferredEntry.buffer();
}
export async function importKmzPlaces(tripId: string, kmzBuffer: Buffer): Promise<PlaceImportResult> {
const kmlBuffer = await unpackKmzToKml(kmzBuffer);
return importKmlPlaces(tripId, kmlBuffer);
}
// ---------------------------------------------------------------------------
// Import Google Maps list
// ---------------------------------------------------------------------------
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml>
<Document>
<Placemark>
<name>Broken Placemark</name>
<Point><coordinates>2.1,48.1,0</coordinates></Point>
</Document>
</kml>
+26
View File
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<Folder>
<name>Food</name>
<Folder>
<name>Parks</name>
<Placemark>
<name>Nested Place</name>
<description>Nested <i>folder</i> placemark<br/>line 2</description>
<Point>
<coordinates>13.4050,52.5200,15</coordinates>
</Point>
</Placemark>
</Folder>
<Placemark>
<name>Empty Placemark</name>
</Placemark>
<Placemark>
<Point>
<coordinates>13.4010,52.5210,0</coordinates>
</Point>
</Placemark>
</Folder>
</Document>
</kml>
+21
View File
@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://www.opengis.net/kml/2.2">
<Document>
<Folder>
<name>Museums</name>
<Placemark>
<name>Eiffel Tower View</name>
<description><![CDATA[Great spot<br>for photos <b>and</b> skyline.]]></description>
<Point>
<coordinates>2.2945,48.8584,0</coordinates>
</Point>
</Placemark>
<Placemark>
<description>Coordinates only placemark</description>
<Point>
<coordinates>2.3333,48.8600,0</coordinates>
</Point>
</Placemark>
</Folder>
</Document>
</kml>
BIN
View File
Binary file not shown.
+123
View File
@@ -53,6 +53,10 @@ import { loginAttempts, mfaAttempts } from '../../src/routes/auth';
const app: Application = createApp();
const GPX_FIXTURE = path.join(__dirname, '../fixtures/test.gpx');
const KML_FIXTURE = path.join(__dirname, '../fixtures/test.kml');
const KML_NESTED_FIXTURE = path.join(__dirname, '../fixtures/test-nested.kml');
const KML_MALFORMED_FIXTURE = path.join(__dirname, '../fixtures/test-malformed.kml');
const KMZ_FIXTURE = path.join(__dirname, '../fixtures/test.kmz');
beforeAll(() => {
createTables(testDb);
@@ -528,3 +532,122 @@ describe('GPX Import', () => {
expect(res.status).toBe(400);
});
});
// ─────────────────────────────────────────────────────────────────────────────
// KML / KMZ Import
// ─────────────────────────────────────────────────────────────────────────────
describe('KML/KMZ Import', () => {
it('PLACE-020 — POST /import/kml with valid KML creates places and returns summary', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
testDb.prepare('INSERT INTO categories (name, color, icon, user_id) VALUES (?, ?, ?, ?)')
.run('Museums', '#3b82f6', 'Landmark', user.id);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kml`)
.set('Cookie', authCookie(user.id))
.attach('file', KML_FIXTURE);
expect(res.status).toBe(201);
expect(res.body.count).toBe(2);
expect(res.body.summary).toBeDefined();
expect(res.body.summary.totalPlacemarks).toBe(2);
expect(res.body.summary.createdCount).toBe(2);
const first = res.body.places.find((p: any) => p.name === 'Eiffel Tower View');
expect(first).toBeDefined();
expect(first.description).toContain('Great spot');
expect(first.description).toContain('\n');
expect(first.description).not.toContain('<b>');
expect(first.category?.name).toBe('Museums');
});
it('PLACE-021 — nested folders, empty placemark, and coordinates-only placemark are handled', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
testDb.prepare('INSERT INTO categories (name, color, icon, user_id) VALUES (?, ?, ?, ?)')
.run('Parks', '#22c55e', 'Trees', user.id);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kml`)
.set('Cookie', authCookie(user.id))
.attach('file', KML_NESTED_FIXTURE);
expect(res.status).toBe(201);
expect(res.body.count).toBe(2);
expect(res.body.summary.totalPlacemarks).toBe(3);
expect(res.body.summary.skippedCount).toBe(1);
expect(Array.isArray(res.body.summary.errors)).toBe(true);
expect(res.body.summary.errors.join(' ')).toContain('missing Point coordinates');
const nested = res.body.places.find((p: any) => p.name === 'Nested Place');
expect(nested).toBeDefined();
expect(nested.category?.name).toBe('Parks');
const fallback = res.body.places.find((p: any) => String(p.name).startsWith('Placemark'));
expect(fallback).toBeDefined();
});
it('PLACE-022 — malformed KML returns 400', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kml`)
.set('Cookie', authCookie(user.id))
.attach('file', KML_MALFORMED_FIXTURE);
expect(res.status).toBe(400);
expect(res.body.error).toBeDefined();
});
it('PLACE-023 — non-UTF8 KML continues with warning', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const prefix = Buffer.from('<?xml version="1.0"?><kml><Document><Placemark><name>Caf');
const invalidByte = Buffer.from([0xe9]); // invalid UTF-8 sequence when used standalone
const suffix = Buffer.from('</name><Point><coordinates>2.1,48.1,0</coordinates></Point></Placemark></Document></kml>');
const nonUtf8Kml = Buffer.concat([prefix, invalidByte, suffix]);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kml`)
.set('Cookie', authCookie(user.id))
.attach('file', nonUtf8Kml, 'non-utf8.kml');
expect(res.status).toBe(201);
expect(res.body.count).toBe(1);
expect(Array.isArray(res.body.summary.warnings)).toBe(true);
expect(res.body.summary.warnings.join(' ')).toContain('not valid UTF-8');
});
it('PLACE-024 — POST /import/kmz with valid KMZ creates places', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kmz`)
.set('Cookie', authCookie(user.id))
.attach('file', KMZ_FIXTURE);
expect(res.status).toBe(201);
expect(res.body.count).toBeGreaterThan(0);
expect(res.body.summary).toBeDefined();
});
it('PLACE-025 — invalid KMZ returns 400', async () => {
const { user } = createUser(testDb);
const trip = createTrip(testDb, user.id);
const res = await request(app)
.post(`/api/trips/${trip.id}/places/import/kmz`)
.set('Cookie', authCookie(user.id))
.attach('file', Buffer.from('not-a-zip-archive'), 'invalid.kmz');
expect(res.status).toBe(400);
expect(String(res.body.error || '')).toContain('KMZ');
});
});
@@ -0,0 +1,78 @@
import { describe, it, expect } from 'vitest';
import {
buildCategoryNameLookup,
decodeUtf8WithWarning,
extractKmlPlacemarkNodes,
parseKmlPointCoordinates,
parsePlacemarkNode,
resolveCategoryIdForFolder,
sanitizeKmlDescription,
stripXmlNamespaces,
} from '../../../src/services/placeImport/kmlImportUtils';
describe('kmlImportUtils', () => {
it('strips KML namespaces and prefixes', () => {
const xml = '<kml xmlns="http://www.opengis.net/kml/2.2"><kml:Document><kml:Placemark /></kml:Document></kml>';
const stripped = stripXmlNamespaces(xml);
expect(stripped).not.toContain('xmlns');
expect(stripped).toContain('<Document>');
expect(stripped).toContain('<Placemark');
});
it('sanitizes HTML descriptions with br to newline', () => {
const input = 'Line 1<br>Line <b>2</b> &amp; more';
const output = sanitizeKmlDescription(input);
expect(output).toBe('Line 1\nLine 2 & more');
});
it('parses KML coordinate order lng,lat,alt', () => {
const parsed = parseKmlPointCoordinates('13.4050,52.5200,15');
expect(parsed).toEqual({ lat: 52.52, lng: 13.405 });
});
it('extracts placemarks from nested folders', () => {
const root = {
Document: {
Folder: {
name: 'Parent',
Folder: {
name: 'Child',
Placemark: { name: 'Nested', Point: { coordinates: '13.4,52.5,0' } },
},
},
},
};
const nodes = extractKmlPlacemarkNodes(root);
expect(nodes).toHaveLength(1);
expect(nodes[0].folderName).toBe('Child');
const parsed = parsePlacemarkNode(nodes[0]);
expect(parsed.name).toBe('Nested');
expect(parsed.lat).toBe(52.5);
expect(parsed.lng).toBe(13.4);
});
it('builds exact case-insensitive category lookup', () => {
const lookup = buildCategoryNameLookup([
{ id: 3, name: 'Museums' },
{ id: 4, name: 'Parks' },
]);
expect(resolveCategoryIdForFolder('museums', lookup)).toBe(3);
expect(resolveCategoryIdForFolder('Museum', lookup)).toBeNull();
expect(resolveCategoryIdForFolder('parks', lookup)).toBe(4);
});
it('returns warning for non-UTF8 payload', () => {
const buffer = Buffer.concat([
Buffer.from('<?xml version="1.0"?><kml><Document><Placemark><name>Caf'),
Buffer.from([0xe9]),
Buffer.from('</name></Placemark></Document></kml>'),
]);
const decoded = decodeUtf8WithWarning(buffer);
expect(decoded.warning).toContain('not valid UTF-8');
expect(decoded.text).toContain('<kml>');
});
});