diff --git a/server/scripts/import-boundaries-baseline.json b/server/scripts/import-boundaries-baseline.json index 8f6fd86f4..1bf5c692c 100644 --- a/server/scripts/import-boundaries-baseline.json +++ b/server/scripts/import-boundaries-baseline.json @@ -46,7 +46,6 @@ "booking-import -> airports/airports.data.ts", "booking-import -> llm-parse/image-input.ts", "collab -> storage/storage-upload.factory.ts", - "collections -> places/gpx-export.helpers.ts", "collections -> places/place-image.ts", "collections -> places/places.dto.ts", "collections -> places/places.helpers.ts", @@ -77,7 +76,6 @@ "reservation-import -> integrations/airtrail.dto.ts", "reservation-import -> llm-parse/image-input.ts", "reservations -> airports/airports.data.ts", - "roadtrip -> places/maps-dir.helpers.ts", "roadtrip -> plugins/contributions/plugin-route-normalize.ts", "roadtrip -> plugins/contributions/plugin-search.helpers.ts", "roadtrip -> plugins/kill-switch.ts", diff --git a/server/scripts/size-baseline.json b/server/scripts/size-baseline.json index 30e22ed75..5ac368cf2 100644 --- a/server/scripts/size-baseline.json +++ b/server/scripts/size-baseline.json @@ -13,7 +13,7 @@ "src/nest/doc-sync/doc-sync.service.ts": 1060, "src/nest/doc-sync/providers/synology-drive.client.ts": 1160, "src/nest/journey/journey-domain.service.ts": 1799, - "src/nest/places/places.service.ts": 2020, + "src/nest/places/places.service.ts": 1601, "src/nest/plugins/plugin-runtime.service.ts": 1463, "src/nest/plugins/runtime/plugin-sdk.ts": 1094, "src/nest/reservations/reservations.service.ts": 1239, diff --git a/server/scripts/tx-writes-baseline.json b/server/scripts/tx-writes-baseline.json index 387d50046..f29104732 100644 --- a/server/scripts/tx-writes-baseline.json +++ b/server/scripts/tx-writes-baseline.json @@ -36,9 +36,6 @@ "src/nest/journey/journey.controller.ts#JourneyController.providerPhotos": 2, "src/nest/journey/journey.controller.ts#JourneyController.uploadEntryPhotos": 4, "src/nest/journey/journey.mcp.ts#JourneyMcp.addJourneyProviderPhotos": 2, - "src/nest/maps/maps.service.ts#MapsService.getPlaceDetails": 2, - "src/nest/maps/maps.service.ts#MapsService.getPlaceDetailsExpanded": 2, - "src/nest/maps/maps.service.ts#MapsService.getPlacePhoto": 7, "src/nest/memories/photo-resolver.service.ts#PhotoResolverService.streamCachedThumbnail": 3, "src/nest/memories/photo-resolver.service.ts#PhotoResolverService.streamPhoto": 2, "src/nest/memories/synology.service.ts#SynologyService._fetchAllSynologyAlbums": 2, @@ -76,7 +73,7 @@ "src/nest/places/places.rpc.ts#PlacesRpc.update": 2, "src/nest/places/places.service.ts#PlacesService.backfillMissingAddresses": 2, "src/nest/places/places.service.ts#PlacesService.enrichImportedList": 2, - "src/nest/places/places.service.ts#PlacesService.enrichOne": 4, + "src/nest/places/places.service.ts#PlacesService.enrichOne": 3, "src/nest/places/places.service.ts#PlacesService.importGoogleDirections": 2, "src/nest/places/places.service.ts#PlacesService.importGoogleList": 2, "src/nest/places/places.service.ts#PlacesService.importNaverList": 2, diff --git a/server/src/nest/app.module.ts b/server/src/nest/app.module.ts index cd23980d2..f558a5283 100644 --- a/server/src/nest/app.module.ts +++ b/server/src/nest/app.module.ts @@ -48,6 +48,7 @@ import { JourneyModule } from './journey/journey.module'; import { LlmParseModule } from './llm-parse/llm-parse.module'; import { ManagedExtModule } from './managed/managed-ext.module'; import { MapsModule } from './maps/maps.module'; +import { PlaceImportModule } from './place-import/place-import.module'; import { McpTransportModule } from './mcp-transport/mcp-transport.module'; import { MemoriesModule } from './memories/memories.module'; import { NotificationsModule } from './notifications/notifications.module'; @@ -119,6 +120,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core'; SystemNoticesModule, GeoModule, MapsModule, + PlaceImportModule, GoogleQuotaModule, PlaceEnrichmentModule, PlaceShadowModule, diff --git a/server/src/nest/collections/collections.controller.ts b/server/src/nest/collections/collections.controller.ts index e9b369aac..411307fc2 100644 --- a/server/src/nest/collections/collections.controller.ts +++ b/server/src/nest/collections/collections.controller.ts @@ -56,7 +56,7 @@ import { CollectionGpxReadDto, } from './collections.dto'; import { PlaceRatingDto } from '../places/places.dto'; -import { CollectionGpxError } from './collection-gpx.helpers'; +import { CollectionGpxError } from '../place-import/place-import.types'; export const MAX_COVER_SIZE = 20 * 1024 * 1024; // Duplicated on purpose from trips.controller.ts (historical parity — no diff --git a/server/src/nest/collections/collections.module.ts b/server/src/nest/collections/collections.module.ts index 49b17e140..2d7ac2b34 100644 --- a/server/src/nest/collections/collections.module.ts +++ b/server/src/nest/collections/collections.module.ts @@ -26,6 +26,7 @@ import { StorageModule } from '../storage/storage.module'; import { StorageService } from '../storage/storage.service'; import { buildStorageUploadOptions } from '../storage/storage-upload.factory'; import { MAX_COVER_SIZE } from './collections.controller'; +import { PlaceImportModule } from '../place-import/place-import.module'; /** Collections domain (saved-places library). Registered in AppModule. * Exports CollectionsService for in-container consumers (PluginsModule's @@ -47,7 +48,7 @@ import { MAX_COVER_SIZE } from './collections.controller'; }), }), StorageModule, - NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule, + NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule, PlaceImportModule, MikroOrmModule.forFeature([ Collections, CollectionMembers, CollectionLabels, CollectionPlaces, CollectionPlaceRatings, Categories, Users, Trips, TripMembers, Places, PlaceRatings, Tags, diff --git a/server/src/nest/collections/collections.service.ts b/server/src/nest/collections/collections.service.ts index a68160fa3..86099f624 100644 --- a/server/src/nest/collections/collections.service.ts +++ b/server/src/nest/collections/collections.service.ts @@ -73,7 +73,7 @@ import type { CollectionImportablesResponse, } from '@trek/shared'; import { NotificationsService } from '../notifications/notifications.service'; -import { collectionFileToGpx, gpxToCollectionFile, type ExportedCollectionFile } from './collection-gpx.helpers'; +import { PlaceImportService, type ExportedCollectionFile } from '../place-import/place-import.service'; import { UnitOfWork } from '../database/unit-of-work'; /** Links are stored as a JSON TEXT column; parse on read, stringify on write. */ @@ -143,6 +143,7 @@ export class CollectionsService { @InjectRepository(PlaceRatings) private readonly tripPlaceRatings: PlaceRatingsRepository, @InjectRepository(Tags) private readonly tags: TagsRepository, @InjectRepository(Users) private readonly users: UsersRepository, + private readonly placeImport: PlaceImportService, ) {} /** @@ -398,17 +399,16 @@ export class CollectionsService { * only ever carry less of it. Same access rule too, since it is the same read. */ async exportCollectionGpx(userId: number, id: number): Promise { - return collectionFileToGpx(await this.exportCollection(userId, id)); + return this.placeImport.writeCollectionGpx(await this.exportCollection(userId, id)); } /** - * A GPX document read into the list file it amounts to. Touches no table: - * the file goes back to the browser to be shown, and comes back through - * importCollection like any other, so there is one import and one - * transaction whatever the format was. + * A GPX document read (by place-import) into the list file it amounts to. Touches no table: the file + * goes back to the browser to be shown, and comes back through importCollection like any other, so + * there is one import and one transaction whatever the format was. */ readCollectionGpx(body: CollectionGpxReadRequest): CollectionGpxReadResult { - return gpxToCollectionFile(body.gpx, body.file_name); + return this.placeImport.readCollectionGpx(body.gpx, body.file_name); } /** diff --git a/server/src/nest/common/google-maps-hosts.ts b/server/src/nest/common/google-maps-hosts.ts new file mode 100644 index 000000000..b6cf7487d --- /dev/null +++ b/server/src/nest/common/google-maps-hosts.ts @@ -0,0 +1,20 @@ +/** + * Which hosts a pasted Google Maps link may come from. Shared kernel because + * the maps link resolver and the directions import (place-import) both follow a + * short link through the SSRF guard and then require the landing host to be + * Google's own. + */ + +export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl']; + +/** + * Google Maps lives on every country domain — google.de, maps.google.co.uk, + * google.com.au — so the host is matched by shape. A fixed list of .com hosts + * would quietly stop resolving the ccTLD links people actually paste. The TLD + * labels stay short (2-3 letters, optionally two of them) so that + * `google.evil.com` is not a Google host. + */ +export function isGoogleMapsHost(hostname: string): boolean { + return GOOGLE_SHORT_HOSTS.includes(hostname) + || /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname); +} diff --git a/server/src/nest/maps/maps-url.resolver.ts b/server/src/nest/maps/maps-url.resolver.ts index afb3466d7..afff9a268 100644 --- a/server/src/nest/maps/maps-url.resolver.ts +++ b/server/src/nest/maps/maps-url.resolver.ts @@ -14,26 +14,13 @@ import { discardBody, exceedsDeclaredLength, readCappedText } from '../../utils/ import { UA, googleFtidFromMapsUrl } from './maps.helpers'; import { AMAP_SHORT_HOSTS, isAmapHost, parseAmapUrl } from './providers/amap.provider'; import { OsmClient } from './providers/osm.client'; +import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts'; // A Google Maps place page is a few hundred KB; the coordinates sit in the // embedded map data near the top, so two megabytes is plenty and keeps an // unbounded body out of memory. const MAX_MAPS_PAGE_BYTES = 2_000_000; -export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl']; - -/** - * Google Maps lives on every country domain — google.de, maps.google.co.uk, - * google.com.au — so the host is matched by shape. A fixed list of .com hosts - * would quietly stop resolving the ccTLD links people actually paste. The TLD - * labels stay short (2-3 letters, optionally two of them) so that - * `google.evil.com` is not a Google host. - */ -export function isGoogleMapsHost(hostname: string): boolean { - return GOOGLE_SHORT_HOSTS.includes(hostname) - || /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname); -} - export interface ResolvedMapsUrl { lat: number; lng: number; diff --git a/server/src/nest/maps/maps.service.ts b/server/src/nest/maps/maps.service.ts index 16bf05daf..6f5183980 100644 --- a/server/src/nest/maps/maps.service.ts +++ b/server/src/nest/maps/maps.service.ts @@ -27,7 +27,7 @@ import { NEARBY_DEFAULT_LIMIT, NEARBY_DEFAULT_RADIUS_M, nearbyCacheKey, nearestF export { readBrandIdentity, readWikiIdentity, type WikiIdentity } from './providers/wiki-identity'; export { withPhotoFetchSlot } from './photo-fetch-slot'; export type { BrandLogo, CommonsCandidate } from './providers/wikimedia.client'; -export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from './maps-url.resolver'; +export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts'; export { PLACES_PROVIDER_SETTING, PLACES_GOOGLE_ONLY_SETTING, type KeyedProvider } from './places-provider.selector'; // Places near a point (#976): cached longer than the POI boxes, because the diff --git a/server/src/nest/maps/place-photo.resolver.ts b/server/src/nest/maps/place-photo.resolver.ts index a0f24bf8f..e654fa597 100644 --- a/server/src/nest/maps/place-photo.resolver.ts +++ b/server/src/nest/maps/place-photo.resolver.ts @@ -29,6 +29,14 @@ export class PlacePhotoResolver { private readonly selector: PlacesProviderSelector, ) {} + /** + * The photo for one place: disk hit, negative cache, in-flight dedupe, then + * Google (for a Google id) and Wikimedia, under the shared photo-fetch slots. + * + * @txIndependent cache bookkeeping around network fetches: the stored photo, + * the negative-cache entry and the image_url backfill are each complete on + * their own, and a failed one costs the cache, never the answer. + */ async resolve( userId: number, placeId: string, diff --git a/server/src/nest/maps/providers/google-places.provider.ts b/server/src/nest/maps/providers/google-places.provider.ts index a4e4661b5..74f30fc2e 100644 --- a/server/src/nest/maps/providers/google-places.provider.ts +++ b/server/src/nest/maps/providers/google-places.provider.ts @@ -286,6 +286,9 @@ export class GooglePlacesClient { * download. The outcome tells a place with no photo apart from a call that * went wrong, because the negative cache keeps the first for a day and the * second for minutes. Throws only when the transport itself does. + * + * @txIndependent two billed calls with network I/O between them: each counts + * against the daily ceiling on its own, as it is made. */ async firstPhoto(placeId: string, apiKey: string): Promise { // Fetch details to get the photo name diff --git a/server/src/nest/places/maps-dir.helpers.ts b/server/src/nest/place-import/directions-url.helpers.ts similarity index 100% rename from server/src/nest/places/maps-dir.helpers.ts rename to server/src/nest/place-import/directions-url.helpers.ts diff --git a/server/src/nest/place-import/gpx.codec.ts b/server/src/nest/place-import/gpx.codec.ts new file mode 100644 index 000000000..222b6392b --- /dev/null +++ b/server/src/nest/place-import/gpx.codec.ts @@ -0,0 +1,103 @@ +/** + * GPX in and out: the one place the XML library is configured for GPX. + * + * Reading comes in two modes, chosen by the caller, because the two GPX + * importers want different things from the same document and have shipped + * that way: + * + * - `geometry` (the trip and Tours importers): tag values are parsed, so + * `12.5` arrives as a number, and the repeating elements are the + * six that carry places and geometry, matched on the name as written. + * `007` becomes the number 7 and character references stay + * undecoded, which is fine for track geometry. + * - `text` (the list importer in collections): text stays text, so a + * waypoint called 007 keeps its zeros, numeric character references + * (`é`) are decoded, and `link` / `label` repeat as well, matched on + * the local name so a namespaced `` counts. + * + * Writing goes through one builder and one document envelope (`writeGpx`), + * shared by the trip export and the list export: GPX 1.1, creator TREK, the + * schema location, then the caller's namespaces, metadata and elements. + */ +import { XMLBuilder, XMLParser } from 'fast-xml-parser'; + +export type GpxReadMode = 'geometry' | 'text'; + +/** Repeating elements in geometry mode: the places and the geometry, by name as written. */ +const GEOMETRY_ARRAYS = ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte']; + +/** Repeating elements in text mode, matched on the local name. */ +const TEXT_ARRAYS = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']); + +const localName = (name: string): string => name.slice(name.indexOf(':') + 1); + +/** The GPX parser for one read mode. Built once per mode below; exported for the tests. */ +export function createGpxParser(mode: GpxReadMode): XMLParser { + if (mode === 'geometry') { + return new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + isArray: (name) => GEOMETRY_ARRAYS.includes(name), + }); + } + // Entities declared in a DOCTYPE never reach the text parser: the list reader + // refuses a document carrying one before parsing. + return new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + parseTagValue: false, + parseAttributeValue: false, + htmlEntities: true, + isArray: (name, _path, _leaf, isAttribute) => !isAttribute && TEXT_ARRAYS.has(localName(name)), + }); +} + +/** Frozen at import, one shared instance per mode. */ +export const gpxGeometryParser = createGpxParser('geometry'); +export const gpxTextParser = createGpxParser('text'); + +export const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1'; + +/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer + * device resolves and keeps the file from carrying float noise. */ +export function coord(n: number): string { + return Number(n.toFixed(7)).toString(); +} + +const gpxBuilder = new XMLBuilder({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + format: true, + indentBy: ' ', + suppressEmptyNode: true, +}); + +/** What goes inside ``; an empty element list is left out entirely. */ +export interface GpxDocument { + /** Extra `xmlns:` bindings, written after the default namespace. */ + namespaces?: Record; + metadata: Record; + wpt?: unknown[]; + rte?: unknown[]; + trk?: unknown[]; +} + +/** A GPX 1.1 document, elements in the schema's order (metadata, wpt, rte, trk). */ +export function writeGpx(doc: GpxDocument): string { + const namespaces = Object.fromEntries(Object.entries(doc.namespaces ?? {}).map(([prefix, uri]) => [`@_xmlns:${prefix}`, uri])); + return gpxBuilder.build({ + '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, + gpx: { + '@_version': '1.1', + '@_creator': 'TREK', + '@_xmlns': GPX_NAMESPACE, + ...namespaces, + '@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', + '@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`, + metadata: doc.metadata, + ...(doc.wpt?.length ? { wpt: doc.wpt } : {}), + ...(doc.rte?.length ? { rte: doc.rte } : {}), + ...(doc.trk?.length ? { trk: doc.trk } : {}), + }, + }); +} diff --git a/server/src/nest/places/kml-import.helpers.ts b/server/src/nest/place-import/kml.codec.ts similarity index 65% rename from server/src/nest/places/kml-import.helpers.ts rename to server/src/nest/place-import/kml.codec.ts index f6d7f3812..b06dbe6c0 100644 --- a/server/src/nest/places/kml-import.helpers.ts +++ b/server/src/nest/place-import/kml.codec.ts @@ -1,27 +1,38 @@ +/** + * KML and KMZ in: the one place the XML library is configured for KML. + * + * A KMZ is unpacked to its KML (doc.kml preferred, the decompressed size + * capped), the bytes are decoded as UTF-8 (loosely, with a warning, when they + * are not), the document is validated and parsed, and every Placemark comes + * back with the folder it sat in, its name, a plain-text description, and + * either a point or a path's first point plus the path as route geometry. + * Which placemarks become places, and under which category, is the importing + * domain's decision. + */ import { TextDecoder } from 'util'; +import { XMLParser, XMLValidator } from 'fast-xml-parser'; +import unzipper from 'unzipper'; import { stripHtmlTags } from '../common/stripHtmlTags'; +import type { KmlDocumentRead, KmlImportSummary, ParsedKmlPlacemark } from './place-import.types'; -export interface ParsedKmlPlacemark { - name: string | null; - description: string | null; - lat: number | null; - lng: number | null; - folderName: string | null; - routeGeometry: string | null; -} +export type { KmlImportSummary, ParsedKmlPlacemark } from './place-import.types'; export interface KmlPlacemarkNode { placemark: any; folderName: string | null; } -export interface KmlImportSummary { - totalPlacemarks: number; - createdCount: number; - skippedCount: number; - warnings: string[]; - errors: string[]; -} +const kmlParser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: '@_', + removeNSPrefix: true, + isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name), + // Treat as raw text so mixed-content HTML (e.g.
, ) + // is returned as a string instead of a parsed object. + stopNodes: ['*.description'], +}); + +export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB const UTF8_DECODER_FATAL = new TextDecoder('utf-8', { fatal: true }); const UTF8_DECODER_LOOSE = new TextDecoder('utf-8'); @@ -144,25 +155,6 @@ export function createKmlImportSummary(totalPlacemarks: number): KmlImportSummar }; } -export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map { - const lookup = new Map(); - for (const category of categories) { - const normalizedName = category.name.trim().toLowerCase(); - if (!normalizedName) continue; - if (!lookup.has(normalizedName)) { - lookup.set(normalizedName, category.id); - } - } - return lookup; -} - -export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map): number | null { - if (!folderName) return null; - const normalizedFolder = folderName.trim().toLowerCase(); - if (!normalizedFolder) return null; - return lookup.get(normalizedFolder) ?? null; -} - export function extractKmlPlacemarkNodes(kmlRoot: any): KmlPlacemarkNode[] { const nodes: KmlPlacemarkNode[] = []; @@ -211,3 +203,62 @@ export function parsePlacemarkNode(node: KmlPlacemarkNode): ParsedKmlPlacemark { routeGeometry, }; } + +/** + * A KML document as its placemarks (each level's own before its folders'), with the summary the + * import fills in. Throws the two messages the import route has always + * answered for a broken file. + */ +export function readKmlDocument(fileBuffer: Buffer): KmlDocumentRead { + const decoded = decodeUtf8WithWarning(fileBuffer); + + const validationResult = XMLValidator.validate(decoded.text); + if (validationResult !== true) { + throw new Error('Malformed KML: invalid XML structure'); + } + + const parsed = kmlParser.parse(decoded.text); + const kmlRoot = parsed?.kml ?? parsed; + + if (!kmlRoot || typeof kmlRoot !== 'object') { + throw new Error('Malformed KML: could not parse XML'); + } + + const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot); + const summary = createKmlImportSummary(placemarkNodes.length); + + if (decoded.warning) { + summary.warnings.push(decoded.warning); + } + + return { placemarks: placemarkNodes.map(parsePlacemarkNode), summary }; +} + +// --------------------------------------------------------------------------- +// KMZ unpacking +// --------------------------------------------------------------------------- + +export async function unpackKmzToKml( + kmzBuffer: Buffer, + decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT, +): Promise { + let zip; + try { + zip = await unzipper.Open.buffer(kmzBuffer); + } catch { + throw new Error('Invalid KMZ archive.'); + } + + const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml')); + if (kmlEntries.length === 0) { + throw new Error('KMZ archive does not contain a KML file.'); + } + + const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0]; + + if (preferredEntry.uncompressedSize > decompressedSizeLimit) { + throw new Error('KMZ archive exceeds the maximum allowed decompressed size.'); + } + + return preferredEntry.buffer(); +} diff --git a/server/src/nest/place-import/place-import.module.ts b/server/src/nest/place-import/place-import.module.ts new file mode 100644 index 000000000..a5e7865e1 --- /dev/null +++ b/server/src/nest/place-import/place-import.module.ts @@ -0,0 +1,30 @@ +import { Module } from '@nestjs/common'; +import { PlaceImportService } from './place-import.service'; +import { GpxProvider } from './providers/gpx.provider'; +import { CollectionGpxProvider } from './providers/collection-gpx.provider'; +import { KmlProvider } from './providers/kml.provider'; +import { GoogleListProvider } from './providers/google-list.provider'; +import { GoogleDirectionsProvider } from './providers/google-directions.provider'; +import { NaverListProvider } from './providers/naver-list.provider'; + +/** + * Place import (leaf module): reads places out of GPX, KML/KMZ, Google lists, + * Google directions links and Naver lists, and writes GPX. No controller, no + * table of its own: PlacesModule, CollectionsModule and ToursModule import it + * and persist what it reads. It imports no other + * domain: the stops of a directions link that carry only a name are geocoded + * through a function the importer hands in (MapsService.geocodeQuery). + */ +@Module({ + providers: [ + PlaceImportService, + GpxProvider, + CollectionGpxProvider, + KmlProvider, + GoogleListProvider, + GoogleDirectionsProvider, + NaverListProvider, + ], + exports: [PlaceImportService], +}) +export class PlaceImportModule {} diff --git a/server/src/nest/place-import/place-import.service.ts b/server/src/nest/place-import/place-import.service.ts new file mode 100644 index 000000000..fa4ea030c --- /dev/null +++ b/server/src/nest/place-import/place-import.service.ts @@ -0,0 +1,159 @@ +import { Injectable } from '@nestjs/common'; +import type { CollectionGpxExport, CollectionGpxReadResult } from '@trek/shared'; +import { GpxProvider, type GpxExportInput, type GpxExportOptions } from './providers/gpx.provider'; +import { CollectionGpxProvider, type ExportedCollectionFile } from './providers/collection-gpx.provider'; +import { KmlProvider } from './providers/kml.provider'; +import { GoogleListProvider } from './providers/google-list.provider'; +import { GoogleDirectionsProvider } from './providers/google-directions.provider'; +import { NaverListProvider } from './providers/naver-list.provider'; +import type { + DirectionsRead, + DirectionsRedirect, + GoogleListPlace, + GpxImportOptions, + KmlDocumentRead, + ListImportError, + ListRead, + NaverListPlace, + PreparedGpxPlace, +} from './place-import.types'; + +// The directions-link reading is also what the road trip planner reads a +// shared route with, and what the import routes dispatch on. +export { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS, type DirWaypoint } from './directions-url.helpers'; +export type { GpxExportDay, GpxExportInput, GpxExportOptions, GpxExportPlace } from './providers/gpx.provider'; +export type { ExportedCollectionFile } from './providers/collection-gpx.provider'; + +/** Places a name, or says nobody can. */ +export type Geocoder = (query: string) => Promise<{ lat: number; lng: number } | null>; + +/** + * Places out of files and shared lists: the one layer that reads GPX, KML/KMZ, + * Google lists, Google directions links and Naver lists, and writes GPX. + * + * One provider per source behind it, over one GPX codec and one KML codec. + * Nothing here persists: a trip import (PlacesService), the Tours import + * (ToursService) and a list import or export (CollectionsService) each take + * what comes back and write their own rows, so the dedup, the transactions and + * the broadcasts stay with the domain that owns them. + */ +@Injectable() +export class PlaceImportService { + constructor( + private readonly gpx: GpxProvider, + private readonly collectionGpx: CollectionGpxProvider, + private readonly kml: KmlProvider, + private readonly googleList: GoogleListProvider, + private readonly googleDirections: GoogleDirectionsProvider, + private readonly naverList: NaverListProvider, + ) {} + + // ── GPX ──────────────────────────────────────────────────────────────────── + + /** The places a trip GPX import would create, nothing written. */ + readGpx(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] { + return this.gpx.read(fileBuffer, opts); + } + + /** A trip as a GPX document, or null when the selection yields nothing. */ + writeTripGpx(input: GpxExportInput, opts: GpxExportOptions = {}): string | null { + return this.gpx.write(input, opts); + } + + tripGpxFilename(tripTitle: string): string { + return this.gpx.filename(tripTitle); + } + + /** A list file as GPX (#2301). */ + writeCollectionGpx(file: ExportedCollectionFile): CollectionGpxExport { + return this.collectionGpx.write(file); + } + + /** A GPX document as the list file it amounts to. Throws CollectionGpxError on a refusal. */ + readCollectionGpx(source: string, fileName?: string): CollectionGpxReadResult { + return this.collectionGpx.read(source, fileName); + } + + // ── KML / KMZ ────────────────────────────────────────────────────────────── + + /** A .kml or .kmz upload, by its file name. */ + readMapFile(fileBuffer: Buffer, filename: string): Promise { + return this.kml.readMapFile(fileBuffer, filename); + } + + readKmz(kmzBuffer: Buffer): Promise { + return this.kml.readKmz(kmzBuffer); + } + + readKml(fileBuffer: Buffer): KmlDocumentRead { + return this.kml.readKml(fileBuffer); + } + + // ── Shared lists ─────────────────────────────────────────────────────────── + + /** A Google Maps list, or the directions link a short link turned out to be. */ + readGoogleList(url: string): Promise | DirectionsRedirect | ListImportError> { + return this.googleList.read(url); + } + + readNaverList(url: string): Promise | ListImportError> { + return this.naverList.read(url); + } + + /** + * The stops of a shared directions link, placed on the map. + * + * `geocode` places a stop the link names without coordinates. It is handed + * in rather than injected so this layer stays below the maps domain: the + * importer passes MapsService.geocodeQuery. + * + * A stop the link spells out in coordinates is taken as it stands; one that + * is only a name is geocoded, one request each. A name nobody can place is + * left out rather than failing the import, because a route of six stops with + * five findable is five stops more than the traveller had. + */ + async readGoogleDirections(url: string, geocode: Geocoder): Promise { + const waypoints = await this.googleDirections.read(url); + if (!Array.isArray(waypoints)) return waypoints; + + const places: GoogleListPlace[] = []; + let unplaceable = 0; + for (const wp of waypoints) { + if (wp.lat !== null && wp.lng !== null) { + // A stop written as coordinates has no name of its own, and the coordinates are a + // better label than "Stop 3": they are what the traveller can look up. + places.push({ + name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`, + lat: wp.lat, + lng: wp.lng, + notes: null, + googleFtid: null, + }); + continue; + } + if (!wp.name) continue; + try { + // Through geocodeQuery, which asks the TREK index first and only falls + // through to Nominatim for what it does not know — and does so on the + // BACKGROUND lane. That matters here more than anywhere: this loop runs + // up to thirty times in one request, each Nominatim call taking the next + // slot on a 1.1 s process-wide throttle, so on the interactive lane one + // pasted link made everybody else's place search queue behind it for + // half a minute. An index hit costs no slot at all. + const hit = await geocode(wp.name); + // The name from the link, not the one the geocoder answers with: somebody who + // typed a nickname into Google should not find a street address on their trip. + if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null }); + else unplaceable++; + } catch { + // A geocoder that is down or rate-limited costs this one stop, not the import. + unplaceable++; + } + } + + if (places.length < 2) { + return { error: 'None of the stops in that link could be placed on the map.', status: 400 }; + } + return { places, unplaceable }; + } +} diff --git a/server/src/nest/place-import/place-import.types.ts b/server/src/nest/place-import/place-import.types.ts new file mode 100644 index 000000000..58df50699 --- /dev/null +++ b/server/src/nest/place-import/place-import.types.ts @@ -0,0 +1,104 @@ +/** + * What the place-import layer hands its callers: places read out of a file or + * a shared list, not yet persisted. Persisting them (dedup against the trip, + * insert, track colours, enrichment) stays with the domain that owns the rows, + * PlacesService for a trip and CollectionsService for a list. + */ +import type { CollectionGpxProblem } from '@trek/shared'; + +/** One place parsed out of a GPX file, not yet persisted. */ +export interface PreparedGpxPlace { + name: string; + lat: number; + lng: number; + description: string | null; + routeGeometry?: string; +} + +export interface GpxImportOptions { + importWaypoints?: boolean; + importRoutes?: boolean; + importTracks?: boolean; + /** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */ + defaultName?: string; +} + +export interface KmlImportOptions { + importPoints?: boolean; + importPaths?: boolean; +} + +export interface KmlImportSummary { + totalPlacemarks: number; + createdCount: number; + skippedCount: number; + warnings: string[]; + errors: string[]; +} + +export interface ParsedKmlPlacemark { + name: string | null; + description: string | null; + lat: number | null; + lng: number | null; + folderName: string | null; + routeGeometry: string | null; +} + +/** A KML document read: every placemark, and the summary the import fills in. */ +export interface KmlDocumentRead { + placemarks: ParsedKmlPlacemark[]; + summary: KmlImportSummary; +} + +/** An import that could not happen, with the status and the exact message the route answers. */ +export interface ListImportError { + error: string; + status: number; +} + +/** A place out of a Google list or a directions link. */ +export interface GoogleListPlace { + name: string; + lat: number; + lng: number; + notes: string | null; + googleFtid: string | null; +} + +/** A place out of a Naver list. */ +export interface NaverListPlace { + name: string; + lat: number; + lng: number; + notes: string | null; + address: string | null; +} + +/** A shared list read: its name and the places that carried coordinates. */ +export interface ListRead

{ + listName: string; + places: P[]; +} + +/** A Google link that turned out to be a route once its redirect was followed. */ +export interface DirectionsRedirect { + directions: string; +} + +/** The stops of a directions link, placed on the map; `unplaceable` counts the ones nobody could find. */ +export interface DirectionsRead { + places: GoogleListPlace[]; + unplaceable: number; +} + +/** Why a GPX document was refused for a list, carried to the controller as the response's `code`. */ +export class CollectionGpxError extends Error { + constructor( + readonly code: CollectionGpxProblem, + message: string, + ) { + super(message); + this.name = 'CollectionGpxError'; + } +} diff --git a/server/src/nest/collections/collection-gpx.helpers.ts b/server/src/nest/place-import/providers/collection-gpx.provider.ts similarity index 89% rename from server/src/nest/collections/collection-gpx.helpers.ts rename to server/src/nest/place-import/providers/collection-gpx.provider.ts index 565001cb7..4417178c4 100644 --- a/server/src/nest/collections/collection-gpx.helpers.ts +++ b/server/src/nest/place-import/providers/collection-gpx.provider.ts @@ -1,4 +1,4 @@ -import { XMLParser } from 'fast-xml-parser'; +import { Injectable } from '@nestjs/common'; import type { ZodType } from 'zod'; import { COLLECTION_FILE_FORMAT, @@ -16,11 +16,13 @@ import { type CollectionFileLabel, type CollectionFilePlace, type CollectionGpxExport, - type CollectionGpxProblem, type CollectionGpxReadResult, type CollectionLink, } from '@trek/shared'; -import { coord, gpxBuilder } from '../places/gpx-export.helpers'; +import { coord, gpxTextParser, writeGpx } from '../gpx.codec'; +import { CollectionGpxError } from '../place-import.types'; + +export { CollectionGpxError } from '../place-import.types'; /** * A list as GPX and back (#2301). @@ -32,16 +34,15 @@ import { coord, gpxBuilder } from '../places/gpx-export.helpers'; * fields of a place map onto a waypoint, and that table lives in the contract * (`COLLECTION_GPX_PLACE_FIELDS`). * - * Same library as the trip importer and exporter in places/, with a reader of - * its own: the trip parser turns `007` into the number 7 and does - * not decode character references, which is fine for track geometry and wrong - * for the names and notes a list is made of. + * Same codec as the trip importer and exporter (gpx.codec), in its `text` + * read mode: the trip parser turns `007` into the number 7 and + * does not decode character references, which is fine for track geometry and + * wrong for the names and notes a list is made of. */ /** A list file as the exporter builds it, its places already in the file's shape. */ export type ExportedCollectionFile = Omit & { places: CollectionFilePlace[] }; -const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1'; /** The prefix this writer binds; the reader looks the namespace up instead. */ const TREK = 'trek'; /** OsmAnd writes its favourites with the address in its own namespace. */ @@ -148,19 +149,7 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx wpt.push(waypoint(place, lat, lng)); } - const gpx: string = gpxBuilder.build({ - '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, - gpx: { - '@_version': '1.1', - '@_creator': 'TREK', - '@_xmlns': GPX_NAMESPACE, - [`@_xmlns:${TREK}`]: COLLECTION_GPX_NAMESPACE, - '@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - '@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`, - metadata: metadata(file), - ...(wpt.length ? { wpt } : {}), - }, - }); + const gpx = writeGpx({ namespaces: { [TREK]: COLLECTION_GPX_NAMESPACE }, metadata: metadata(file), wpt }); return { name: file.name, gpx, waypoints: wpt.length, omitted }; } @@ -168,17 +157,6 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx // Reading // --------------------------------------------------------------------------- -/** Why a document was refused, carried to the controller as the response's `code`. */ -export class CollectionGpxError extends Error { - constructor( - readonly code: CollectionGpxProblem, - message: string, - ) { - super(message); - this.name = 'CollectionGpxError'; - } -} - /** * The most elements a document may open before it is parsed at all. * @@ -189,27 +167,6 @@ export class CollectionGpxError extends Error { */ export const MAX_GPX_ELEMENTS = 100_000; -/** Elements that may repeat, so they always parse as arrays. Matched on the local name. */ -const REPEATED = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']); - -const localName = (name: string): string => name.slice(name.indexOf(':') + 1); - -/** - * Text stays text (`parseTagValue: false`), so a waypoint called 007 keeps its - * zeros. `htmlEntities` is on for the numeric character references it also - * unlocks (`é`), which this library otherwise leaves undecoded. Entities - * declared in a DOCTYPE never reach it: a document carrying one is refused - * before parsing. - */ -const reader = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - parseTagValue: false, - parseAttributeValue: false, - htmlEntities: true, - isArray: (name, _path, _leaf, isAttribute) => !isAttribute && REPEATED.has(localName(name)), -}); - type XmlNode = Record; function isNode(value: unknown): value is XmlNode { @@ -410,7 +367,7 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti const xml = refuseBeforeParsing(source); let doc: unknown; try { - doc = reader.parse(xml, true); + doc = gpxTextParser.parse(xml, true); } catch { throw new CollectionGpxError('unreadable', 'That GPX file is not well-formed XML'); } @@ -481,3 +438,15 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti if (!file) throw new CollectionGpxError('unreadable', 'That GPX file could not be read as a list'); return { file, skipped, track_points: trackPoints }; } + +/** The GPX source for lists: the list file out as GPX, a GPX file in as a list file. */ +@Injectable() +export class CollectionGpxProvider { + write(file: ExportedCollectionFile): CollectionGpxExport { + return collectionFileToGpx(file); + } + + read(source: string, fileName?: string): CollectionGpxReadResult { + return gpxToCollectionFile(source, fileName); + } +} diff --git a/server/src/nest/place-import/providers/google-directions.provider.ts b/server/src/nest/place-import/providers/google-directions.provider.ts new file mode 100644 index 000000000..591c72a97 --- /dev/null +++ b/server/src/nest/place-import/providers/google-directions.provider.ts @@ -0,0 +1,59 @@ +/** + * The stops of a shared Google Maps directions link, read off the link itself. + * + * The other half of the request the list import answers: people share a drive + * far more often than they share a list. No API key is involved and no call is + * made to Google for the link itself — the stops are in the URL, which is the + * whole reason this is possible at all. A short link is followed hop by hop + * through the SSRF guard first, and the host that counts is the one the link + * lands on. + * + * A stop the link spells out in coordinates is taken as it stands; one that is + * only a name comes back without coordinates for the caller to geocode. This + * provider asks no geocoder: choosing which source answers is MapsService's + * job, reached through PlaceImportService. + */ +import { Injectable } from '@nestjs/common'; +import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard'; +import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../../common/google-maps-hosts'; +import { parseDirectionsUrl, type DirWaypoint } from '../directions-url.helpers'; +import type { ListImportError } from '../place-import.types'; + +@Injectable() +export class GoogleDirectionsProvider { + /** The stops in driving order (two at least), or the refusal the route answers. */ + async read(url: string): Promise { + const ssrf = await checkSsrf(url); + if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; + + let parsed: URL; + try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; } + + // Short links are resolved hop by hop through the guard, exactly as the list import + // does it: a maps.app.goo.gl that 302s to an internal address is still blocked. + let resolvedUrl = url; + if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) { + try { + const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); + resolvedUrl = redirectRes.url; + } catch (err) { + if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; + throw err; + } + } + + // Checked after resolving, not before: the host that counts is the one the link lands + // on, and `/maps/dir/` is a path anybody could serve. + let host = ''; + try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ } + if (!isGoogleMapsHost(host)) { + return { error: 'That link is not a Google Maps link.', status: 400 }; + } + + const waypoints = parseDirectionsUrl(resolvedUrl); + if (waypoints.length < 2) { + return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 }; + } + return waypoints; + } +} diff --git a/server/src/nest/place-import/providers/google-list.provider.ts b/server/src/nest/place-import/providers/google-list.provider.ts new file mode 100644 index 000000000..66d35c819 --- /dev/null +++ b/server/src/nest/place-import/providers/google-list.provider.ts @@ -0,0 +1,177 @@ +/** + * A shared Google Maps list, read from Google's own list endpoint. + * + * The link is user input, so it is checked by the SSRF guard first and a short + * link is followed hop by hop through it. A short link that lands on a route + * (the Share sheet on a phone produces exactly that) is handed back as a + * directions link for the directions source; everything else must carry a list + * id. The answer is capped before and after it is read, because its size is + * attacker-influenced through the list id. Every refusal is the exact message + * and status the import route has always answered. + */ +import { Injectable } from '@nestjs/common'; +import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard'; +import { isDirectionsUrl } from '../directions-url.helpers'; +import type { DirectionsRedirect, GoogleListPlace, ListImportError, ListRead } from '../place-import.types'; + +/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */ +export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB + +/** The browser user agent the list endpoints answer. */ +export const LIST_USER_AGENT = + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; + +export function googleMapsHexId(value: unknown): string | null { + if (typeof value !== 'string' && typeof value !== 'number') return null; + const raw = String(value).trim(); + if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase(); + if (!/^-?\d+$/.test(raw)) return null; + try { + const parsed = BigInt(raw); + const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed; + return `0x${unsigned.toString(16)}`; + } catch { + return null; + } +} + +export function googleMapsFeatureIdFromItem(item: unknown): string | null { + if (!Array.isArray(item)) return null; + const candidates = [ + Array.isArray(item[1]) ? item[1][6] : null, + Array.isArray(item[7]) ? item[7][1] : null, + ]; + + for (const ids of candidates) { + if (!Array.isArray(ids) || ids.length < 2) continue; + const first = googleMapsHexId(ids[0]); + const second = googleMapsHexId(ids[1]); + if (first && second) return `${first}:${second}`; + } + + return null; +} + +@Injectable() +export class GoogleListProvider { + async read(url: string): Promise | DirectionsRedirect | ListImportError> { + let listId: string | null = null; + let resolvedUrl = url; + + // SSRF guard: validate user-supplied URL before fetching + const ssrf = await checkSsrf(url); + if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; + + // Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are + // followed manually so every hop is re-checked against the SSRF guard — a + // short link that 302s to an internal IP is blocked even though the initial + // host is public. + if (url.includes('goo.gl') || url.includes('maps.app')) { + try { + const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); + resolvedUrl = redirectRes.url; + } catch (err) { + if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; + throw err; + } + } + + // A route, once the redirect is followed. The dispatch upstream decides on + // the raw URL, and a short link's path is `/` — it matches nothing, so + // every route shared from the Google Maps app arrived here and was answered + // with "could not extract list ID", which is the complaint the directions + // import was written to remove. The Share sheet on a phone produces exactly + // this shape, and the box says a directions link works. + // + // Handed on with the resolved URL, so the hop is not made twice. + if (isDirectionsUrl(resolvedUrl)) return { directions: resolvedUrl }; + + // Pattern: /placelists/list/{ID} + const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/); + if (plMatch) listId = plMatch[1]; + + // Pattern: !2s{ID} in data URL params + if (!listId) { + const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/); + if (dataMatch) listId = dataMatch[1]; + } + + if (!listId) { + // A single-place share link (…/maps/place/…) carries no list id — point the user at + // the place search box instead of a cryptic "could not extract list ID" (#1304). + if (resolvedUrl.includes('/maps/place/')) { + return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 }; + } + return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 }; + } + + // Fetch list data from Google Maps internal API + const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`; + const apiRes = await fetch(apiUrl, { + headers: { 'User-Agent': LIST_USER_AGENT }, + signal: AbortSignal.timeout(15000), + }); + + if (!apiRes.ok) { + return { error: 'Failed to fetch list from Google Maps', status: 502 }; + } + + // Cap the declared body before reading it (transit.service precedent): the + // response is attacker-influenced via the list id, and buffering it whole + // used to be unbounded. + const declared = Number(apiRes.headers?.get('content-length') ?? 0); + if (declared > MAX_LIST_RESPONSE_BYTES) { + return { error: 'Failed to fetch list from Google Maps', status: 502 }; + } + + const rawText = await apiRes.text(); + if (rawText.length > MAX_LIST_RESPONSE_BYTES) { + return { error: 'Failed to fetch list from Google Maps', status: 502 }; + } + const jsonStr = rawText.substring(rawText.indexOf('\n') + 1); + // The provider hands back a JS-prefixed array; a malformed body is a + // provider problem, not a crash — surface the same 400 an unreadable + // payload already produced instead of throwing a SyntaxError. + let listData: unknown; + try { + listData = JSON.parse(jsonStr); + } catch { + return { error: 'Invalid list data received from Google Maps', status: 400 }; + } + if (!Array.isArray(listData)) { + return { error: 'Invalid list data received from Google Maps', status: 400 }; + } + + const meta = listData[0]; + if (!meta) { + return { error: 'Invalid list data received from Google Maps', status: 400 }; + } + + const listName = meta[4] || 'Google Maps List'; + const items = meta[8]; + + if (!Array.isArray(items) || items.length === 0) { + return { error: 'List is empty or could not be read', status: 400 }; + } + + // Parse place data from items + const places: GoogleListPlace[] = []; + for (const item of items) { + const coords = item?.[1]?.[5]; + const lat = coords?.[2]; + const lng = coords?.[3]; + const name = item?.[2]; + const note = item?.[3] || null; + + if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) { + places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) }); + } + } + + if (places.length === 0) { + return { error: 'No places with coordinates found in list', status: 400 }; + } + + return { listName, places }; + } +} diff --git a/server/src/nest/places/gpx-export.helpers.ts b/server/src/nest/place-import/providers/gpx.provider.ts similarity index 53% rename from server/src/nest/places/gpx-export.helpers.ts rename to server/src/nest/place-import/providers/gpx.provider.ts index db7db2666..6dec27197 100644 --- a/server/src/nest/places/gpx-export.helpers.ts +++ b/server/src/nest/place-import/providers/gpx.provider.ts @@ -1,8 +1,11 @@ -import { XMLBuilder } from 'fast-xml-parser'; +import { Injectable } from '@nestjs/common'; +import { coord, gpxGeometryParser, writeGpx } from '../gpx.codec'; +import type { GpxImportOptions, PreparedGpxPlace } from '../place-import.types'; /** - * GPX writer, the mirror of the importer in places.helpers.ts. Same library, the - * builder half rather than the parser half, so nothing new is pulled in. + * A trip as GPX and back: the reader that turns a GPX file into the places an + * import would create, and the writer that is its mirror. Both go through + * gpx.codec (the geometry read mode, the shared document writer). * * The import decides the shape here: a `` becomes a place with coordinates, a * `` or `` becomes a place carrying `route_geometry`, so writing back is @@ -46,12 +49,6 @@ export interface GpxExportOptions { type Pt = { lat: number; lng: number; ele: number | null }; -/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer - * device resolves and keeps the file from carrying float noise. */ -export function coord(n: number): string { - return Number(n.toFixed(7)).toString(); -} - function parseGeometry(raw: string | null): Pt[] { if (!raw) return []; let parsed: unknown; @@ -81,15 +78,6 @@ function describe(place: GpxExportPlace): string | undefined { return parts.length ? parts.join(', ') : undefined; } -/** Shared with the list writer in collections/, which writes the same dialect. */ -export const gpxBuilder = new XMLBuilder({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - format: true, - indentBy: ' ', - suppressEmptyNode: true, -}); - /** * Build a GPX 1.1 document. Returns null when the selection produced nothing at all, * so the caller can answer 404 rather than hand over an empty file that silently @@ -154,22 +142,7 @@ export function buildGpx(input: GpxExportInput, opts: GpxExportOptions = {}): st if (wpt.length === 0 && trk.length === 0 && rte.length === 0) return null; - const doc = { - '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, - gpx: { - '@_version': '1.1', - '@_creator': 'TREK', - '@_xmlns': 'http://www.topografix.com/GPX/1/1', - '@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - '@_xsi:schemaLocation': 'http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd', - metadata: { name: input.tripTitle }, - ...(wpt.length ? { wpt } : {}), - ...(rte.length ? { rte } : {}), - ...(trk.length ? { trk } : {}), - }, - }; - - return gpxBuilder.build(doc); + return writeGpx({ metadata: { name: input.tripTitle }, wpt, rte, trk }); } /** Filenames land on the receiving filesystem, so its reserved characters are @@ -187,3 +160,96 @@ export function gpxFilename(tripTitle: string): string { // lone surrogate is exactly what URI-encoding chokes on. return `${[...base].slice(0, 60).join('') || 'trip'}.gpx`; } + +/** + * Parses a GPX file into the places an import would create, without writing + * anything. Waypoints become places, each route and each track becomes one + * place carrying its geometry. + */ +export function prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] { + const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts; + + const parsed = gpxGeometryParser.parse(fileBuffer.toString('utf-8')); + const gpx = parsed?.gpx; + if (!gpx) return []; + + const str = (v: unknown) => (v != null ? String(v).trim() : null); + const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; }; + + // Routes and tracks rarely carry their own . Without one they all fall back to the + // same generic label, so name-based dedup drops every import after the first. Derive a + // base from the source filename (the requested behaviour) and suffix an index so multiple + // geometries from one file stay distinct. + const rawName = str(defaultName); + const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null; + let geoSeq = 0; + const geoName = (explicit: string | null, fallback: string): string => { + if (explicit) return explicit; + geoSeq++; + const base = baseName || fallback; + return geoSeq === 1 ? base : `${base} ${geoSeq}`; + }; + + const waypoints: PreparedGpxPlace[] = []; + + // 1) Parse elements (named waypoints / POIs) + if (importWaypoints) { + for (const wpt of gpx.wpt ?? []) { + const lat = num(wpt['@_lat']); + const lng = num(wpt['@_lon']); + if (lat === null || lng === null) continue; + waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) }); + } + } + + // 2) Parse routes as polyline-places (one place per route with route_geometry) + if (importRoutes) { + for (const rte of gpx.rte ?? []) { + const pts = (rte.rtept ?? []) + .map((pt: Record) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) })) + .filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>; + if (pts.length === 0) continue; + const hasAllEle = pts.every(p => p.ele !== null); + const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]); + waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) }); + } + } + + // 3) Extract full track geometry from + if (importTracks) { + for (const trk of gpx.trk ?? []) { + const trackPoints: { lat: number; lng: number; ele: number | null }[] = []; + for (const seg of trk.trkseg ?? []) { + for (const pt of seg.trkpt ?? []) { + const lat = num(pt['@_lat']); + const lng = num(pt['@_lon']); + if (lat === null || lng === null) continue; + trackPoints.push({ lat, lng, ele: num(pt.ele) }); + } + } + if (trackPoints.length === 0) continue; + const start = trackPoints[0]; + const hasAllEle = trackPoints.every(p => p.ele !== null); + const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]); + waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) }); + } + } + + return waypoints; +} + +/** The GPX source: a trip file in, a trip file out. */ +@Injectable() +export class GpxProvider { + read(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] { + return prepareGpxRows(fileBuffer, opts); + } + + write(input: GpxExportInput, opts: GpxExportOptions = {}): string | null { + return buildGpx(input, opts); + } + + filename(tripTitle: string): string { + return gpxFilename(tripTitle); + } +} diff --git a/server/src/nest/place-import/providers/kml.provider.ts b/server/src/nest/place-import/providers/kml.provider.ts new file mode 100644 index 000000000..d28a984ec --- /dev/null +++ b/server/src/nest/place-import/providers/kml.provider.ts @@ -0,0 +1,26 @@ +/** + * A KML or KMZ map file (Google My Maps, Google Earth), read through kml.codec + * into its placemarks. The file type is the upload's extension, as it has + * always been; anything else is refused with the route's own message. + */ +import { Injectable } from '@nestjs/common'; +import { readKmlDocument, unpackKmzToKml } from '../kml.codec'; +import type { KmlDocumentRead } from '../place-import.types'; + +@Injectable() +export class KmlProvider { + async readMapFile(fileBuffer: Buffer, filename: string): Promise { + const ext = filename.toLowerCase().split('.').pop(); + if (ext === 'kmz') return this.readKmz(fileBuffer); + if (ext === 'kml') return this.readKml(fileBuffer); + throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`); + } + + async readKmz(kmzBuffer: Buffer): Promise { + return this.readKml(await unpackKmzToKml(kmzBuffer)); + } + + readKml(fileBuffer: Buffer): KmlDocumentRead { + return readKmlDocument(fileBuffer); + } +} diff --git a/server/src/nest/place-import/providers/naver-list.provider.ts b/server/src/nest/place-import/providers/naver-list.provider.ts new file mode 100644 index 000000000..805220189 --- /dev/null +++ b/server/src/nest/place-import/providers/naver-list.provider.ts @@ -0,0 +1,137 @@ +/** + * A shared Naver Maps list (a bookmark folder), read page by page from Naver's + * share endpoint. + * + * The link is checked by the SSRF guard and a naver.me short link is followed + * hop by hop through it. Every page is capped before and after it is read: the + * folder id is attacker-influenced, and the pager buffers a fresh body on every + * iteration. Every refusal is the exact message and status the import route + * has always answered. + */ +import { Injectable } from '@nestjs/common'; +import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard'; +import { LIST_USER_AGENT, MAX_LIST_RESPONSE_BYTES } from './google-list.provider'; +import type { ListImportError, ListRead, NaverListPlace } from '../place-import.types'; + +/** Bookmarks asked for per page. */ +const PAGE_SIZE = 20; + +type NaverPage = { + folder?: { bookmarkCount?: number; name?: string }; + bookmarkList?: Record[]; +}; + +@Injectable() +export class NaverListProvider { + async read(url: string): Promise | ListImportError> { + let resolvedUrl = url; + const limit = PAGE_SIZE; + + // SSRF guard: validate user-supplied URL before fetching + const ssrf = await checkSsrf(url); + if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; + + // Resolve naver.me short links to the canonical map.naver.com folder URL. + // Redirects are followed manually so each hop is re-validated against the + // SSRF guard (a short link could otherwise 302 to an internal address). + let parsedUrl: URL; + try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; } + if (parsedUrl.hostname === 'naver.me') { + try { + const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); + resolvedUrl = redirectRes.url; + } catch (err) { + if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; + throw err; + } + } + + const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i); + const folderId = folderMatch?.[1] || null; + if (!folderId) { + return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 }; + } + + const fetchPage = async (start: number) => { + const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`; + const apiRes = await fetch(apiUrl, { + headers: { + Accept: 'application/json', + 'User-Agent': LIST_USER_AGENT, + }, + signal: AbortSignal.timeout(15000), + }); + + if (!apiRes.ok) { + return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; + } + + // Same cap as the Google import: the URL is attacker-influenced via the + // folder id, and this pager buffers a fresh body on every iteration, so an + // uncapped read is worse here than there. The declared length is checked + // before the read; the post-read check covers a chunked response that + // carries no content-length at all. + const declared = Number(apiRes.headers?.get('content-length') ?? 0); + if (declared > MAX_LIST_RESPONSE_BYTES) { + return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; + } + + try { + const rawText = await apiRes.text(); + if (rawText.length > MAX_LIST_RESPONSE_BYTES) { + return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; + } + const data = JSON.parse(rawText) as NaverPage; + return { data } as const; + } catch { + return { error: 'Invalid list data received from Naver Maps', status: 400 } as const; + } + }; + + const firstPage = await fetchPage(0); + if ('error' in firstPage) { + return { error: firstPage.error, status: firstPage.status }; + } + + const listName = firstPage.data.folder?.name || 'Naver Maps List'; + const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number' + ? firstPage.data.folder.bookmarkCount + : (firstPage.data.bookmarkList?.length || 0); + + const allItems: Record[] = [...(firstPage.data.bookmarkList || [])]; + for (let start = limit; start < totalCount; start += limit) { + const page = await fetchPage(start); + if ('error' in page) { + return { error: page.error, status: page.status }; + } + const pageItems = page.data.bookmarkList || []; + if (!Array.isArray(pageItems) || pageItems.length === 0) break; + allItems.push(...pageItems); + } + + if (allItems.length === 0) { + return { error: 'List is empty or could not be read', status: 400 }; + } + + const places: NaverListPlace[] = []; + for (const item of allItems) { + const lat = Number(item?.py); + const lng = Number(item?.px); + const name = typeof item?.name === 'string' && item.name.trim() + ? item.name.trim() + : (typeof item?.displayName === 'string' ? item.displayName.trim() : ''); + const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null; + const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null; + + if (name && Number.isFinite(lat) && Number.isFinite(lng)) { + places.push({ name, lat, lng, notes: note, address }); + } + } + + if (places.length === 0) { + return { error: 'No places with coordinates found in list', status: 400 }; + } + + return { listName, places }; + } +} diff --git a/server/src/nest/places/places.controller.ts b/server/src/nest/places/places.controller.ts index 74d058f1c..a03dd792a 100644 --- a/server/src/nest/places/places.controller.ts +++ b/server/src/nest/places/places.controller.ts @@ -24,7 +24,7 @@ import { memoryStorage } from 'multer'; import { hexColorSchema, placeImageUrlSchema, placeWebsiteSchema } from '@trek/shared'; import type { User } from '../../types'; import { PlacesService } from './places.service'; -import { isDirectionsUrl } from './maps-dir.helpers'; +import { isDirectionsUrl } from '../place-import/place-import.service'; import { isUpdateConflict } from '../common/conflictResult'; import { JwtAuthGuard } from '../auth-core/jwt-auth.guard'; import { CurrentUser } from '../auth-core/current-user.decorator'; diff --git a/server/src/nest/places/places.helpers.ts b/server/src/nest/places/places.helpers.ts index d4d523c58..7b4b43d71 100644 --- a/server/src/nest/places/places.helpers.ts +++ b/server/src/nest/places/places.helpers.ts @@ -1,5 +1,3 @@ -import { XMLParser } from 'fast-xml-parser'; -import unzipper from 'unzipper'; import { externalIdsOf, normalizePlaceName, @@ -8,7 +6,7 @@ import { } from '@trek/shared'; import type { Place } from '../../types'; import type { PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository'; -import type { KmlImportSummary } from './kml-import.helpers'; +import type { KmlImportSummary } from '../place-import/place-import.types'; import type { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service'; import { haversineMetres } from '../common/geo'; @@ -17,36 +15,15 @@ import { haversineMetres } from '../common/geo'; * out of the legacy services/placeService.ts when it went DI-native. Same * split as maps.helpers.ts / transit.helpers.ts / files.constants.ts: nothing * here touches the DB, so it stays plain exports rather than becoming methods - * on PlacesService — which also keeps the frozen-at-import XML parsers a - * single shared instance and lets the KMZ unpacker be unit-tested on its own. + * on PlacesService. Reading files and shared lists (the XML parsers, the KMZ + * unpacker, the list fetchers) lives in place-import/. */ -const gpxParser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - isArray: (name) => ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'].includes(name), -}); - -const kmlParser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '@_', - removeNSPrefix: true, - isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name), - // Treat as raw text so mixed-content HTML (e.g.
, ) - // is returned as a string instead of a parsed object. - stopNodes: ['*.description'], -}); - -export { gpxParser, kmlParser }; - -export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB - // Re-exported so the importers that already read these from here keep working, // while the values themselves live in @trek/shared with the rule that uses them. export { COORD_DEDUP_TOLERANCE, externalIdsOf } from '@trek/shared'; -/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */ -export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB +export type { GpxImportOptions, KmlImportOptions, ListImportError } from '../place-import/place-import.types'; /** * Escape the LIKE metacharacters in a user-supplied search term so `%` and `_` @@ -88,24 +65,6 @@ export interface ListImportResult { skipped: number; } -export interface ListImportError { - error: string; - status: number; -} - -export interface GpxImportOptions { - importWaypoints?: boolean; - importRoutes?: boolean; - importTracks?: boolean; - /** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */ - defaultName?: string; -} - -export interface KmlImportOptions { - importPoints?: boolean; - importPaths?: boolean; -} - // Reclaim a deleted place's cached marker photo if nothing else references it. // The cache key is the Google place_id, or — for coordinate-only places — the // pseudo-id embedded in the stored proxy URL (/api/maps/place-photo/{id}/bytes). @@ -173,41 +132,6 @@ export function trackInsertedInDedupSet(place: PlaceMatchCandidate, dedup: Dedup } } -// --------------------------------------------------------------------------- -// Google Maps list id parsing -// --------------------------------------------------------------------------- - -export function googleMapsHexId(value: unknown): string | null { - if (typeof value !== 'string' && typeof value !== 'number') return null; - const raw = String(value).trim(); - if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase(); - if (!/^-?\d+$/.test(raw)) return null; - try { - const parsed = BigInt(raw); - const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed; - return `0x${unsigned.toString(16)}`; - } catch { - return null; - } -} - -export function googleMapsFeatureIdFromItem(item: unknown): string | null { - if (!Array.isArray(item)) return null; - const candidates = [ - Array.isArray(item[1]) ? item[1][6] : null, - Array.isArray(item[7]) ? item[7][1] : null, - ]; - - for (const ids of candidates) { - if (!Array.isArray(ids) || ids.length < 2) continue; - const first = googleMapsHexId(ids[0]); - const second = googleMapsHexId(ids[1]); - if (first && second) return `${first}:${second}`; - } - - return null; -} - // --------------------------------------------------------------------------- // Import enrichment (#886) — the pure half of the former // services/placeEnrichment.ts; the DB/websocket/Maps half is PlacesService. @@ -283,30 +207,24 @@ export async function mapWithConcurrency(items: T[], limit: number, fn: (item export const trimOrNull = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null); // --------------------------------------------------------------------------- -// KMZ unpacking +// KML folder → trip category // --------------------------------------------------------------------------- -export async function unpackKmzToKml( - kmzBuffer: Buffer, - decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT, -): Promise { - let zip; - try { - zip = await unzipper.Open.buffer(kmzBuffer); - } catch { - throw new Error('Invalid KMZ archive.'); +export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map { + const lookup = new Map(); + for (const category of categories) { + const normalizedName = category.name.trim().toLowerCase(); + if (!normalizedName) continue; + if (!lookup.has(normalizedName)) { + lookup.set(normalizedName, category.id); + } } - - const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml')); - if (kmlEntries.length === 0) { - throw new Error('KMZ archive does not contain a KML file.'); - } - - const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0]; - - if (preferredEntry.uncompressedSize > decompressedSizeLimit) { - throw new Error('KMZ archive exceeds the maximum allowed decompressed size.'); - } - - return preferredEntry.buffer(); + return lookup; +} + +export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map): number | null { + if (!folderName) return null; + const normalizedFolder = folderName.trim().toLowerCase(); + if (!normalizedFolder) return null; + return lookup.get(normalizedFolder) ?? null; } diff --git a/server/src/nest/places/places.mcp.ts b/server/src/nest/places/places.mcp.ts index c75dc2e57..28dcb1eed 100644 --- a/server/src/nest/places/places.mcp.ts +++ b/server/src/nest/places/places.mcp.ts @@ -26,7 +26,7 @@ import { UnitOfWork } from '../database/unit-of-work'; import { MapsService } from '../maps/maps.service'; import { PlacesService } from './places.service'; import { isUpdateConflict } from '../common/conflictResult'; -import { isDirectionsUrl } from './maps-dir.helpers'; +import { isDirectionsUrl } from '../place-import/place-import.service'; import { TripAccessService } from '../trip-membership/trip-access.service'; function parseId(value: string | string[]): number | null { diff --git a/server/src/nest/places/places.module.ts b/server/src/nest/places/places.module.ts index 3d237b557..dde7d4a17 100644 --- a/server/src/nest/places/places.module.ts +++ b/server/src/nest/places/places.module.ts @@ -24,6 +24,7 @@ import { UnsplashModule } from '../unsplash/unsplash.module'; import { PlacePhotosModule } from '../place-photos/place-photos.module'; import { QueryHelpersModule } from '../query-helpers/query-helpers.module'; import { MapsModule } from '../maps/maps.module'; +import { PlaceImportModule } from '../place-import/place-import.module'; import { McpSharedModule } from '../mcp-shared/mcp-shared.module'; import { MulterModule } from '@nestjs/platform-express'; import { StorageModule } from '../storage/storage.module'; @@ -55,6 +56,8 @@ import { TripMembershipModule } from '../trip-membership/trip-membership.module' StorageModule, // AccommodationsDomainModule: deleting a place takes the nights booked at it with // it, and the cascade behind a stay belongs to the domain that owns it. + // PlaceImportModule reads the GPX/KML files and shared lists this domain imports. + PlaceImportModule, McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule, AccommodationsDomainModule, // Plan 3c Task 4: `PlacesService`'s and `PlacesMcp`'s own `@InjectRepository` // constructor params (`PlacesRepository`, `TagsRepository`, diff --git a/server/src/nest/places/places.service.ts b/server/src/nest/places/places.service.ts index e43c5d933..621828738 100644 --- a/server/src/nest/places/places.service.ts +++ b/server/src/nest/places/places.service.ts @@ -2,30 +2,20 @@ import path from 'node:path'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@mikro-orm/nestjs'; import { resolveCountryCodeSync } from '../atlas/atlas-geo'; -import { XMLValidator } from 'fast-xml-parser'; import { TRACK_COLORS, placeMatchStrategies, type PlaceMatchCandidate } from '@trek/shared'; import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared'; import { RealtimeService } from '../realtime/realtime.service'; import type { TripAccess } from '../../db/repositories/Trips.repository'; import { UnitOfWork } from '../database/unit-of-work'; import { PermissionsService } from '../permissions/permissions.service'; -import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service'; -import { isDirectionsUrl, parseDirectionsUrl } from './maps-dir.helpers'; +import { MapsService } from '../maps/maps.service'; +import { PlaceImportService } from '../place-import/place-import.service'; +import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from '../place-import/place-import.service'; +import type { GoogleListPlace, KmlDocumentRead, PreparedGpxPlace } from '../place-import/place-import.types'; import { toRowId } from '../common/row-id'; import type { User } from '../../types'; import { QueryHelpersService } from '../query-helpers/query-helpers.service'; import { ratingAggregate } from '../common/rowShape'; -import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../utils/ssrfGuard'; -import { - buildCategoryNameLookup, - createKmlImportSummary, - decodeUtf8WithWarning, - extractKmlPlacemarkNodes, - parsePlacemarkNode, - resolveCategoryIdForFolder, -} from './kml-import.helpers'; -import { buildGpx, gpxFilename } from './gpx-export.helpers'; -import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from './gpx-export.helpers'; import { UnsplashService } from '../unsplash/unsplash.service'; import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service'; import { type UpdateConflict, isUpdateConflict } from '../common/conflictResult'; @@ -64,20 +54,17 @@ const noCancelledStays = (): CancelledStays => ({ reservationIds: [], budgetItem import { ENRICH_CONCURRENCY, ADDRESS_BACKFILL_MAX_PLACES, + buildCategoryNameLookup, escapeLikePattern, - MAX_LIST_RESPONSE_BYTES, - googleMapsFeatureIdFromItem, - gpxParser, externalIdsOf, isPlaceDuplicate, - kmlParser, mapWithConcurrency, pickEnrichmentMatch, reclaimPhotoCache, SEARCH_BIAS_RADIUS_METERS, + resolveCategoryIdForFolder, trackInsertedInDedupSet, trimOrNull, - unpackKmzToKml, type DedupSet, type EnrichablePlace, type GpxImportOptions, @@ -93,14 +80,7 @@ type Trip = TripAccess; type ImportedPlace = { id: number; route_geometry?: string | null; route_color?: string | null }; -/** One place parsed out of a GPX file, not yet persisted. */ -export interface PreparedGpxPlace { - name: string; - lat: number; - lng: number; - description: string | null; - routeGeometry?: string; -} +export type { PreparedGpxPlace } from '../place-import/place-import.types'; /** Fields accepted when creating a place. */ export interface PlaceCreateInput { @@ -187,6 +167,8 @@ export class PlacesService { @InjectRepository(BudgetItems) private readonly budgetItemsRepo: BudgetItemsRepository, // Plan 3h Task 6 (survivors) — additive, SV-PI2's `reclaimPlaceImage` only. @InjectRepository(CollectionPlaces) private readonly collectionPlacesRepo: CollectionPlacesRepository, + // Reading files and shared lists; this service only persists what it reads. + private readonly placeImport: PlaceImportService, ) {} /** @@ -903,95 +885,18 @@ export class PlacesService { day.points.push({ name: stop.name, lat: stop.lat, lng: stop.lng }); } - const gpx = buildGpx({ tripTitle: title, places, days: [...days.values()] }, opts); - return gpx ? { gpx, filename: gpxFilename(title) } : null; + const gpx = this.placeImport.writeTripGpx({ tripTitle: title, places, days: [...days.values()] }, opts); + return gpx ? { gpx, filename: this.placeImport.tripGpxFilename(title) } : null; } private async importGpxRows(tripId: string, fileBuffer: Buffer, opts: GpxImportOptions = {}): Promise { - const rows = this.prepareGpxRows(fileBuffer, opts); + const rows = this.placeImport.readGpx(fileBuffer, opts); if (!rows.length) return null; return await this.uow.transactional(() => this.persistGpxRows(tripId, rows)); } /** - * Parses a GPX file into the places an import would create, without writing - * anything. The Tours import reuses it to build its own selection before - * handing the rows to {@link importPreparedGpx}. - */ - prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] { - const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts; - - const parsed = gpxParser.parse(fileBuffer.toString('utf-8')); - const gpx = parsed?.gpx; - if (!gpx) return []; - - const str = (v: unknown) => (v != null ? String(v).trim() : null); - const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; }; - - // Routes and tracks rarely carry their own . Without one they all fall back to the - // same generic label, so name-based dedup drops every import after the first. Derive a - // base from the source filename (the requested behaviour) and suffix an index so multiple - // geometries from one file stay distinct. - const rawName = str(defaultName); - const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null; - let geoSeq = 0; - const geoName = (explicit: string | null, fallback: string): string => { - if (explicit) return explicit; - geoSeq++; - const base = baseName || fallback; - return geoSeq === 1 ? base : `${base} ${geoSeq}`; - }; - - const waypoints: PreparedGpxPlace[] = []; - - // 1) Parse elements (named waypoints / POIs) - if (importWaypoints) { - for (const wpt of gpx.wpt ?? []) { - const lat = num(wpt['@_lat']); - const lng = num(wpt['@_lon']); - if (lat === null || lng === null) continue; - waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) }); - } - } - - // 2) Parse routes as polyline-places (one place per route with route_geometry) - if (importRoutes) { - for (const rte of gpx.rte ?? []) { - const pts = (rte.rtept ?? []) - .map((pt: Record) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) })) - .filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>; - if (pts.length === 0) continue; - const hasAllEle = pts.every(p => p.ele !== null); - const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]); - waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) }); - } - } - - // 3) Extract full track geometry from - if (importTracks) { - for (const trk of gpx.trk ?? []) { - const trackPoints: { lat: number; lng: number; ele: number | null }[] = []; - for (const seg of trk.trkseg ?? []) { - for (const pt of seg.trkpt ?? []) { - const lat = num(pt['@_lat']); - const lng = num(pt['@_lon']); - if (lat === null || lng === null) continue; - trackPoints.push({ lat, lng, ele: num(pt.ele) }); - } - } - if (trackPoints.length === 0) continue; - const start = trackPoints[0]; - const hasAllEle = trackPoints.every(p => p.ele !== null); - const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]); - waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) }); - } - } - - return waypoints; - } - - /** - * Persists rows from {@link prepareGpxRows} and colours their tracks, both in + * Persists rows PlaceImportService.readGpx prepared and colours their tracks, both in * one transaction. Called from inside the Tours import's own transaction, * where this becomes a savepoint, so the places and their tour facets * commit or roll back together. @@ -1080,39 +985,21 @@ export class PlacesService { } private async importMapFileRows(tripId: string, fileBuffer: Buffer, filename: string, opts: KmlImportOptions = {}): Promise { - const ext = filename.toLowerCase().split('.').pop(); - if (ext === 'kmz') return this.importKmzPlaces(tripId, fileBuffer, opts); - if (ext === 'kml') return this.importKmlPlaces(tripId, fileBuffer, opts); - throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`); + return this.persistKmlPlaces(tripId, await this.placeImport.readMapFile(fileBuffer, filename), opts); } async importKmzPlaces(tripId: string, kmzBuffer: Buffer, opts: KmlImportOptions = {}): Promise { - const kmlBuffer = await unpackKmzToKml(kmzBuffer); - return this.importKmlPlaces(tripId, kmlBuffer, opts); + return this.persistKmlPlaces(tripId, await this.placeImport.readKmz(kmzBuffer), opts); } async importKmlPlaces(tripId: string, fileBuffer: Buffer, opts: KmlImportOptions = {}): Promise { + return this.persistKmlPlaces(tripId, this.placeImport.readKml(fileBuffer), opts); + } + + /** The placemarks of a read KML document as places, skipping duplicates. */ + private async persistKmlPlaces(tripId: string, read: KmlDocumentRead, opts: KmlImportOptions): Promise { const { importPoints = true, importPaths = true } = opts; - const decoded = decodeUtf8WithWarning(fileBuffer); - - const validationResult = XMLValidator.validate(decoded.text); - if (validationResult !== true) { - throw new Error('Malformed KML: invalid XML structure'); - } - - const parsed = kmlParser.parse(decoded.text); - const kmlRoot = parsed?.kml ?? parsed; - - if (!kmlRoot || typeof kmlRoot !== 'object') { - throw new Error('Malformed KML: could not parse XML'); - } - - const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot); - const summary = createKmlImportSummary(placemarkNodes.length); - - if (decoded.warning) { - summary.warnings.push(decoded.warning); - } + const { placemarks, summary } = read; // PL33 — `CategoriesRepository.listIdName` (Plan 3a's repository). const categories = await this.categoriesRepo.listIdName(); @@ -1126,8 +1013,7 @@ export class PlacesService { await this.uow.transactional(async () => { let fallbackIndex = 1; - for (const node of placemarkNodes) { - const parsedPlacemark = parsePlacemarkNode(node); + for (const parsedPlacemark of placemarks) { const isPath = parsedPlacemark.routeGeometry !== null; // Unsupported geometry type (polygon, multi-geometry, no geometry, etc.) @@ -1259,124 +1145,12 @@ export class PlacesService { // ------------------------------------------------------------------------- async importGoogleList(tripId: string, url: string, opts?: ListImportOptions): Promise { - let listId: string | null = null; - let resolvedUrl = url; - - // SSRF guard: validate user-supplied URL before fetching - const ssrf = await checkSsrf(url); - if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; - - // Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are - // followed manually so every hop is re-checked against the SSRF guard — a - // short link that 302s to an internal IP is blocked even though the initial - // host is public. - if (url.includes('goo.gl') || url.includes('maps.app')) { - try { - const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); - resolvedUrl = redirectRes.url; - } catch (err) { - if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; - throw err; - } - } - - // A route, once the redirect is followed. The dispatch upstream decides on - // the raw URL, and a short link's path is `/` — it matches nothing, so - // every route shared from the Google Maps app arrived here and was answered - // with "could not extract list ID", which is the complaint the directions - // import was written to remove. The Share sheet on a phone produces exactly - // this shape, and the box says a directions link works. - // - // Handed on with the resolved URL, so the hop is not made twice. - if (isDirectionsUrl(resolvedUrl)) { - return this.importGoogleDirections(tripId, resolvedUrl, opts); - } - - // Pattern: /placelists/list/{ID} - const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/); - if (plMatch) listId = plMatch[1]; - - // Pattern: !2s{ID} in data URL params - if (!listId) { - const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/); - if (dataMatch) listId = dataMatch[1]; - } - - if (!listId) { - // A single-place share link (…/maps/place/…) carries no list id — point the user at - // the place search box instead of a cryptic "could not extract list ID" (#1304). - if (resolvedUrl.includes('/maps/place/')) { - return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 }; - } - return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 }; - } - - // Fetch list data from Google Maps internal API - const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`; - const apiRes = await fetch(apiUrl, { - headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' }, - signal: AbortSignal.timeout(15000), - }); - - if (!apiRes.ok) { - return { error: 'Failed to fetch list from Google Maps', status: 502 }; - } - - // Cap the declared body before reading it (transit.service precedent): the - // response is attacker-influenced via the list id, and buffering it whole - // used to be unbounded. - const declared = Number(apiRes.headers?.get('content-length') ?? 0); - if (declared > MAX_LIST_RESPONSE_BYTES) { - return { error: 'Failed to fetch list from Google Maps', status: 502 }; - } - - const rawText = await apiRes.text(); - if (rawText.length > MAX_LIST_RESPONSE_BYTES) { - return { error: 'Failed to fetch list from Google Maps', status: 502 }; - } - const jsonStr = rawText.substring(rawText.indexOf('\n') + 1); - // The provider hands back a JS-prefixed array; a malformed body is a - // provider problem, not a crash — surface the same 400 an unreadable - // payload already produced instead of throwing a SyntaxError. - let listData: unknown; - try { - listData = JSON.parse(jsonStr); - } catch { - return { error: 'Invalid list data received from Google Maps', status: 400 }; - } - if (!Array.isArray(listData)) { - return { error: 'Invalid list data received from Google Maps', status: 400 }; - } - - const meta = listData[0]; - if (!meta) { - return { error: 'Invalid list data received from Google Maps', status: 400 }; - } - - const listName = meta[4] || 'Google Maps List'; - const items = meta[8]; - - if (!Array.isArray(items) || items.length === 0) { - return { error: 'List is empty or could not be read', status: 400 }; - } - - // Parse place data from items - const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = []; - for (const item of items) { - const coords = item?.[1]?.[5]; - const lat = coords?.[2]; - const lng = coords?.[3]; - const name = item?.[2]; - const note = item?.[3] || null; - - if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) { - places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) }); - } - } - - if (places.length === 0) { - return { error: 'No places with coordinates found in list', status: 400 }; - } + const read = await this.placeImport.readGoogleList(url); + if ('error' in read) return read; + // A short link that lands on a route is a directions link: imported as one, + // with the resolved URL, so the hop is not made twice. + if ('directions' in read) return this.importGoogleDirections(tripId, read.directions, opts); + const { listName, places } = read; const { created, skipped } = await this.storeGooglePlaces(tripId, places); @@ -1396,7 +1170,7 @@ export class PlacesService { */ private async storeGooglePlaces( tripId: string, - places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[], + places: GoogleListPlace[], ): Promise<{ created: PlaceWithTags[]; skipped: number }> { // Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the // caller's `requireTrip` gate already `toRowId`-parsed this `tripId`. @@ -1469,90 +1243,13 @@ export class PlacesService { // ------------------------------------------------------------------------- /** - * The stops of a route somebody else planned. - * - * The other half of the request the list import answers: people share a drive far more - * often than they share a list, and until now a pasted `/maps/dir/` link came back as a - * cryptic "could not extract list ID". No API key is involved and no call is made to - * Google for the link itself — the stops are in the URL, which is the whole reason - * this is possible at all. - * - * A stop the link spells out in coordinates is taken as it stands; one that is only a - * name is geocoded, one request each. A name nobody can place is left out rather than - * failing the import, because a route of six stops with five findable is five stops - * more than the traveller had. + * The stops of a route somebody else planned, read and placed by + * PlaceImportService (no API key, the stops are in the link), stored here. */ async importGoogleDirections(tripId: string, url: string, opts?: ListImportOptions): Promise { - const ssrf = await checkSsrf(url); - if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; - - let parsed: URL; - try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; } - - // Short links are resolved hop by hop through the guard, exactly as the list import - // does it: a maps.app.goo.gl that 302s to an internal address is still blocked. - let resolvedUrl = url; - if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) { - try { - const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); - resolvedUrl = redirectRes.url; - } catch (err) { - if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; - throw err; - } - } - - // Checked after resolving, not before: the host that counts is the one the link lands - // on, and `/maps/dir/` is a path anybody could serve. - let host = ''; - try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ } - if (!isGoogleMapsHost(host)) { - return { error: 'That link is not a Google Maps link.', status: 400 }; - } - - const waypoints = parseDirectionsUrl(resolvedUrl); - if (waypoints.length < 2) { - return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 }; - } - - const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = []; - let unplaceable = 0; - for (const wp of waypoints) { - if (wp.lat !== null && wp.lng !== null) { - // A stop written as coordinates has no name of its own, and the coordinates are a - // better label than "Stop 3": they are what the traveller can look up. - places.push({ - name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`, - lat: wp.lat, - lng: wp.lng, - notes: null, - googleFtid: null, - }); - continue; - } - if (!wp.name) continue; - try { - // Through geocodeQuery, which asks the TREK index first and only falls - // through to Nominatim for what it does not know — and does so on the - // BACKGROUND lane. That matters here more than anywhere: this loop runs - // up to thirty times in one request, each Nominatim call taking the next - // slot on a 1.1 s process-wide throttle, so on the interactive lane one - // pasted link made everybody else's place search queue behind it for - // half a minute. An index hit costs no slot at all. - const hit = await this.maps.geocodeQuery(wp.name); - // The name from the link, not the one the geocoder answers with: somebody who - // typed a nickname into Google should not find a street address on their trip. - if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null }); - else unplaceable++; - } catch { - // A geocoder that is down or rate-limited costs this one stop, not the import. - unplaceable++; - } - } - - if (places.length < 2) { - return { error: 'None of the stops in that link could be placed on the map.', status: 400 }; - } + const read = await this.placeImport.readGoogleDirections(url, (query) => this.maps.geocodeQuery(query)); + if ('error' in read) return read; + const { places, unplaceable } = read; const { created, skipped } = await this.storeGooglePlaces(tripId, places); if (created.length) { @@ -1571,116 +1268,9 @@ export class PlacesService { // ------------------------------------------------------------------------- async importNaverList(tripId: string, url: string, opts?: ListImportOptions): Promise { - let resolvedUrl = url; - const limit = 20; - - // SSRF guard: validate user-supplied URL before fetching - const ssrf = await checkSsrf(url); - if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 }; - - // Resolve naver.me short links to the canonical map.naver.com folder URL. - // Redirects are followed manually so each hop is re-validated against the - // SSRF guard (a short link could otherwise 302 to an internal address). - let parsedUrl: URL; - try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; } - if (parsedUrl.hostname === 'naver.me') { - try { - const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) }); - resolvedUrl = redirectRes.url; - } catch (err) { - if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 }; - throw err; - } - } - - const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i); - const folderId = folderMatch?.[1] || null; - if (!folderId) { - return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 }; - } - - const fetchPage = async (start: number) => { - const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`; - const apiRes = await fetch(apiUrl, { - headers: { - Accept: 'application/json', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - }, - signal: AbortSignal.timeout(15000), - }); - - if (!apiRes.ok) { - return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; - } - - // Same cap as the Google import: the URL is attacker-influenced via the - // folder id, and this pager buffers a fresh body on every iteration, so an - // uncapped read is worse here than there. The declared length is checked - // before the read; the post-read check covers a chunked response that - // carries no content-length at all. - const declared = Number(apiRes.headers?.get('content-length') ?? 0); - if (declared > MAX_LIST_RESPONSE_BYTES) { - return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; - } - - try { - const rawText = await apiRes.text(); - if (rawText.length > MAX_LIST_RESPONSE_BYTES) { - return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const; - } - const data = JSON.parse(rawText) as { - folder?: { bookmarkCount?: number; name?: string }; - bookmarkList?: Record[]; - }; - return { data } as const; - } catch { - return { error: 'Invalid list data received from Naver Maps', status: 400 } as const; - } - }; - - const firstPage = await fetchPage(0); - if ('error' in firstPage) { - return { error: firstPage.error, status: firstPage.status }; - } - - const listName = firstPage.data.folder?.name || 'Naver Maps List'; - const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number' - ? firstPage.data.folder.bookmarkCount - : (firstPage.data.bookmarkList?.length || 0); - - const allItems: Record[] = [...(firstPage.data.bookmarkList || [])]; - for (let start = limit; start < totalCount; start += limit) { - const page = await fetchPage(start); - if ('error' in page) { - return { error: page.error, status: page.status }; - } - const pageItems = page.data.bookmarkList || []; - if (!Array.isArray(pageItems) || pageItems.length === 0) break; - allItems.push(...pageItems); - } - - if (allItems.length === 0) { - return { error: 'List is empty or could not be read', status: 400 }; - } - - const places: { name: string; lat: number; lng: number; notes: string | null; address: string | null }[] = []; - for (const item of allItems) { - const lat = Number(item?.py); - const lng = Number(item?.px); - const name = typeof item?.name === 'string' && item.name.trim() - ? item.name.trim() - : (typeof item?.displayName === 'string' ? item.displayName.trim() : ''); - const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null; - const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null; - - if (name && Number.isFinite(lat) && Number.isFinite(lng)) { - places.push({ name, lat, lng, notes: note, address }); - } - } - - if (places.length === 0) { - return { error: 'No places with coordinates found in list', status: 400 }; - } + const read = await this.placeImport.readNaverList(url); + if ('error' in read) return read; + const { listName, places } = read; // Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the // caller's `requireTrip` gate already `toRowId`-parsed this `tripId`. diff --git a/server/src/nest/roadtrip/google-route.service.ts b/server/src/nest/roadtrip/google-route.service.ts index 463c80966..ff17cf6e5 100644 --- a/server/src/nest/roadtrip/google-route.service.ts +++ b/server/src/nest/roadtrip/google-route.service.ts @@ -2,7 +2,7 @@ import { HttpException, Injectable } from '@nestjs/common'; import { InjectRepository } from '@mikro-orm/nestjs'; import type { GoogleRouteImport, GoogleRoutePreview } from '@trek/shared'; import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service'; -import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../places/maps-dir.helpers'; +import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../place-import/place-import.service'; import { safeFetchFollow } from '../../utils/ssrfGuard'; import { UnitOfWork } from '../database/unit-of-work'; import { PlacesService } from '../places/places.service'; diff --git a/server/src/nest/tours/tours.module.ts b/server/src/nest/tours/tours.module.ts index 3fa1f4ebd..20dec0bf4 100644 --- a/server/src/nest/tours/tours.module.ts +++ b/server/src/nest/tours/tours.module.ts @@ -8,12 +8,13 @@ import { ToursController } from './tours.controller'; import { ToursImportController } from './tours-import.controller'; import { ToursService } from './tours.service'; import { PlacesModule } from '../places/places.module'; +import { PlaceImportModule } from '../place-import/place-import.module'; import { PermissionsModule } from '../permissions/permissions.module'; import { AddonsModule } from '../addons/addons.module'; /** - * Tours domain: an isolated bounded context that imports PlacesModule to reuse - * GPX preparation and persistence through PlacesService. The tour SQL lives in + * Tours domain: an isolated bounded context that reads GPX through PlaceImportModule and + * persists the places through PlacesService (PlacesModule). The tour SQL lives in * the Tours and TourWaypoints repositories; the owning place's route columns go * through PlacesRepository, registered here because forFeature only reaches * this module's own providers. @@ -21,6 +22,7 @@ import { AddonsModule } from '../addons/addons.module'; @Module({ imports: [ PlacesModule, + PlaceImportModule, PermissionsModule, AddonsModule, diff --git a/server/src/nest/tours/tours.service.ts b/server/src/nest/tours/tours.service.ts index 5dcc10309..4933df54e 100644 --- a/server/src/nest/tours/tours.service.ts +++ b/server/src/nest/tours/tours.service.ts @@ -12,6 +12,7 @@ import type { TourWaypointsRepository } from '../../db/repositories/TourWaypoint import { toRowId } from '../common/row-id'; import { UnitOfWork } from '../database/unit-of-work'; import { PlacesService } from '../places/places.service'; +import { PlaceImportService } from '../place-import/place-import.service'; import { computeTourMetrics, parseRouteGeometry, LOW_CONFIDENCE_THRESHOLD, type GeometryPoint } from './tours.helpers'; export interface ImportGpxAsTourResult { @@ -23,8 +24,8 @@ export interface ImportGpxAsTourResult { /** * Tours domain: the `tours` facet table (place_id PK/FK) carries tour-specific * metadata while existing places and day assignments are reused. - * GPX parsing and persistence are reused through PlacesService.prepareGpxRows - * and importPreparedGpx. Places and facet rows are persisted in one transaction, + * GPX parsing goes through PlaceImportService.readGpx, persistence through + * PlacesService.importPreparedGpx. Places and facet rows are persisted in one transaction, * with tour metrics derived from each place's route_geometry. */ @Injectable() @@ -38,6 +39,7 @@ export class ToursService { @InjectRepository(TourTypes) private readonly tourTypesRepo: TourTypesRepository, @InjectRepository(TourWaypoints) private readonly waypointsRepo: TourWaypointsRepository, @InjectRepository(Places) private readonly placesRepo: PlacesRepository, + private readonly placeImport: PlaceImportService, ) {} private toItem(r: TourListRow): TourListItem { @@ -196,7 +198,7 @@ export class ToursService { } /** - * The tours-mode GPX import prepares rows through PlacesService.prepareGpxRows + * The tours-mode GPX import reads rows through PlaceImportService.readGpx * with waypoints excluded. PlacesService.importPreparedGpx persists the places * in the same transaction as their `tours` facet rows, whose metrics are * derived from route_geometry. @@ -206,7 +208,7 @@ export class ToursService { * match_confidence so the client can surface a "with caution" toast. */ async importGpxAsTour(tripId: string, fileBuffer: Buffer, defaultName?: string, socketId?: string): Promise { - const rows = this.places.prepareGpxRows(fileBuffer, { + const rows = this.placeImport.readGpx(fileBuffer, { importWaypoints: false, importRoutes: true, importTracks: true, defaultName, }); if (rows.length === 0) return null; diff --git a/server/tests/helpers/mcp-test-controllers.ts b/server/tests/helpers/mcp-test-controllers.ts index b7df1233c..3edc77fa3 100644 --- a/server/tests/helpers/mcp-test-controllers.ts +++ b/server/tests/helpers/mcp-test-controllers.ts @@ -36,6 +36,7 @@ import { PackingService } from '../../src/nest/packing/packing.service'; import { PermissionsService } from '../../src/nest/permissions/permissions.service'; import { PlacesMcp } from '../../src/nest/places/places.mcp'; import { PlacesService } from '../../src/nest/places/places.service'; +import { buildPlaceImportService } from './place-import'; import { ReservationsMcp } from '../../src/nest/reservations/reservations.mcp'; import { ReservationsService } from '../../src/nest/reservations/reservations.service'; import { ReservationsReadService } from '../../src/nest/reservations/reservations-read.service'; @@ -304,6 +305,7 @@ export async function createMcpTestRegistry(): Promise { await createTestTripsRepo(db), await createTestBudgetItemsRepo(db), await createTestCollectionPlacesRepo(db), + buildPlaceImportService(), ); // Built after it: a hotel booking writes the stay's day stop through this one. const reservationsService = new ReservationsService(permissionsService, budgetService, realtimeService, notificationsStub(), new ReservationsReadService(await createTestReservationsRepo(db), await createTestReservationEndpointsRepo(db), await createTestReservationTravelersRepo(db)), accommodationsService, await createTestUnitOfWork(db), await createTestReservationsRepo(db), await createTestReservationEndpointsRepo(db), await createTestReservationTravelersRepo(db), await createTestReservationDayPositionsRepo(db), await createTestDayAccommodationsRepo(db), await createTestDaysRepo(db), await createTestPlacesRepo(db), await createTestDayAssignmentsRepo(db), await createTestTripMembersRepo(db), await createTestUsersRepo(db), await createTestTripsRepo(db), await createTestBudgetItemsRepo(db)); @@ -455,6 +457,7 @@ export async function createMcpTestRegistry(): Promise { await createTestTripsRepo(db), await createTestTripMembersRepo(db), await createTestPlacesRepo(db), await createTestPlaceRatingsRepo(db), await createTestTagsRepo(db), usersRepo, + buildPlaceImportService(), ), // Plan 3h Task 1 — `CollectionsMcp`'s own constructor-ripple fix: // `DatabaseService` dropped (CL89's only use), `UsersRepository` diff --git a/server/tests/helpers/place-import.ts b/server/tests/helpers/place-import.ts new file mode 100644 index 000000000..9dd753ee0 --- /dev/null +++ b/server/tests/helpers/place-import.ts @@ -0,0 +1,22 @@ +import { PlaceImportService } from '../../src/nest/place-import/place-import.service'; +import { GpxProvider } from '../../src/nest/place-import/providers/gpx.provider'; +import { CollectionGpxProvider } from '../../src/nest/place-import/providers/collection-gpx.provider'; +import { KmlProvider } from '../../src/nest/place-import/providers/kml.provider'; +import { GoogleListProvider } from '../../src/nest/place-import/providers/google-list.provider'; +import { GoogleDirectionsProvider } from '../../src/nest/place-import/providers/google-directions.provider'; +import { NaverListProvider } from '../../src/nest/place-import/providers/naver-list.provider'; + +/** + * PlaceImportService built by hand the way PlaceImportModule wires it: real + * providers over the global fetch a test stubs. + */ +export function buildPlaceImportService(): PlaceImportService { + return new PlaceImportService( + new GpxProvider(), + new CollectionGpxProvider(), + new KmlProvider(), + new GoogleListProvider(), + new GoogleDirectionsProvider(), + new NaverListProvider(), + ); +} diff --git a/server/tests/helpers/plugin-host.ts b/server/tests/helpers/plugin-host.ts index aac7940b1..71972f431 100644 --- a/server/tests/helpers/plugin-host.ts +++ b/server/tests/helpers/plugin-host.ts @@ -31,6 +31,7 @@ import { RateLimitService } from '../../src/nest/common/rate-limit.service'; import { VacayService } from '../../src/nest/vacay/vacay.service'; import { TripsService } from '../../src/nest/trips/trips.service'; import { PlacesService } from '../../src/nest/places/places.service'; +import { buildPlaceImportService } from './place-import'; import { CollectionsService } from '../../src/nest/collections/collections.service'; import { AtlasService } from '../../src/nest/atlas/atlas.service'; import { buildMapsService } from './maps-service'; @@ -260,6 +261,7 @@ export async function createPluginRpcHostParts( await createTestTripsRepo(db), await createTestTripMembersRepo(db), await createTestPlacesRepo(db), await createTestPlaceRatingsRepo(db), await createTestTagsRepo(db), usersRepo, + buildPlaceImportService(), ); const atlas = new AtlasService( await createTestBucketListRepo(db), await createTestHiddenCountriesRepo(db), @@ -309,6 +311,7 @@ export async function createPluginRpcHostParts( await createTestTripsRepo(db), await createTestBudgetItemsRepo(db), await createTestCollectionPlacesRepo(db), + buildPlaceImportService(), ); // After accommodations: a hotel booking writes the stay's day stop through it. const reservations = new ReservationsService(permissions, budget, realtime, notificationsStub(), new ReservationsReadService(await createTestReservationsRepo(db), await createTestReservationEndpointsRepo(db), await createTestReservationTravelersRepo(db)), accommodations, await createTestUnitOfWork(db), await createTestReservationsRepo(db), await createTestReservationEndpointsRepo(db), await createTestReservationTravelersRepo(db), await createTestReservationDayPositionsRepo(db), await createTestDayAccommodationsRepo(db), await createTestDaysRepo(db), await createTestPlacesRepo(db), await createTestDayAssignmentsRepo(db), await createTestTripMembersRepo(db), await createTestUsersRepo(db), await createTestTripsRepo(db), await createTestBudgetItemsRepo(db)); diff --git a/server/tests/unit/nest/collections.controller.test.ts b/server/tests/unit/nest/collections.controller.test.ts index c456037c8..ab3a9ff12 100644 --- a/server/tests/unit/nest/collections.controller.test.ts +++ b/server/tests/unit/nest/collections.controller.test.ts @@ -4,7 +4,7 @@ import { HttpException } from '@nestjs/common'; import { CollectionsController } from '../../../src/nest/collections/collections.controller'; import type { CollectionsService } from '../../../src/nest/collections/collections.service'; import type { StorageService } from '../../../src/nest/storage/storage.service'; -import { CollectionGpxError } from '../../../src/nest/collections/collection-gpx.helpers'; +import { CollectionGpxError } from '../../../src/nest/place-import/place-import.types'; import type { User } from '../../../src/types'; import type { CollectionGpxProblem } from '@trek/shared'; diff --git a/server/tests/unit/nest/collections.service.test.ts b/server/tests/unit/nest/collections.service.test.ts index a5b6aa478..641008513 100644 --- a/server/tests/unit/nest/collections.service.test.ts +++ b/server/tests/unit/nest/collections.service.test.ts @@ -39,6 +39,7 @@ import { createUser, createTrip, createPlace, createCategory, createTag, addTrip import { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { CollectionsService } from '../../../src/nest/collections/collections.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { PlacePhotoCacheService } from '../../../src/nest/place-photos/place-photo-cache.service'; import { makeStorageFixture } from '../../helpers/storage-fixture'; import { notificationsStub } from '../../helpers/notifications'; @@ -131,6 +132,7 @@ beforeAll(async () => { tripsRepoForSpy, await createTestTripMembersRepo(testDb), await createTestPlacesRepo(testDb), await createTestPlaceRatingsRepo(testDb), await createTestTagsRepo(testDb), await createTestUsersRepo(testDb), + buildPlaceImportService(), ); photoCache = new PlacePhotoCacheService( makeStorageFixture('photos/google/').storage, diff --git a/server/tests/unit/nest/collection-gpx.helpers.test.ts b/server/tests/unit/nest/place-import/collection-gpx.provider.test.ts similarity index 99% rename from server/tests/unit/nest/collection-gpx.helpers.test.ts rename to server/tests/unit/nest/place-import/collection-gpx.provider.test.ts index 8fd4f6b8c..6b98a3ebc 100644 --- a/server/tests/unit/nest/collection-gpx.helpers.test.ts +++ b/server/tests/unit/nest/place-import/collection-gpx.provider.test.ts @@ -17,9 +17,9 @@ import { collectionFileToGpx, gpxToCollectionFile, type ExportedCollectionFile, -} from '../../../src/nest/collections/collection-gpx.helpers'; +} from '../../../../src/nest/place-import/providers/collection-gpx.provider'; -const fixture = (name: string) => fs.readFileSync(path.join(__dirname, '../../fixtures/gpx', name), 'utf8'); +const fixture = (name: string) => fs.readFileSync(path.join(__dirname, '../../../fixtures/gpx', name), 'utf8'); const listFile = (over: Partial = {}): ExportedCollectionFile => ({ format: 'trek.collection', version: 1, name: 'Lisbon', places: [], ...over, diff --git a/server/tests/unit/nest/maps-dir.helpers.test.ts b/server/tests/unit/nest/place-import/directions-url.helpers.test.ts similarity index 98% rename from server/tests/unit/nest/maps-dir.helpers.test.ts rename to server/tests/unit/nest/place-import/directions-url.helpers.test.ts index b90159ed0..54bdda1b3 100644 --- a/server/tests/unit/nest/maps-dir.helpers.test.ts +++ b/server/tests/unit/nest/place-import/directions-url.helpers.test.ts @@ -7,7 +7,7 @@ * that matter, because the full ones run to four hundred characters of protobuf. */ import { describe, it, expect } from 'vitest'; -import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../../../src/nest/places/maps-dir.helpers'; +import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../../../../src/nest/place-import/directions-url.helpers'; describe('isDirectionsUrl', () => { it('MAPS-DIR-001: tells a route apart from a list and from a single place', () => { diff --git a/server/tests/unit/nest/place-import/google-list.provider.test.ts b/server/tests/unit/nest/place-import/google-list.provider.test.ts new file mode 100644 index 000000000..6ee21ce7b --- /dev/null +++ b/server/tests/unit/nest/place-import/google-list.provider.test.ts @@ -0,0 +1,142 @@ +/** + * The Google list source on its own: the feature-id parsing (moved from + * places.helpers.test.ts with its code) and the answers the import route + * gets back from it. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +const { mockCheckSsrf, mockFollow } = vi.hoisted(() => ({ + mockCheckSsrf: vi.fn(async (_url: string) => ({ allowed: true })), + mockFollow: vi.fn(), +})); +vi.mock('../../../../src/utils/ssrfGuard', () => ({ + checkSsrf: mockCheckSsrf, + safeFetchFollow: mockFollow, + SsrfBlockedError: class SsrfBlockedError extends Error {}, +})); + +import { + GoogleListProvider, + googleMapsFeatureIdFromItem, + googleMapsHexId, +} from '../../../../src/nest/place-import/providers/google-list.provider'; +import { SsrfBlockedError } from '../../../../src/utils/ssrfGuard'; + +afterEach(() => { + vi.unstubAllGlobals(); + mockCheckSsrf.mockReset(); + mockCheckSsrf.mockResolvedValue({ allowed: true }); + mockFollow.mockReset(); +}); + +// ── Google Maps feature ids ─────────────────────────────────────────────────── + +describe('googleMapsHexId / googleMapsFeatureIdFromItem', () => { + it('passes an already-hex id through lower-cased', () => { + expect(googleMapsHexId('0x882BF179E806D471')).toBe('0x882bf179e806d471'); + }); + + it("converts Google's signed 64-bit decimals to unsigned hex", () => { + expect(googleMapsHexId('-8634542354666695567')).toBe('0x882bf179e806d471'); + expect(googleMapsHexId(255)).toBe('0xff'); + }); + + it('rejects anything that is not a hex or decimal id', () => { + expect(googleMapsHexId('not-an-id')).toBeNull(); + expect(googleMapsHexId(null)).toBeNull(); + expect(googleMapsHexId({})).toBeNull(); + }); + + it('reads the ftid pair from either item slot, else null', () => { + expect(googleMapsFeatureIdFromItem([null, [0, 0, 0, 0, 0, 0, ['0x1', '0x2']]])).toBe('0x1:0x2'); + expect(googleMapsFeatureIdFromItem([null, null, null, null, null, null, null, [null, ['0x3', '0x4']]])).toBe('0x3:0x4'); + expect(googleMapsFeatureIdFromItem([null, [0, 0, 0, 0, 0, 0, ['0x1']]])).toBeNull(); + expect(googleMapsFeatureIdFromItem('not an array')).toBeNull(); + }); +}); + +describe('GoogleListProvider.read', () => { + it('refuses a URL the SSRF guard blocks, before fetching anything', async () => { + mockCheckSsrf.mockResolvedValueOnce({ allowed: false }); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + expect(await new GoogleListProvider().read('https://10.0.0.1/maps')).toEqual({ error: 'URL is not allowed', status: 400 }); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('hands a short link that lands on a route back as a directions link', async () => { + mockFollow.mockResolvedValueOnce({ url: 'https://www.google.com/maps/dir/Berlin/Dresden/' }); + expect(await new GoogleListProvider().read('https://maps.app.goo.gl/abc')).toEqual({ + directions: 'https://www.google.com/maps/dir/Berlin/Dresden/', + }); + }); + + it('points a single-place link at the search box', async () => { + const out = await new GoogleListProvider().read('https://www.google.com/maps/place/Berlin'); + expect(out).toMatchObject({ status: 400 }); + expect((out as { error: string }).error).toContain('single place, not a list'); + }); + + it('reads the list name and the places that carry coordinates', async () => { + const items = [ + [null, [null, null, null, null, null, [null, null, 52.5, 13.4], ['0x1', '0x2']], 'Cafe', 'note'], + [null, [null, null, null, null, null, [null, null, null, null]], 'No coords'], + ]; + const body = `)]}'\n${JSON.stringify([[null, null, null, null, 'Trip ideas', null, null, null, items]])}`; + vi.stubGlobal('fetch', vi.fn(async () => new Response(body, { status: 200 }))); + expect(await new GoogleListProvider().read('https://www.google.com/maps/placelists/list/ABCDEFGHIJKLMNOP')).toEqual({ + listName: 'Trip ideas', + places: [{ name: 'Cafe', lat: 52.5, lng: 13.4, notes: 'note', googleFtid: '0x1:0x2' }], + }); + }); +}); + +describe('GoogleListProvider.read: what Google sends back', () => { + const LIST = 'https://www.google.com/maps/@52,13,10z/data=!4m3!11m2!2sABCDEFGHIJKLMNOPQRS!3e3'; + const respond = (body: string, init: ResponseInit = { status: 200 }) => vi.stubGlobal('fetch', vi.fn(async () => new Response(body, init))); + const read = () => new GoogleListProvider().read(LIST); + + it('finds the list id in the data parameter too', async () => { + const fetchMock = vi.fn(async () => new Response('x', { status: 500 })); + vi.stubGlobal('fetch', fetchMock); + await read(); + expect(String((fetchMock.mock.calls[0] as unknown[])[0])).toContain('!1sABCDEFGHIJKLMNOPQRS'); + }); + + it('answers a failed or oversized fetch with the provider message', async () => { + respond('x', { status: 500 }); + expect(await read()).toEqual({ error: 'Failed to fetch list from Google Maps', status: 502 }); + respond('x', { status: 200, headers: { 'content-length': String(9 * 1024 * 1024) } }); + expect(await read()).toEqual({ error: 'Failed to fetch list from Google Maps', status: 502 }); + }); + + it('answers an unreadable payload with the invalid-data message', async () => { + respond('prefix\nnot json'); + expect(await read()).toEqual({ error: 'Invalid list data received from Google Maps', status: 400 }); + respond('prefix\n{"a":1}'); + expect(await read()).toEqual({ error: 'Invalid list data received from Google Maps', status: 400 }); + respond('prefix\n[]'); + expect(await read()).toEqual({ error: 'Invalid list data received from Google Maps', status: 400 }); + }); + + it('refuses an empty list and one without a single coordinate', async () => { + respond('prefix\n[[null,null,null,null,"L",null,null,null,[]]]'); + expect(await read()).toEqual({ error: 'List is empty or could not be read', status: 400 }); + respond('prefix\n[[null,null,null,null,null,null,null,null,[[null,[],"Nowhere"]]]]'); + expect(await read()).toEqual({ error: 'No places with coordinates found in list', status: 400 }); + }); + + it('refuses a short link the guard blocks on a hop, and lets a network failure through', async () => { + mockFollow.mockRejectedValueOnce(new SsrfBlockedError('blocked')); + expect(await new GoogleListProvider().read('https://maps.app.goo.gl/x')).toEqual({ error: 'URL is not allowed', status: 400 }); + mockFollow.mockRejectedValueOnce(new Error('network')); + await expect(new GoogleListProvider().read('https://maps.app.goo.gl/x')).rejects.toThrow('network'); + }); + + it('needs a list id', async () => { + expect(await new GoogleListProvider().read('https://www.google.com/maps/@52,13,10z')).toEqual({ + error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', + status: 400, + }); + }); +}); diff --git a/server/tests/unit/nest/gpx-export.helpers.test.ts b/server/tests/unit/nest/place-import/gpx.provider.test.ts similarity index 99% rename from server/tests/unit/nest/gpx-export.helpers.test.ts rename to server/tests/unit/nest/place-import/gpx.provider.test.ts index ceabaa808..6f954f835 100644 --- a/server/tests/unit/nest/gpx-export.helpers.test.ts +++ b/server/tests/unit/nest/place-import/gpx.provider.test.ts @@ -6,7 +6,7 @@ */ import { describe, it, expect } from 'vitest'; import { XMLParser, XMLValidator } from 'fast-xml-parser'; -import { buildGpx, gpxFilename, type GpxExportPlace, type GpxExportDay } from '../../../src/nest/places/gpx-export.helpers'; +import { buildGpx, gpxFilename, type GpxExportPlace, type GpxExportDay } from '../../../../src/nest/place-import/providers/gpx.provider'; const place = (over: Partial = {}): GpxExportPlace => ({ name: 'Somewhere', description: null, address: null, lat: 48.8566, lng: 2.3522, diff --git a/server/tests/unit/nest/place-import/kml.codec.test.ts b/server/tests/unit/nest/place-import/kml.codec.test.ts new file mode 100644 index 000000000..939091051 --- /dev/null +++ b/server/tests/unit/nest/place-import/kml.codec.test.ts @@ -0,0 +1,172 @@ +import { describe, it, expect } from 'vitest'; +import path from 'path'; +import fs from 'fs'; +import { + decodeUtf8WithWarning, + extractKmlPlacemarkNodes, + KMZ_DECOMPRESSED_SIZE_LIMIT, + parseKmlPointCoordinates, + parsePlacemarkNode, + readKmlDocument, + sanitizeKmlDescription, + unpackKmzToKml, +} from '../../../../src/nest/place-import/kml.codec'; +import { buildCategoryNameLookup, resolveCategoryIdForFolder } from '../../../../src/nest/places/places.helpers'; + +describe('kmlImportUtils', () => { + it('sanitizes HTML descriptions with br to newline', () => { + const input = 'Line 1
Line 2 & more'; + const output = sanitizeKmlDescription(input); + expect(output).toBe('Line 1\nLine 2 & more'); + }); + + it('unwraps CDATA sections before stripping tags', () => { + const input = 'for photos and skyline.]]>'; + expect(sanitizeKmlDescription(input)).toBe('Great spot\nfor photos and skyline.'); + }); + + it('parses KML coordinate order lng,lat,alt', () => { + const parsed = parseKmlPointCoordinates('13.4050,52.5200,15'); + expect(parsed).toEqual({ lat: 52.52, lng: 13.405 }); + }); + + it('extracts placemarks from nested folders', () => { + const root = { + Document: { + Folder: { + name: 'Parent', + Folder: { + name: 'Child', + Placemark: { name: 'Nested', Point: { coordinates: '13.4,52.5,0' } }, + }, + }, + }, + }; + + const nodes = extractKmlPlacemarkNodes(root); + expect(nodes).toHaveLength(1); + expect(nodes[0].folderName).toBe('Child'); + + const parsed = parsePlacemarkNode(nodes[0]); + expect(parsed.name).toBe('Nested'); + expect(parsed.lat).toBe(52.5); + expect(parsed.lng).toBe(13.4); + }); + + it('builds exact case-insensitive category lookup', () => { + const lookup = buildCategoryNameLookup([ + { id: 3, name: 'Museums' }, + { id: 4, name: 'Parks' }, + ]); + + expect(resolveCategoryIdForFolder('museums', lookup)).toBe(3); + expect(resolveCategoryIdForFolder('Museum', lookup)).toBeNull(); + expect(resolveCategoryIdForFolder('parks', lookup)).toBe(4); + }); + + it('decodes non-BMP decimal HTML entities (emoji)', () => { + // 😀 = U+1F600 = 😀 — requires String.fromCodePoint, not fromCharCode + expect(sanitizeKmlDescription('😀')).toBe('😀'); + }); + + it('decodes non-BMP hex HTML entities (emoji)', () => { + // 😀 = U+1F600 = 😀 + expect(sanitizeKmlDescription('😀')).toBe('😀'); + }); + + it('does not produce [object Object] when description is a parsed object with #text', () => { + // fast-xml-parser can return an object for mixed-content nodes when stopNodes + // is not configured; the fallback in asTrimmedString must extract #text. + const result = sanitizeKmlDescription({ '#text': 'Hello world' } as any); + expect(result).not.toBe('[object Object]'); + expect(result).toBe('Hello world'); + }); + + it('returns null when description object has no #text', () => { + expect(sanitizeKmlDescription({ i: 'bold' } as any)).toBeNull(); + }); + + it('returns warning for non-UTF8 payload', () => { + const buffer = Buffer.concat([ + Buffer.from('Caf'), + Buffer.from([0xe9]), + Buffer.from(''), + ]); + + const decoded = decodeUtf8WithWarning(buffer); + expect(decoded.warning).toContain('not valid UTF-8'); + expect(decoded.text).toContain(''); + }); +}); + +const KMZ_FIXTURE = path.join(__dirname, '../../../fixtures/test.kmz'); + +describe('unpackKmzToKml', () => { + it('extracts the KML entry from a valid KMZ', async () => { + const kmzBuffer = fs.readFileSync(KMZ_FIXTURE); + const kmlBuffer = await unpackKmzToKml(kmzBuffer); + expect(kmlBuffer.length).toBeGreaterThan(0); + expect(kmlBuffer.toString('utf-8')).toContain(' { + const kmzBuffer = fs.readFileSync(KMZ_FIXTURE); + // test.kmz contains a KML with uncompressedSize 634 — set limit to 1 byte + await expect(unpackKmzToKml(kmzBuffer, 1)).rejects.toThrow('exceeds the maximum allowed decompressed size'); + }); + + it('rejects a KMZ that contains no KML file', async () => { + // Craft a minimal ZIP containing only a non-KML entry using raw ZIP bytes + // We use the test GPX fixture (a real file) re-zipped via Node's zlib/archiver + // Simplest: a KMZ whose only file has a .txt extension + const Archiver = await import('archiver'); + const archiver = Archiver.default; + const { PassThrough } = await import('stream'); + + const chunks: Buffer[] = []; + const output = new PassThrough(); + output.on('data', (chunk) => chunks.push(chunk)); + + const archive = archiver('zip', { zlib: { level: 1 } }); + archive.pipe(output); + archive.append(Buffer.from('not a kml'), { name: 'data.txt' }); + await archive.finalize(); + + const zipBuffer = Buffer.concat(chunks); + await expect(unpackKmzToKml(zipBuffer)).rejects.toThrow('does not contain a KML file'); + }); + + it('rejects a buffer that is not a valid ZIP archive', async () => { + await expect(unpackKmzToKml(Buffer.from('this is not a zip'))).rejects.toThrow('Invalid KMZ archive'); + }); + + it('exports KMZ_DECOMPRESSED_SIZE_LIMIT as 50 MB', () => { + expect(KMZ_DECOMPRESSED_SIZE_LIMIT).toBe(50 * 1024 * 1024); + }); +}); + +describe('readKmlDocument', () => { + it('reads every placemark (a level before its folders) and counts them in the summary', () => { + const kml = `Food + Cafe13.4,52.5,0 + Walk13.4,52.5 13.5,52.6 + `; + const read = readKmlDocument(Buffer.from(kml, 'utf-8')); + expect(read.summary).toMatchObject({ totalPlacemarks: 2, createdCount: 0, skippedCount: 0, warnings: [], errors: [] }); + expect(read.placemarks.map((p) => [p.name, p.folderName, p.routeGeometry !== null])).toEqual([ + ['Walk', null, true], + ['Cafe', 'Food', false], + ]); + }); + + it('refuses a document that is not well-formed XML with the route message', () => { + expect(() => readKmlDocument(Buffer.from('', 'utf-8'))).toThrow('Malformed KML: invalid XML structure'); + }); + + it('keeps reading a file that is not valid UTF-8 and says so in the summary', () => { + const bytes = Buffer.concat([Buffer.from(''), Buffer.from([0xff]), Buffer.from('')]); + expect(readKmlDocument(bytes).summary.warnings).toEqual([ + 'The uploaded file is not valid UTF-8. Some characters may be shown incorrectly.', + ]); + }); +}); diff --git a/server/tests/unit/nest/place-import/list-providers.test.ts b/server/tests/unit/nest/place-import/list-providers.test.ts new file mode 100644 index 000000000..b2253da6d --- /dev/null +++ b/server/tests/unit/nest/place-import/list-providers.test.ts @@ -0,0 +1,153 @@ +/** + * The Naver list and Google directions sources on their own: every refusal + * they hand the import route (status and exact message), the short-link hop + * through the SSRF guard, and what a good link reads as. The persistence half + * is PlacesService's and covered in places.service.test.ts. + */ +import { describe, it, expect, vi, afterEach } from 'vitest'; + +const { mockCheckSsrf, mockFollow, SsrfBlocked } = vi.hoisted(() => { + class SsrfBlocked extends Error {} + return { + mockCheckSsrf: vi.fn(async (_url: string) => ({ allowed: true })), + mockFollow: vi.fn(), + SsrfBlocked, + }; +}); +vi.mock('../../../../src/utils/ssrfGuard', () => ({ + checkSsrf: mockCheckSsrf, + safeFetchFollow: mockFollow, + SsrfBlockedError: SsrfBlocked, +})); + +import { NaverListProvider } from '../../../../src/nest/place-import/providers/naver-list.provider'; +import { GoogleDirectionsProvider } from '../../../../src/nest/place-import/providers/google-directions.provider'; +import { MAX_LIST_RESPONSE_BYTES } from '../../../../src/nest/place-import/providers/google-list.provider'; + +const FOLDER_URL = 'https://map.naver.com/p/favorite/myPlace/folder/abc123'; + +const page = (bookmarkList: Record[], folder: Record = { name: 'Seoul', bookmarkCount: bookmarkList.length }) => + new Response(JSON.stringify({ folder, bookmarkList }), { status: 200 }); + +afterEach(() => { + vi.unstubAllGlobals(); + mockCheckSsrf.mockReset(); + mockCheckSsrf.mockResolvedValue({ allowed: true }); + mockFollow.mockReset(); +}); + +describe('NaverListProvider.read', () => { + const naver = () => new NaverListProvider(); + + it('refuses what the SSRF guard blocks and an unparseable URL', async () => { + mockCheckSsrf.mockResolvedValueOnce({ allowed: false }); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'URL is not allowed', status: 400 }); + expect(await naver().read('not a url')).toEqual({ error: 'Invalid URL', status: 400 }); + }); + + it('follows a naver.me short link through the guard, and refuses a hop it blocks', async () => { + mockFollow.mockResolvedValueOnce({ url: FOLDER_URL }); + vi.stubGlobal('fetch', vi.fn(async () => page([{ name: 'Cafe', py: '37.5', px: '127.0' }]))); + expect(await naver().read('https://naver.me/xyz')).toEqual({ + listName: 'Seoul', + places: [{ name: 'Cafe', lat: 37.5, lng: 127, notes: null, address: null }], + }); + + mockFollow.mockRejectedValueOnce(new SsrfBlocked('blocked')); + expect(await naver().read('https://naver.me/xyz')).toEqual({ error: 'URL is not allowed', status: 400 }); + + mockFollow.mockRejectedValueOnce(new Error('network')); + await expect(naver().read('https://naver.me/xyz')).rejects.toThrow('network'); + }); + + it('needs a folder id in the link', async () => { + expect(await naver().read('https://map.naver.com/p/somewhere')).toEqual({ + error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', + status: 400, + }); + }); + + it('answers a failed, oversized or unreadable page with the provider message', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('nope', { status: 500 }))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'Failed to fetch list from Naver Maps', status: 502 }); + + vi.stubGlobal('fetch', vi.fn(async () => new Response('{}', { status: 200, headers: { 'content-length': String(MAX_LIST_RESPONSE_BYTES + 1) } }))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'Failed to fetch list from Naver Maps', status: 502 }); + + vi.stubGlobal('fetch', vi.fn(async () => new Response('not json', { status: 200 }))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'Invalid list data received from Naver Maps', status: 400 }); + }); + + it('pages through a folder bigger than one page and keeps what has coordinates', async () => { + const first = Array.from({ length: 20 }, (_, i) => ({ name: `P${i}`, py: '37', px: '127', memo: i === 0 ? ' note ' : '', address: i === 0 ? ' Addr ' : '' })); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(page(first, { name: 'Big', bookmarkCount: 22 })) + .mockResolvedValueOnce(page([{ displayName: ' Second ', py: '36', px: '128' }, { name: 'No coords' }])); + vi.stubGlobal('fetch', fetchMock); + const out = await naver().read(FOLDER_URL); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(String(fetchMock.mock.calls[1][0])).toContain('start=20'); + expect(out).toMatchObject({ listName: 'Big' }); + const places = (out as { places: unknown[] }).places; + expect(places).toHaveLength(21); + expect(places[0]).toEqual({ name: 'P0', lat: 37, lng: 127, notes: 'note', address: 'Addr' }); + expect(places[20]).toEqual({ name: 'Second', lat: 36, lng: 128, notes: null, address: null }); + }); + + it('stops paging on an empty page and fails on a failed one', async () => { + const twenty = Array.from({ length: 20 }, (_, i) => ({ name: `P${i}`, py: '37', px: '127' })); + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(page(twenty, { bookmarkCount: 60 })).mockResolvedValueOnce(page([]))); + expect(((await naver().read(FOLDER_URL)) as { places: unknown[]; listName: string }).listName).toBe('Naver Maps List'); + + vi.stubGlobal('fetch', vi.fn().mockResolvedValueOnce(page(twenty, { bookmarkCount: 60 })).mockResolvedValueOnce(new Response('', { status: 503 }))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'Failed to fetch list from Naver Maps', status: 502 }); + }); + + it('refuses an empty folder and one without a single coordinate', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ folder: { name: 'Empty' } }), { status: 200 }))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'List is empty or could not be read', status: 400 }); + + vi.stubGlobal('fetch', vi.fn(async () => page([{ name: 'Nowhere' }]))); + expect(await naver().read(FOLDER_URL)).toEqual({ error: 'No places with coordinates found in list', status: 400 }); + }); +}); + +describe('GoogleDirectionsProvider.read', () => { + const directions = () => new GoogleDirectionsProvider(); + const ROUTE = 'https://www.google.com/maps/dir/52.52,13.405/51.05,13.74/'; + + it('reads the stops of a route link, in driving order', async () => { + expect(await directions().read(ROUTE)).toEqual([ + { name: null, lat: 52.52, lng: 13.405 }, + { name: null, lat: 51.05, lng: 13.74 }, + ]); + }); + + it('refuses what the SSRF guard blocks and an unparseable URL', async () => { + mockCheckSsrf.mockResolvedValueOnce({ allowed: false }); + expect(await directions().read(ROUTE)).toEqual({ error: 'URL is not allowed', status: 400 }); + expect(await directions().read('not a url')).toEqual({ error: 'Invalid URL', status: 400 }); + }); + + it('follows a short link through the guard and judges the host it lands on', async () => { + mockFollow.mockResolvedValueOnce({ url: ROUTE }); + expect(await directions().read('https://maps.app.goo.gl/abc')).toHaveLength(2); + + mockFollow.mockResolvedValueOnce({ url: 'https://evil.example.com/maps/dir/A/B/' }); + expect(await directions().read('https://maps.app.goo.gl/abc')).toEqual({ error: 'That link is not a Google Maps link.', status: 400 }); + + mockFollow.mockRejectedValueOnce(new SsrfBlocked('blocked')); + expect(await directions().read('https://maps.app.goo.gl/abc')).toEqual({ error: 'URL is not allowed', status: 400 }); + + mockFollow.mockRejectedValueOnce(new Error('network')); + await expect(directions().read('https://maps.app.goo.gl/abc')).rejects.toThrow('network'); + }); + + it('refuses a link with fewer than two stops', async () => { + expect(await directions().read('https://www.google.com/maps/dir/Berlin/')).toEqual({ + error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', + status: 400, + }); + }); +}); diff --git a/server/tests/unit/nest/place-import/place-import.service.test.ts b/server/tests/unit/nest/place-import/place-import.service.test.ts new file mode 100644 index 000000000..0d16f325b --- /dev/null +++ b/server/tests/unit/nest/place-import/place-import.service.test.ts @@ -0,0 +1,114 @@ +/** + * PlaceImportService: the facade the importing domains call. Each read and + * write reaches its provider, the KML/KMZ upload is dispatched by extension, + * and a directions link's named stops are placed through the geocoder the + * caller hands in, a failed or missing answer costing only that stop. + */ +import fs from 'fs'; +import path from 'path'; +import { describe, it, expect, vi, afterEach } from 'vitest'; + +const { mockCheckSsrf } = vi.hoisted(() => ({ mockCheckSsrf: vi.fn(async (_url: string) => ({ allowed: true })) })); +vi.mock('../../../../src/utils/ssrfGuard', () => ({ + checkSsrf: mockCheckSsrf, + safeFetchFollow: vi.fn(), + SsrfBlockedError: class SsrfBlockedError extends Error {}, +})); + +import { buildPlaceImportService } from '../../../helpers/place-import'; +import { gpxGeometryParser, gpxTextParser, createGpxParser, writeGpx } from '../../../../src/nest/place-import/gpx.codec'; +import { COLLECTION_FILE_FORMAT, COLLECTION_FILE_VERSION } from '@trek/shared'; + +const KMZ_FIXTURE = path.join(__dirname, '../../../fixtures/test.kmz'); + +afterEach(() => { + vi.unstubAllGlobals(); + mockCheckSsrf.mockReset(); + mockCheckSsrf.mockResolvedValue({ allowed: true }); +}); + +describe('PlaceImportService: directions', () => { + const ROUTE = 'https://www.google.com/maps/dir/Berlin/52.52,13.405/Dresden/Prague/'; + + it('places named stops through the geocoder it is handed, coordinates as they stand', async () => { + const geocode = vi.fn(async (q: string) => (q === 'Berlin' ? { lat: 52.5, lng: 13.4 } : q === 'Dresden' ? null : Promise.reject(new Error('down')))); + const out = await buildPlaceImportService().readGoogleDirections(ROUTE, geocode); + expect(geocode).toHaveBeenCalledTimes(3); + expect(out).toEqual({ + places: [ + { name: 'Berlin', lat: 52.5, lng: 13.4, notes: null, googleFtid: null }, + { name: '52.52000, 13.40500', lat: 52.52, lng: 13.405, notes: null, googleFtid: null }, + ], + unplaceable: 2, + }); + }); + + it('refuses a route of which fewer than two stops could be placed', async () => { + const out = await buildPlaceImportService().readGoogleDirections('https://www.google.com/maps/dir/A/B/', async () => null); + expect(out).toEqual({ error: 'None of the stops in that link could be placed on the map.', status: 400 }); + }); + + it('hands a provider refusal straight back without geocoding', async () => { + mockCheckSsrf.mockResolvedValueOnce({ allowed: false }); + const geocode = vi.fn(); + expect(await buildPlaceImportService().readGoogleDirections(ROUTE, geocode)).toEqual({ error: 'URL is not allowed', status: 400 }); + expect(geocode).not.toHaveBeenCalled(); + }); +}); + +describe('PlaceImportService: files and lists', () => { + const svc = buildPlaceImportService(); + + it('dispatches a map upload by its extension, and refuses anything else', async () => { + const kml = Buffer.from('A13.4,52.5'); + expect((await svc.readMapFile(kml, 'trip.KML')).placemarks).toHaveLength(1); + expect((await svc.readMapFile(fs.readFileSync(KMZ_FIXTURE), 'trip.kmz')).summary.totalPlacemarks).toBeGreaterThan(0); + await expect(svc.readMapFile(kml, 'trip.txt')).rejects.toThrow('Unsupported map file format: .txt. Please upload a .kml or .kmz file.'); + }); + + it('reads a trip GPX and writes one back under a filename', () => { + const gpx = Buffer.from('Here'); + expect(svc.readGpx(gpx)).toEqual([{ lat: 1, lng: 2, name: 'Here', description: null }]); + const doc = svc.writeTripGpx({ tripTitle: 'T', places: [{ name: 'Here', description: null, address: null, lat: 1, lng: 2, route_geometry: null, category: null }], days: [] }); + expect(doc).toContain(''); + expect(svc.tripGpxFilename('My Trip')).toBe('My-Trip.gpx'); + }); + + it('writes a list as GPX and reads it back', () => { + const exported = svc.writeCollectionGpx({ + format: COLLECTION_FILE_FORMAT, + version: COLLECTION_FILE_VERSION, + name: 'Coffee', + exported_at: '2026-01-01T00:00:00.000Z', + places: [{ name: 'Bar', lat: 1, lng: 2 }], + } as never); + expect(exported.waypoints).toBe(1); + expect(svc.readCollectionGpx(exported.gpx, 'coffee.gpx').file.places[0]).toMatchObject({ name: 'Bar', lat: 1, lng: 2 }); + }); + + it('answers a blocked list link before fetching anything', async () => { + mockCheckSsrf.mockResolvedValue({ allowed: false }); + expect(await svc.readGoogleList('https://www.google.com/maps/placelists/list/x')).toEqual({ error: 'URL is not allowed', status: 400 }); + expect(await svc.readNaverList('https://map.naver.com/p/favorite/myPlace/folder/x')).toEqual({ error: 'URL is not allowed', status: 400 }); + }); +}); + +describe('gpx.codec', () => { + it('reads the same document two ways: geometry parses values, text keeps them', () => { + const xml = '007'; + const geometry = gpxGeometryParser.parse(xml).gpx.wpt[0]; + const text = gpxTextParser.parse(xml).gpx.wpt[0]; + expect(geometry.name).toBe(7); + expect(text.name).toBe('007'); + // `link` repeats only for the list reader. + expect(Array.isArray(text.link)).toBe(true); + expect(Array.isArray(createGpxParser('geometry').parse('').gpx.wpt[0].link)).toBe(false); + }); + + it('writes one envelope: namespaces after the default one, empty element lists left out', () => { + const doc = writeGpx({ namespaces: { trek: 'https://trek.example/gpx' }, metadata: { name: 'X' }, wpt: [], trk: [{ name: 'T' }] }); + expect(doc).toContain('xmlns="http://www.topografix.com/GPX/1/1" xmlns:trek="https://trek.example/gpx" xmlns:xsi='); + expect(doc).not.toContain(''); + }); +}); diff --git a/server/tests/unit/nest/places.directions.test.ts b/server/tests/unit/nest/places.directions.test.ts index 5463c2cf5..30df5c0cb 100644 --- a/server/tests/unit/nest/places.directions.test.ts +++ b/server/tests/unit/nest/places.directions.test.ts @@ -34,6 +34,7 @@ import { accommodationsOver } from '../../helpers/accommodations-service'; import { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import type { MapsService } from '../../../src/nest/maps/maps.service'; import type { OsmClient } from '../../../src/nest/maps/providers/osm.client'; import { QueryHelpersService } from '../../../src/nest/query-helpers/query-helpers.service'; @@ -63,24 +64,25 @@ const hit = (name: string, lat: number, lng: number) => ({ }); async function svc(searchNominatim: OsmClient['searchNominatim']): Promise { + // The address backfill runs fire-and-forget after every import, so the stub answers + // it too — otherwise every passing test prints a rejected promise. + // geocodeQuery is what the importer calls now: it asks the TREK index first + // and falls through to Nominatim on the background lane. Stubbed in terms of + // the searchNominatim each case already provides, so the cases keep driving + // one seam while the code under test uses the real one. + const maps = { + searchNominatim, + geocodeQuery: async (query: string) => { + const hit = (await searchNominatim(query, undefined, 'background')) + .find((h: { lat: number | null; lng: number | null }) => h.lat !== null && h.lng !== null); + return hit ? { lat: hit.lat as number, lng: hit.lng as number } : null; + }, + reverseGeocode: vi.fn(async () => null), + } as unknown as MapsService; return new PlacesService( new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), new RealtimeService(), - // The address backfill runs fire-and-forget after every import, so the stub answers - // it too — otherwise every passing test prints a rejected promise. - // geocodeQuery is what the importer calls now: it asks the TREK index first - // and falls through to Nominatim on the background lane. Stubbed in terms of - // the searchNominatim each case already provides, so the cases keep driving - // one seam while the code under test uses the real one. - { - searchNominatim, - geocodeQuery: async (query: string) => { - const hit = (await searchNominatim(query, undefined, 'background')) - .find((h: { lat: number | null; lng: number | null }) => h.lat !== null && h.lng !== null); - return hit ? { lat: hit.lat as number, lng: hit.lng as number } : null; - }, - reverseGeocode: vi.fn(async () => null), - } as unknown as MapsService, + maps, new QueryHelpersService(await createTestTagsRepo(testDb), await createTestPlaceRatingsRepo(testDb), await createTestAssignmentParticipantsRepo(testDb)), new UnsplashService(await createTestAppSettingsRepo(testDb), await createTestUsersRepo(testDb), new RuntimeEnvService(), storageFx.storage), photoCacheStub, @@ -102,6 +104,7 @@ async function svc(searchNominatim: OsmClient['searchNominatim']): Promise ({ db: { prepare: vi.fn() }, @@ -19,64 +17,14 @@ vi.mock('../../../src/config', () => ({ import { COORD_DEDUP_TOLERANCE, - googleMapsFeatureIdFromItem, - googleMapsHexId, externalIdsOf, isPlaceDuplicate, - KMZ_DECOMPRESSED_SIZE_LIMIT, mapWithConcurrency, trackInsertedInDedupSet, trimOrNull, - unpackKmzToKml, type DedupSet, } from '../../../src/nest/places/places.helpers'; -const KMZ_FIXTURE = path.join(__dirname, '../../fixtures/test.kmz'); - -describe('unpackKmzToKml', () => { - it('extracts the KML entry from a valid KMZ', async () => { - const kmzBuffer = fs.readFileSync(KMZ_FIXTURE); - const kmlBuffer = await unpackKmzToKml(kmzBuffer); - expect(kmlBuffer.length).toBeGreaterThan(0); - expect(kmlBuffer.toString('utf-8')).toContain(' { - const kmzBuffer = fs.readFileSync(KMZ_FIXTURE); - // test.kmz contains a KML with uncompressedSize 634 — set limit to 1 byte - await expect(unpackKmzToKml(kmzBuffer, 1)).rejects.toThrow('exceeds the maximum allowed decompressed size'); - }); - - it('rejects a KMZ that contains no KML file', async () => { - // Craft a minimal ZIP containing only a non-KML entry using raw ZIP bytes - // We use the test GPX fixture (a real file) re-zipped via Node's zlib/archiver - // Simplest: a KMZ whose only file has a .txt extension - const Archiver = await import('archiver'); - const archiver = Archiver.default; - const { PassThrough } = await import('stream'); - - const chunks: Buffer[] = []; - const output = new PassThrough(); - output.on('data', (chunk) => chunks.push(chunk)); - - const archive = archiver('zip', { zlib: { level: 1 } }); - archive.pipe(output); - archive.append(Buffer.from('not a kml'), { name: 'data.txt' }); - await archive.finalize(); - - const zipBuffer = Buffer.concat(chunks); - await expect(unpackKmzToKml(zipBuffer)).rejects.toThrow('does not contain a KML file'); - }); - - it('rejects a buffer that is not a valid ZIP archive', async () => { - await expect(unpackKmzToKml(Buffer.from('this is not a zip'))).rejects.toThrow('Invalid KMZ archive'); - }); - - it('exports KMZ_DECOMPRESSED_SIZE_LIMIT as 50 MB', () => { - expect(KMZ_DECOMPRESSED_SIZE_LIMIT).toBe(50 * 1024 * 1024); - }); -}); - // ── Import dedup predicates ─────────────────────────────────────────────────── const emptyDedup = (): DedupSet => ({ names: new Set(), coords: [], externalIds: new Set() }); @@ -149,32 +97,6 @@ describe('isPlaceDuplicate / trackInsertedInDedupSet', () => { }); }); -// ── Google Maps feature ids ─────────────────────────────────────────────────── - -describe('googleMapsHexId / googleMapsFeatureIdFromItem', () => { - it('passes an already-hex id through lower-cased', () => { - expect(googleMapsHexId('0x882BF179E806D471')).toBe('0x882bf179e806d471'); - }); - - it("converts Google's signed 64-bit decimals to unsigned hex", () => { - expect(googleMapsHexId('-8634542354666695567')).toBe('0x882bf179e806d471'); - expect(googleMapsHexId(255)).toBe('0xff'); - }); - - it('rejects anything that is not a hex or decimal id', () => { - expect(googleMapsHexId('not-an-id')).toBeNull(); - expect(googleMapsHexId(null)).toBeNull(); - expect(googleMapsHexId({})).toBeNull(); - }); - - it('reads the ftid pair from either item slot, else null', () => { - expect(googleMapsFeatureIdFromItem([null, [0, 0, 0, 0, 0, 0, ['0x1', '0x2']]])).toBe('0x1:0x2'); - expect(googleMapsFeatureIdFromItem([null, null, null, null, null, null, null, [null, ['0x3', '0x4']]])).toBe('0x3:0x4'); - expect(googleMapsFeatureIdFromItem([null, [0, 0, 0, 0, 0, 0, ['0x1']]])).toBeNull(); - expect(googleMapsFeatureIdFromItem('not an array')).toBeNull(); - }); -}); - // ── Enrichment plumbing ─────────────────────────────────────────────────────── describe('mapWithConcurrency / trimOrNull', () => { diff --git a/server/tests/unit/nest/places.service.test.ts b/server/tests/unit/nest/places.service.test.ts index 17f3a12c6..1bd741ffe 100644 --- a/server/tests/unit/nest/places.service.test.ts +++ b/server/tests/unit/nest/places.service.test.ts @@ -62,6 +62,7 @@ import fs from 'fs'; import { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { legacyBoundIntegerText } from '../../../src/nest/common/row-id'; import { MapsService } from '../../../src/nest/maps/maps.service'; import { buildMapsService } from '../../helpers/maps-service'; @@ -163,6 +164,7 @@ async function makePlacesService( await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestCollectionPlacesRepo(testDb), + buildPlaceImportService(), ); } diff --git a/server/tests/unit/nest/places/kml-import.helpers.test.ts b/server/tests/unit/nest/places/kml-import.helpers.test.ts deleted file mode 100644 index d2d42f8e1..000000000 --- a/server/tests/unit/nest/places/kml-import.helpers.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { - buildCategoryNameLookup, - decodeUtf8WithWarning, - extractKmlPlacemarkNodes, - parseKmlPointCoordinates, - parsePlacemarkNode, - resolveCategoryIdForFolder, - sanitizeKmlDescription, -} from '../../../../src/nest/places/kml-import.helpers'; - -describe('kmlImportUtils', () => { - it('sanitizes HTML descriptions with br to newline', () => { - const input = 'Line 1
Line 2 & more'; - const output = sanitizeKmlDescription(input); - expect(output).toBe('Line 1\nLine 2 & more'); - }); - - it('unwraps CDATA sections before stripping tags', () => { - const input = 'for photos and skyline.]]>'; - expect(sanitizeKmlDescription(input)).toBe('Great spot\nfor photos and skyline.'); - }); - - it('parses KML coordinate order lng,lat,alt', () => { - const parsed = parseKmlPointCoordinates('13.4050,52.5200,15'); - expect(parsed).toEqual({ lat: 52.52, lng: 13.405 }); - }); - - it('extracts placemarks from nested folders', () => { - const root = { - Document: { - Folder: { - name: 'Parent', - Folder: { - name: 'Child', - Placemark: { name: 'Nested', Point: { coordinates: '13.4,52.5,0' } }, - }, - }, - }, - }; - - const nodes = extractKmlPlacemarkNodes(root); - expect(nodes).toHaveLength(1); - expect(nodes[0].folderName).toBe('Child'); - - const parsed = parsePlacemarkNode(nodes[0]); - expect(parsed.name).toBe('Nested'); - expect(parsed.lat).toBe(52.5); - expect(parsed.lng).toBe(13.4); - }); - - it('builds exact case-insensitive category lookup', () => { - const lookup = buildCategoryNameLookup([ - { id: 3, name: 'Museums' }, - { id: 4, name: 'Parks' }, - ]); - - expect(resolveCategoryIdForFolder('museums', lookup)).toBe(3); - expect(resolveCategoryIdForFolder('Museum', lookup)).toBeNull(); - expect(resolveCategoryIdForFolder('parks', lookup)).toBe(4); - }); - - it('decodes non-BMP decimal HTML entities (emoji)', () => { - // 😀 = U+1F600 = 😀 — requires String.fromCodePoint, not fromCharCode - expect(sanitizeKmlDescription('😀')).toBe('😀'); - }); - - it('decodes non-BMP hex HTML entities (emoji)', () => { - // 😀 = U+1F600 = 😀 - expect(sanitizeKmlDescription('😀')).toBe('😀'); - }); - - it('does not produce [object Object] when description is a parsed object with #text', () => { - // fast-xml-parser can return an object for mixed-content nodes when stopNodes - // is not configured; the fallback in asTrimmedString must extract #text. - const result = sanitizeKmlDescription({ '#text': 'Hello world' } as any); - expect(result).not.toBe('[object Object]'); - expect(result).toBe('Hello world'); - }); - - it('returns null when description object has no #text', () => { - expect(sanitizeKmlDescription({ i: 'bold' } as any)).toBeNull(); - }); - - it('returns warning for non-UTF8 payload', () => { - const buffer = Buffer.concat([ - Buffer.from('Caf'), - Buffer.from([0xe9]), - Buffer.from(''), - ]); - - const decoded = decodeUtf8WithWarning(buffer); - expect(decoded.warning).toContain('not valid UTF-8'); - expect(decoded.text).toContain(''); - }); -}); diff --git a/server/tests/unit/nest/tours.gpx.atomic.test.ts b/server/tests/unit/nest/tours.gpx.atomic.test.ts index 6dc03c4f7..8e98159ca 100644 --- a/server/tests/unit/nest/tours.gpx.atomic.test.ts +++ b/server/tests/unit/nest/tours.gpx.atomic.test.ts @@ -29,12 +29,13 @@ import { RuntimeEnvService } from '../../../src/nest/app-config/runtime-env.serv import { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { buildMapsService } from '../../helpers/maps-service'; import { QueryHelpersService } from '../../../src/nest/query-helpers/query-helpers.service'; import { JourneyDomainService } from '../../../src/nest/journey/journey-domain.service'; import { TrekPhotoRegistrationService } from '../../../src/nest/photos/trek-photo-registration.service'; import { ToursService } from '../../../src/nest/tours/tours.service'; -import { gpxParser } from '../../../src/nest/places/places.helpers'; +import { gpxGeometryParser } from '../../../src/nest/place-import/gpx.codec'; import { TrekPhotos } from '../../../src/db/entities/TrekPhotos.entity'; import { TripPhotos } from '../../../src/db/entities/TripPhotos.entity'; import { resetTestDb } from '../../helpers/test-db'; @@ -101,6 +102,7 @@ async function makePlacesService(): Promise { await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestCollectionPlacesRepo(testDb), + buildPlaceImportService(), ); } @@ -128,6 +130,7 @@ beforeAll(async () => { places = await makePlacesService(); tours = new ToursService( uow, places, await createTestToursRepo(testDb), await createTestTourTypesRepo(testDb), await createTestTourWaypointsRepo(testDb), await createTestPlacesRepo(testDb), + buildPlaceImportService(), ); }); @@ -221,7 +224,7 @@ describe('Tours GPX atomic persistence and postcommit publication', () => { it('propagates parser failures before entering the transaction', async () => { const transactional = vi.spyOn(uow, 'transactional'); - vi.spyOn(gpxParser, 'parse').mockImplementationOnce(() => { throw new Error('parser failure'); }); + vi.spyOn(gpxGeometryParser, 'parse').mockImplementationOnce(() => { throw new Error('parser failure'); }); await expect(tours.importGpxAsTour(tripId, mixedGpx)).rejects.toThrow('parser failure'); expect(transactional).not.toHaveBeenCalled(); await expectEmpty(); diff --git a/server/tests/unit/nest/tours.permissions.boundaries.mock-only.test.ts b/server/tests/unit/nest/tours.permissions.boundaries.mock-only.test.ts index 071d785e2..3469618b9 100644 --- a/server/tests/unit/nest/tours.permissions.boundaries.mock-only.test.ts +++ b/server/tests/unit/nest/tours.permissions.boundaries.mock-only.test.ts @@ -40,6 +40,7 @@ vi.mock('../../../src/nest/auth-core/jwt-verify', () => ({ extractToken: vi.fn() import { PlacesController } from '../../../src/nest/places/places.controller'; import { DayAssignmentsController } from '../../../src/nest/assignments/assignments.controller'; import { ToursService } from '../../../src/nest/tours/tours.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { JwtAuthGuard } from '../../../src/nest/auth-core/jwt-auth.guard'; import { TripAccessGuard, TRIP_PERMISSION_KEY } from '../../../src/nest/permissions/trip-access.guard'; import type { PermissionsService } from '../../../src/nest/permissions/permissions.service'; @@ -230,6 +231,7 @@ function tourRepositoryFixture(ownerTripId: number) { { isEnabled: vi.fn(async () => true) } as unknown as TourTypesRepository, waypointsRepo as unknown as TourWaypointsRepository, placesRepo as unknown as PlacesRepository, + buildPlaceImportService(), ); return { uow, toursRepo, waypointsRepo, placesRepo, places, service }; } diff --git a/server/tests/unit/nest/tours.service.permissions.test.ts b/server/tests/unit/nest/tours.service.permissions.test.ts index 1e301d55a..de64ce500 100644 --- a/server/tests/unit/nest/tours.service.permissions.test.ts +++ b/server/tests/unit/nest/tours.service.permissions.test.ts @@ -27,6 +27,7 @@ vi.mock('../../../src/db/database', () => ({ vi.mock('../../../src/nest/places/places.service', () => ({ PlacesService: class {} })); import { ToursService } from '../../../src/nest/tours/tours.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { tourCreateRequestSchema, type TourCreateRequest } from '@trek/shared'; import type { UnitOfWork } from '../../../src/nest/database/unit-of-work'; import type { PlacesService } from '../../../src/nest/places/places.service'; @@ -92,6 +93,7 @@ function makeService() { { isEnabled: vi.fn(async () => true) } as unknown as TourTypesRepository, waypointsRepo as unknown as TourWaypointsRepository, placesRepo as unknown as PlacesRepository, + buildPlaceImportService(), ); return { uow, toursRepo, waypointsRepo, placesRepo, places, place, service }; } diff --git a/server/tests/unit/nest/tours.service.test.ts b/server/tests/unit/nest/tours.service.test.ts index f00fa5dd3..e7bd97e86 100644 --- a/server/tests/unit/nest/tours.service.test.ts +++ b/server/tests/unit/nest/tours.service.test.ts @@ -23,6 +23,7 @@ vi.mock('../../../src/config', () => ({ import { db as testDb } from '../../../src/db/database'; import { tourCreateRequestSchema, type TourCreateRequest } from '@trek/shared'; import { ToursService } from '../../../src/nest/tours/tours.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import type { PlacesService } from '../../../src/nest/places/places.service'; import { resetTestDb } from '../../helpers/test-db'; import { createPlace, createTrip, createUser } from '../../helpers/factories'; @@ -72,6 +73,7 @@ beforeAll(async () => { await createTestTourTypesRepo(testDb), await createTestTourWaypointsRepo(testDb), await createTestPlacesRepo(testDb), + buildPlaceImportService(), ); }); diff --git a/server/tests/unit/nest/trip-read-model.service.test.ts b/server/tests/unit/nest/trip-read-model.service.test.ts index d4c0370fd..44bd3a35f 100644 --- a/server/tests/unit/nest/trip-read-model.service.test.ts +++ b/server/tests/unit/nest/trip-read-model.service.test.ts @@ -50,6 +50,7 @@ import { ExchangeRatesService } from '../../../src/nest/budget/exchange-rates.se import { CollabService } from '../../../src/nest/collab/collab.service'; import { RateLimitService } from '../../../src/nest/common/rate-limit.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service'; import { TripMembersService } from '../../../src/nest/trip-members/trip-members.service'; import { TripReadModelService } from '../../../src/nest/trip-read-model/trip-read-model.service'; @@ -173,6 +174,7 @@ beforeAll(async () => { await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestCollectionPlacesRepo(testDb), + buildPlaceImportService(), ); membersSvc = new TripMembersService(budgetSvc, new UserCleanupService(new MaintenanceRepository((await sharedTestOrm(testDb)).em), budgetSvc, await createTestUnitOfWork(testDb), await createTestUsersRepo(testDb), await createTestTripMembersRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestBudgetSettlementsRepo(testDb), await createTestJourneyShareTokensRepo(testDb), await createTestJourneysRepo(testDb), await createTestJourneyEntriesRepo(testDb), await createTestJourneyContributorsRepo(testDb), await createTestShareTokensRepo(testDb), await createTestPluginsRepo(testDb), await createTestPluginUserErasureQueueRepo(testDb)), new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), new RealtimeService(), notificationsStub(), await createTestUnitOfWork(testDb), await createTestTripsRepo(testDb), await createTestTripMembersRepo(testDb), await createTestUsersRepo(testDb)); }); diff --git a/server/tests/unit/nest/trips.service.test.ts b/server/tests/unit/nest/trips.service.test.ts index c195bc803..d7b81deac 100644 --- a/server/tests/unit/nest/trips.service.test.ts +++ b/server/tests/unit/nest/trips.service.test.ts @@ -59,6 +59,7 @@ import { RateLimitService } from '../../../src/nest/common/rate-limit.service'; import { VacayService } from '../../../src/nest/vacay/vacay.service'; import { TripsService } from '../../../src/nest/trips/trips.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { UserCleanupService } from '../../../src/nest/auth/user-cleanup.service'; import { TripMembersService } from '../../../src/nest/trip-members/trip-members.service'; import { TripReadModelService } from '../../../src/nest/trip-read-model/trip-read-model.service'; @@ -280,6 +281,7 @@ beforeAll(async () => { await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestCollectionPlacesRepo(testDb), + buildPlaceImportService(), ); svc = new TripsService( new ReservationsService(new PermissionsService(await createTestAppSettingsRepo(testDb), await createTestUnitOfWork(testDb)), budgetSvc, new RealtimeService(), notificationsStub(), new ReservationsReadService(await createTestReservationsRepo(testDb), await createTestReservationEndpointsRepo(testDb), await createTestReservationTravelersRepo(testDb)), accommodationsSvc, await createTestUnitOfWork(testDb), await createTestReservationsRepo(testDb), await createTestReservationEndpointsRepo(testDb), await createTestReservationTravelersRepo(testDb), await createTestReservationDayPositionsRepo(testDb), await createTestDayAccommodationsRepo(testDb), await createTestDaysRepo(testDb), await createTestPlacesRepo(testDb), await createTestDayAssignmentsRepo(testDb), await createTestTripMembersRepo(testDb), await createTestUsersRepo(testDb), await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb)), diff --git a/server/tests/unit/services/conflictUpdate.test.ts b/server/tests/unit/services/conflictUpdate.test.ts index 276c01633..483d06985 100644 --- a/server/tests/unit/services/conflictUpdate.test.ts +++ b/server/tests/unit/services/conflictUpdate.test.ts @@ -31,6 +31,7 @@ import { accommodationsOver } from '../../helpers/accommodations-service'; import { isUpdateConflict } from '../../../src/nest/common/conflictResult'; import { PackingService } from '../../../src/nest/packing/packing.service'; import { PlacesService } from '../../../src/nest/places/places.service'; +import { buildPlaceImportService } from '../../helpers/place-import'; import { buildMapsService } from '../../helpers/maps-service'; import { PermissionsService } from '../../../src/nest/permissions/permissions.service'; import { RealtimeService } from '../../../src/nest/realtime/realtime.service'; @@ -138,6 +139,7 @@ beforeAll(async () => { await createTestTripsRepo(testDb), await createTestBudgetItemsRepo(testDb), await createTestCollectionPlacesRepo(testDb), + buildPlaceImportService(), ); }); diff --git a/server/vitest.config.ts b/server/vitest.config.ts index b795a264d..3f9e0f5ee 100644 --- a/server/vitest.config.ts +++ b/server/vitest.config.ts @@ -138,6 +138,10 @@ export default defineConfig({ // measuring 99.5/99.0/97.4/100 — the gap the per-domain ratchet exists to // close. Set with a few points of slack against the Linux/Windows drift. 'src/nest/place-enrichment/**/*.ts': { statements: 96, branches: 95, functions: 94, lines: 97 }, + // New domain: the file and list readers that moved out of places/ and + // collections/. On the floor until a full CI run measures it; raise it + // with scripts/coverage-thresholds.mjs then, never lower it. + 'src/nest/place-import/**/*.ts': { statements: 80, branches: 80, functions: 80, lines: 80 }, 'src/nest/place-photos/**/*.ts': { statements: 87, branches: 79, functions: 72, lines: 89 }, // New domain in this change. Measured over its own suites at // 98.3/89.8/96.2/100, and pinned well under that on purpose: the