refactor(place-import): file and shared-list readers move into their own domain over one gpx and one kml codec

This commit is contained in:
Maurice
2026-10-10 13:02:42 +02:00
parent dc42fe9a5a
commit 1389fe6984
56 changed files with 1784 additions and 911 deletions
+2
View File
@@ -48,6 +48,7 @@ import { JourneyModule } from './journey/journey.module';
import { LlmParseModule } from './llm-parse/llm-parse.module';
import { ManagedExtModule } from './managed/managed-ext.module';
import { MapsModule } from './maps/maps.module';
import { PlaceImportModule } from './place-import/place-import.module';
import { McpTransportModule } from './mcp-transport/mcp-transport.module';
import { MemoriesModule } from './memories/memories.module';
import { NotificationsModule } from './notifications/notifications.module';
@@ -119,6 +120,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
SystemNoticesModule,
GeoModule,
MapsModule,
PlaceImportModule,
GoogleQuotaModule,
PlaceEnrichmentModule,
PlaceShadowModule,
@@ -56,7 +56,7 @@ import {
CollectionGpxReadDto,
} from './collections.dto';
import { PlaceRatingDto } from '../places/places.dto';
import { CollectionGpxError } from './collection-gpx.helpers';
import { CollectionGpxError } from '../place-import/place-import.types';
export const MAX_COVER_SIZE = 20 * 1024 * 1024;
// Duplicated on purpose from trips.controller.ts (historical parity — no
@@ -26,6 +26,7 @@ import { StorageModule } from '../storage/storage.module';
import { StorageService } from '../storage/storage.service';
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
import { MAX_COVER_SIZE } from './collections.controller';
import { PlaceImportModule } from '../place-import/place-import.module';
/** Collections domain (saved-places library). Registered in AppModule.
* Exports CollectionsService for in-container consumers (PluginsModule's
@@ -47,7 +48,7 @@ import { MAX_COVER_SIZE } from './collections.controller';
}),
}),
StorageModule,
NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule,
NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule, PlaceImportModule,
MikroOrmModule.forFeature([
Collections, CollectionMembers, CollectionLabels, CollectionPlaces, CollectionPlaceRatings,
Categories, Users, Trips, TripMembers, Places, PlaceRatings, Tags,
@@ -73,7 +73,7 @@ import type {
CollectionImportablesResponse,
} from '@trek/shared';
import { NotificationsService } from '../notifications/notifications.service';
import { collectionFileToGpx, gpxToCollectionFile, type ExportedCollectionFile } from './collection-gpx.helpers';
import { PlaceImportService, type ExportedCollectionFile } from '../place-import/place-import.service';
import { UnitOfWork } from '../database/unit-of-work';
/** Links are stored as a JSON TEXT column; parse on read, stringify on write. */
@@ -143,6 +143,7 @@ export class CollectionsService {
@InjectRepository(PlaceRatings) private readonly tripPlaceRatings: PlaceRatingsRepository,
@InjectRepository(Tags) private readonly tags: TagsRepository,
@InjectRepository(Users) private readonly users: UsersRepository,
private readonly placeImport: PlaceImportService,
) {}
/**
@@ -398,17 +399,16 @@ export class CollectionsService {
* only ever carry less of it. Same access rule too, since it is the same read.
*/
async exportCollectionGpx(userId: number, id: number): Promise<CollectionGpxExport> {
return collectionFileToGpx(await this.exportCollection(userId, id));
return this.placeImport.writeCollectionGpx(await this.exportCollection(userId, id));
}
/**
* A GPX document read into the list file it amounts to. Touches no table:
* the file goes back to the browser to be shown, and comes back through
* importCollection like any other, so there is one import and one
* transaction whatever the format was.
* A GPX document read (by place-import) into the list file it amounts to. Touches no table: the file
* goes back to the browser to be shown, and comes back through importCollection like any other, so
* there is one import and one transaction whatever the format was.
*/
readCollectionGpx(body: CollectionGpxReadRequest): CollectionGpxReadResult {
return gpxToCollectionFile(body.gpx, body.file_name);
return this.placeImport.readCollectionGpx(body.gpx, body.file_name);
}
/**
@@ -0,0 +1,20 @@
/**
* Which hosts a pasted Google Maps link may come from. Shared kernel because
* the maps link resolver and the directions import (place-import) both follow a
* short link through the SSRF guard and then require the landing host to be
* Google's own.
*/
export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl'];
/**
* Google Maps lives on every country domain — google.de, maps.google.co.uk,
* google.com.au — so the host is matched by shape. A fixed list of .com hosts
* would quietly stop resolving the ccTLD links people actually paste. The TLD
* labels stay short (2-3 letters, optionally two of them) so that
* `google.evil.com` is not a Google host.
*/
export function isGoogleMapsHost(hostname: string): boolean {
return GOOGLE_SHORT_HOSTS.includes(hostname)
|| /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname);
}
+1 -14
View File
@@ -14,26 +14,13 @@ import { discardBody, exceedsDeclaredLength, readCappedText } from '../../utils/
import { UA, googleFtidFromMapsUrl } from './maps.helpers';
import { AMAP_SHORT_HOSTS, isAmapHost, parseAmapUrl } from './providers/amap.provider';
import { OsmClient } from './providers/osm.client';
import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts';
// A Google Maps place page is a few hundred KB; the coordinates sit in the
// embedded map data near the top, so two megabytes is plenty and keeps an
// unbounded body out of memory.
const MAX_MAPS_PAGE_BYTES = 2_000_000;
export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl'];
/**
* Google Maps lives on every country domain — google.de, maps.google.co.uk,
* google.com.au — so the host is matched by shape. A fixed list of .com hosts
* would quietly stop resolving the ccTLD links people actually paste. The TLD
* labels stay short (2-3 letters, optionally two of them) so that
* `google.evil.com` is not a Google host.
*/
export function isGoogleMapsHost(hostname: string): boolean {
return GOOGLE_SHORT_HOSTS.includes(hostname)
|| /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname);
}
export interface ResolvedMapsUrl {
lat: number;
lng: number;
+1 -1
View File
@@ -27,7 +27,7 @@ import { NEARBY_DEFAULT_LIMIT, NEARBY_DEFAULT_RADIUS_M, nearbyCacheKey, nearestF
export { readBrandIdentity, readWikiIdentity, type WikiIdentity } from './providers/wiki-identity';
export { withPhotoFetchSlot } from './photo-fetch-slot';
export type { BrandLogo, CommonsCandidate } from './providers/wikimedia.client';
export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from './maps-url.resolver';
export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts';
export { PLACES_PROVIDER_SETTING, PLACES_GOOGLE_ONLY_SETTING, type KeyedProvider } from './places-provider.selector';
// Places near a point (#976): cached longer than the POI boxes, because the
@@ -29,6 +29,14 @@ export class PlacePhotoResolver {
private readonly selector: PlacesProviderSelector,
) {}
/**
* The photo for one place: disk hit, negative cache, in-flight dedupe, then
* Google (for a Google id) and Wikimedia, under the shared photo-fetch slots.
*
* @txIndependent cache bookkeeping around network fetches: the stored photo,
* the negative-cache entry and the image_url backfill are each complete on
* their own, and a failed one costs the cache, never the answer.
*/
async resolve(
userId: number,
placeId: string,
@@ -286,6 +286,9 @@ export class GooglePlacesClient {
* download. The outcome tells a place with no photo apart from a call that
* went wrong, because the negative cache keeps the first for a day and the
* second for minutes. Throws only when the transport itself does.
*
* @txIndependent two billed calls with network I/O between them: each counts
* against the daily ceiling on its own, as it is made.
*/
async firstPhoto(placeId: string, apiKey: string): Promise<GooglePhotoOutcome> {
// Fetch details to get the photo name
+103
View File
@@ -0,0 +1,103 @@
/**
* GPX in and out: the one place the XML library is configured for GPX.
*
* Reading comes in two modes, chosen by the caller, because the two GPX
* importers want different things from the same document and have shipped
* that way:
*
* - `geometry` (the trip and Tours importers): tag values are parsed, so
* `<ele>12.5</ele>` arrives as a number, and the repeating elements are the
* six that carry places and geometry, matched on the name as written.
* `<name>007</name>` becomes the number 7 and character references stay
* undecoded, which is fine for track geometry.
* - `text` (the list importer in collections): text stays text, so a
* waypoint called 007 keeps its zeros, numeric character references
* (`&#233;`) are decoded, and `link` / `label` repeat as well, matched on
* the local name so a namespaced `<trek:label>` counts.
*
* Writing goes through one builder and one document envelope (`writeGpx`),
* shared by the trip export and the list export: GPX 1.1, creator TREK, the
* schema location, then the caller's namespaces, metadata and elements.
*/
import { XMLBuilder, XMLParser } from 'fast-xml-parser';
export type GpxReadMode = 'geometry' | 'text';
/** Repeating elements in geometry mode: the places and the geometry, by name as written. */
const GEOMETRY_ARRAYS = ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'];
/** Repeating elements in text mode, matched on the local name. */
const TEXT_ARRAYS = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']);
const localName = (name: string): string => name.slice(name.indexOf(':') + 1);
/** The GPX parser for one read mode. Built once per mode below; exported for the tests. */
export function createGpxParser(mode: GpxReadMode): XMLParser {
if (mode === 'geometry') {
return new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
isArray: (name) => GEOMETRY_ARRAYS.includes(name),
});
}
// Entities declared in a DOCTYPE never reach the text parser: the list reader
// refuses a document carrying one before parsing.
return new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
parseTagValue: false,
parseAttributeValue: false,
htmlEntities: true,
isArray: (name, _path, _leaf, isAttribute) => !isAttribute && TEXT_ARRAYS.has(localName(name)),
});
}
/** Frozen at import, one shared instance per mode. */
export const gpxGeometryParser = createGpxParser('geometry');
export const gpxTextParser = createGpxParser('text');
export const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1';
/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer
* device resolves and keeps the file from carrying float noise. */
export function coord(n: number): string {
return Number(n.toFixed(7)).toString();
}
const gpxBuilder = new XMLBuilder({
ignoreAttributes: false,
attributeNamePrefix: '@_',
format: true,
indentBy: ' ',
suppressEmptyNode: true,
});
/** What goes inside `<gpx>`; an empty element list is left out entirely. */
export interface GpxDocument {
/** Extra `xmlns:<prefix>` bindings, written after the default namespace. */
namespaces?: Record<string, string>;
metadata: Record<string, unknown>;
wpt?: unknown[];
rte?: unknown[];
trk?: unknown[];
}
/** A GPX 1.1 document, elements in the schema's order (metadata, wpt, rte, trk). */
export function writeGpx(doc: GpxDocument): string {
const namespaces = Object.fromEntries(Object.entries(doc.namespaces ?? {}).map(([prefix, uri]) => [`@_xmlns:${prefix}`, uri]));
return gpxBuilder.build({
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
gpx: {
'@_version': '1.1',
'@_creator': 'TREK',
'@_xmlns': GPX_NAMESPACE,
...namespaces,
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`,
metadata: doc.metadata,
...(doc.wpt?.length ? { wpt: doc.wpt } : {}),
...(doc.rte?.length ? { rte: doc.rte } : {}),
...(doc.trk?.length ? { trk: doc.trk } : {}),
},
});
}
@@ -1,27 +1,38 @@
/**
* KML and KMZ in: the one place the XML library is configured for KML.
*
* A KMZ is unpacked to its KML (doc.kml preferred, the decompressed size
* capped), the bytes are decoded as UTF-8 (loosely, with a warning, when they
* are not), the document is validated and parsed, and every Placemark comes
* back with the folder it sat in, its name, a plain-text description, and
* either a point or a path's first point plus the path as route geometry.
* Which placemarks become places, and under which category, is the importing
* domain's decision.
*/
import { TextDecoder } from 'util';
import { XMLParser, XMLValidator } from 'fast-xml-parser';
import unzipper from 'unzipper';
import { stripHtmlTags } from '../common/stripHtmlTags';
import type { KmlDocumentRead, KmlImportSummary, ParsedKmlPlacemark } from './place-import.types';
export interface ParsedKmlPlacemark {
name: string | null;
description: string | null;
lat: number | null;
lng: number | null;
folderName: string | null;
routeGeometry: string | null;
}
export type { KmlImportSummary, ParsedKmlPlacemark } from './place-import.types';
export interface KmlPlacemarkNode {
placemark: any;
folderName: string | null;
}
export interface KmlImportSummary {
totalPlacemarks: number;
createdCount: number;
skippedCount: number;
warnings: string[];
errors: string[];
}
const kmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
removeNSPrefix: true,
isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name),
// Treat <description> as raw text so mixed-content HTML (e.g. <br/>, <i>)
// is returned as a string instead of a parsed object.
stopNodes: ['*.description'],
});
export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB
const UTF8_DECODER_FATAL = new TextDecoder('utf-8', { fatal: true });
const UTF8_DECODER_LOOSE = new TextDecoder('utf-8');
@@ -144,25 +155,6 @@ export function createKmlImportSummary(totalPlacemarks: number): KmlImportSummar
};
}
export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map<string, number> {
const lookup = new Map<string, number>();
for (const category of categories) {
const normalizedName = category.name.trim().toLowerCase();
if (!normalizedName) continue;
if (!lookup.has(normalizedName)) {
lookup.set(normalizedName, category.id);
}
}
return lookup;
}
export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map<string, number>): number | null {
if (!folderName) return null;
const normalizedFolder = folderName.trim().toLowerCase();
if (!normalizedFolder) return null;
return lookup.get(normalizedFolder) ?? null;
}
export function extractKmlPlacemarkNodes(kmlRoot: any): KmlPlacemarkNode[] {
const nodes: KmlPlacemarkNode[] = [];
@@ -211,3 +203,62 @@ export function parsePlacemarkNode(node: KmlPlacemarkNode): ParsedKmlPlacemark {
routeGeometry,
};
}
/**
* A KML document as its placemarks (each level's own before its folders'), with the summary the
* import fills in. Throws the two messages the import route has always
* answered for a broken file.
*/
export function readKmlDocument(fileBuffer: Buffer): KmlDocumentRead {
const decoded = decodeUtf8WithWarning(fileBuffer);
const validationResult = XMLValidator.validate(decoded.text);
if (validationResult !== true) {
throw new Error('Malformed KML: invalid XML structure');
}
const parsed = kmlParser.parse(decoded.text);
const kmlRoot = parsed?.kml ?? parsed;
if (!kmlRoot || typeof kmlRoot !== 'object') {
throw new Error('Malformed KML: could not parse XML');
}
const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot);
const summary = createKmlImportSummary(placemarkNodes.length);
if (decoded.warning) {
summary.warnings.push(decoded.warning);
}
return { placemarks: placemarkNodes.map(parsePlacemarkNode), summary };
}
// ---------------------------------------------------------------------------
// KMZ unpacking
// ---------------------------------------------------------------------------
export async function unpackKmzToKml(
kmzBuffer: Buffer,
decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT,
): Promise<Buffer> {
let zip;
try {
zip = await unzipper.Open.buffer(kmzBuffer);
} catch {
throw new Error('Invalid KMZ archive.');
}
const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml'));
if (kmlEntries.length === 0) {
throw new Error('KMZ archive does not contain a KML file.');
}
const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0];
if (preferredEntry.uncompressedSize > decompressedSizeLimit) {
throw new Error('KMZ archive exceeds the maximum allowed decompressed size.');
}
return preferredEntry.buffer();
}
@@ -0,0 +1,30 @@
import { Module } from '@nestjs/common';
import { PlaceImportService } from './place-import.service';
import { GpxProvider } from './providers/gpx.provider';
import { CollectionGpxProvider } from './providers/collection-gpx.provider';
import { KmlProvider } from './providers/kml.provider';
import { GoogleListProvider } from './providers/google-list.provider';
import { GoogleDirectionsProvider } from './providers/google-directions.provider';
import { NaverListProvider } from './providers/naver-list.provider';
/**
* Place import (leaf module): reads places out of GPX, KML/KMZ, Google lists,
* Google directions links and Naver lists, and writes GPX. No controller, no
* table of its own: PlacesModule, CollectionsModule and ToursModule import it
* and persist what it reads. It imports no other
* domain: the stops of a directions link that carry only a name are geocoded
* through a function the importer hands in (MapsService.geocodeQuery).
*/
@Module({
providers: [
PlaceImportService,
GpxProvider,
CollectionGpxProvider,
KmlProvider,
GoogleListProvider,
GoogleDirectionsProvider,
NaverListProvider,
],
exports: [PlaceImportService],
})
export class PlaceImportModule {}
@@ -0,0 +1,159 @@
import { Injectable } from '@nestjs/common';
import type { CollectionGpxExport, CollectionGpxReadResult } from '@trek/shared';
import { GpxProvider, type GpxExportInput, type GpxExportOptions } from './providers/gpx.provider';
import { CollectionGpxProvider, type ExportedCollectionFile } from './providers/collection-gpx.provider';
import { KmlProvider } from './providers/kml.provider';
import { GoogleListProvider } from './providers/google-list.provider';
import { GoogleDirectionsProvider } from './providers/google-directions.provider';
import { NaverListProvider } from './providers/naver-list.provider';
import type {
DirectionsRead,
DirectionsRedirect,
GoogleListPlace,
GpxImportOptions,
KmlDocumentRead,
ListImportError,
ListRead,
NaverListPlace,
PreparedGpxPlace,
} from './place-import.types';
// The directions-link reading is also what the road trip planner reads a
// shared route with, and what the import routes dispatch on.
export { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS, type DirWaypoint } from './directions-url.helpers';
export type { GpxExportDay, GpxExportInput, GpxExportOptions, GpxExportPlace } from './providers/gpx.provider';
export type { ExportedCollectionFile } from './providers/collection-gpx.provider';
/** Places a name, or says nobody can. */
export type Geocoder = (query: string) => Promise<{ lat: number; lng: number } | null>;
/**
* Places out of files and shared lists: the one layer that reads GPX, KML/KMZ,
* Google lists, Google directions links and Naver lists, and writes GPX.
*
* One provider per source behind it, over one GPX codec and one KML codec.
* Nothing here persists: a trip import (PlacesService), the Tours import
* (ToursService) and a list import or export (CollectionsService) each take
* what comes back and write their own rows, so the dedup, the transactions and
* the broadcasts stay with the domain that owns them.
*/
@Injectable()
export class PlaceImportService {
constructor(
private readonly gpx: GpxProvider,
private readonly collectionGpx: CollectionGpxProvider,
private readonly kml: KmlProvider,
private readonly googleList: GoogleListProvider,
private readonly googleDirections: GoogleDirectionsProvider,
private readonly naverList: NaverListProvider,
) {}
// ── GPX ────────────────────────────────────────────────────────────────────
/** The places a trip GPX import would create, nothing written. */
readGpx(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
return this.gpx.read(fileBuffer, opts);
}
/** A trip as a GPX document, or null when the selection yields nothing. */
writeTripGpx(input: GpxExportInput, opts: GpxExportOptions = {}): string | null {
return this.gpx.write(input, opts);
}
tripGpxFilename(tripTitle: string): string {
return this.gpx.filename(tripTitle);
}
/** A list file as GPX (#2301). */
writeCollectionGpx(file: ExportedCollectionFile): CollectionGpxExport {
return this.collectionGpx.write(file);
}
/** A GPX document as the list file it amounts to. Throws CollectionGpxError on a refusal. */
readCollectionGpx(source: string, fileName?: string): CollectionGpxReadResult {
return this.collectionGpx.read(source, fileName);
}
// ── KML / KMZ ──────────────────────────────────────────────────────────────
/** A .kml or .kmz upload, by its file name. */
readMapFile(fileBuffer: Buffer, filename: string): Promise<KmlDocumentRead> {
return this.kml.readMapFile(fileBuffer, filename);
}
readKmz(kmzBuffer: Buffer): Promise<KmlDocumentRead> {
return this.kml.readKmz(kmzBuffer);
}
readKml(fileBuffer: Buffer): KmlDocumentRead {
return this.kml.readKml(fileBuffer);
}
// ── Shared lists ───────────────────────────────────────────────────────────
/** A Google Maps list, or the directions link a short link turned out to be. */
readGoogleList(url: string): Promise<ListRead<GoogleListPlace> | DirectionsRedirect | ListImportError> {
return this.googleList.read(url);
}
readNaverList(url: string): Promise<ListRead<NaverListPlace> | ListImportError> {
return this.naverList.read(url);
}
/**
* The stops of a shared directions link, placed on the map.
*
* `geocode` places a stop the link names without coordinates. It is handed
* in rather than injected so this layer stays below the maps domain: the
* importer passes MapsService.geocodeQuery.
*
* A stop the link spells out in coordinates is taken as it stands; one that
* is only a name is geocoded, one request each. A name nobody can place is
* left out rather than failing the import, because a route of six stops with
* five findable is five stops more than the traveller had.
*/
async readGoogleDirections(url: string, geocode: Geocoder): Promise<DirectionsRead | ListImportError> {
const waypoints = await this.googleDirections.read(url);
if (!Array.isArray(waypoints)) return waypoints;
const places: GoogleListPlace[] = [];
let unplaceable = 0;
for (const wp of waypoints) {
if (wp.lat !== null && wp.lng !== null) {
// A stop written as coordinates has no name of its own, and the coordinates are a
// better label than "Stop 3": they are what the traveller can look up.
places.push({
name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`,
lat: wp.lat,
lng: wp.lng,
notes: null,
googleFtid: null,
});
continue;
}
if (!wp.name) continue;
try {
// Through geocodeQuery, which asks the TREK index first and only falls
// through to Nominatim for what it does not know — and does so on the
// BACKGROUND lane. That matters here more than anywhere: this loop runs
// up to thirty times in one request, each Nominatim call taking the next
// slot on a 1.1 s process-wide throttle, so on the interactive lane one
// pasted link made everybody else's place search queue behind it for
// half a minute. An index hit costs no slot at all.
const hit = await geocode(wp.name);
// The name from the link, not the one the geocoder answers with: somebody who
// typed a nickname into Google should not find a street address on their trip.
if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null });
else unplaceable++;
} catch {
// A geocoder that is down or rate-limited costs this one stop, not the import.
unplaceable++;
}
}
if (places.length < 2) {
return { error: 'None of the stops in that link could be placed on the map.', status: 400 };
}
return { places, unplaceable };
}
}
@@ -0,0 +1,104 @@
/**
* What the place-import layer hands its callers: places read out of a file or
* a shared list, not yet persisted. Persisting them (dedup against the trip,
* insert, track colours, enrichment) stays with the domain that owns the rows,
* PlacesService for a trip and CollectionsService for a list.
*/
import type { CollectionGpxProblem } from '@trek/shared';
/** One place parsed out of a GPX file, not yet persisted. */
export interface PreparedGpxPlace {
name: string;
lat: number;
lng: number;
description: string | null;
routeGeometry?: string;
}
export interface GpxImportOptions {
importWaypoints?: boolean;
importRoutes?: boolean;
importTracks?: boolean;
/** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */
defaultName?: string;
}
export interface KmlImportOptions {
importPoints?: boolean;
importPaths?: boolean;
}
export interface KmlImportSummary {
totalPlacemarks: number;
createdCount: number;
skippedCount: number;
warnings: string[];
errors: string[];
}
export interface ParsedKmlPlacemark {
name: string | null;
description: string | null;
lat: number | null;
lng: number | null;
folderName: string | null;
routeGeometry: string | null;
}
/** A KML document read: every placemark, and the summary the import fills in. */
export interface KmlDocumentRead {
placemarks: ParsedKmlPlacemark[];
summary: KmlImportSummary;
}
/** An import that could not happen, with the status and the exact message the route answers. */
export interface ListImportError {
error: string;
status: number;
}
/** A place out of a Google list or a directions link. */
export interface GoogleListPlace {
name: string;
lat: number;
lng: number;
notes: string | null;
googleFtid: string | null;
}
/** A place out of a Naver list. */
export interface NaverListPlace {
name: string;
lat: number;
lng: number;
notes: string | null;
address: string | null;
}
/** A shared list read: its name and the places that carried coordinates. */
export interface ListRead<P> {
listName: string;
places: P[];
}
/** A Google link that turned out to be a route once its redirect was followed. */
export interface DirectionsRedirect {
directions: string;
}
/** The stops of a directions link, placed on the map; `unplaceable` counts the ones nobody could find. */
export interface DirectionsRead {
places: GoogleListPlace[];
unplaceable: number;
}
/** Why a GPX document was refused for a list, carried to the controller as the response's `code`. */
export class CollectionGpxError extends Error {
constructor(
readonly code: CollectionGpxProblem,
message: string,
) {
super(message);
this.name = 'CollectionGpxError';
}
}
@@ -1,4 +1,4 @@
import { XMLParser } from 'fast-xml-parser';
import { Injectable } from '@nestjs/common';
import type { ZodType } from 'zod';
import {
COLLECTION_FILE_FORMAT,
@@ -16,11 +16,13 @@ import {
type CollectionFileLabel,
type CollectionFilePlace,
type CollectionGpxExport,
type CollectionGpxProblem,
type CollectionGpxReadResult,
type CollectionLink,
} from '@trek/shared';
import { coord, gpxBuilder } from '../places/gpx-export.helpers';
import { coord, gpxTextParser, writeGpx } from '../gpx.codec';
import { CollectionGpxError } from '../place-import.types';
export { CollectionGpxError } from '../place-import.types';
/**
* A list as GPX and back (#2301).
@@ -32,16 +34,15 @@ import { coord, gpxBuilder } from '../places/gpx-export.helpers';
* fields of a place map onto a waypoint, and that table lives in the contract
* (`COLLECTION_GPX_PLACE_FIELDS`).
*
* Same library as the trip importer and exporter in places/, with a reader of
* its own: the trip parser turns `<name>007</name>` into the number 7 and does
* not decode character references, which is fine for track geometry and wrong
* for the names and notes a list is made of.
* Same codec as the trip importer and exporter (gpx.codec), in its `text`
* read mode: the trip parser turns `<name>007</name>` into the number 7 and
* does not decode character references, which is fine for track geometry and
* wrong for the names and notes a list is made of.
*/
/** A list file as the exporter builds it, its places already in the file's shape. */
export type ExportedCollectionFile = Omit<CollectionFile, 'places'> & { places: CollectionFilePlace[] };
const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1';
/** The prefix this writer binds; the reader looks the namespace up instead. */
const TREK = 'trek';
/** OsmAnd writes its favourites with the address in its own namespace. */
@@ -148,19 +149,7 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx
wpt.push(waypoint(place, lat, lng));
}
const gpx: string = gpxBuilder.build({
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
gpx: {
'@_version': '1.1',
'@_creator': 'TREK',
'@_xmlns': GPX_NAMESPACE,
[`@_xmlns:${TREK}`]: COLLECTION_GPX_NAMESPACE,
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`,
metadata: metadata(file),
...(wpt.length ? { wpt } : {}),
},
});
const gpx = writeGpx({ namespaces: { [TREK]: COLLECTION_GPX_NAMESPACE }, metadata: metadata(file), wpt });
return { name: file.name, gpx, waypoints: wpt.length, omitted };
}
@@ -168,17 +157,6 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx
// Reading
// ---------------------------------------------------------------------------
/** Why a document was refused, carried to the controller as the response's `code`. */
export class CollectionGpxError extends Error {
constructor(
readonly code: CollectionGpxProblem,
message: string,
) {
super(message);
this.name = 'CollectionGpxError';
}
}
/**
* The most elements a document may open before it is parsed at all.
*
@@ -189,27 +167,6 @@ export class CollectionGpxError extends Error {
*/
export const MAX_GPX_ELEMENTS = 100_000;
/** Elements that may repeat, so they always parse as arrays. Matched on the local name. */
const REPEATED = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']);
const localName = (name: string): string => name.slice(name.indexOf(':') + 1);
/**
* Text stays text (`parseTagValue: false`), so a waypoint called 007 keeps its
* zeros. `htmlEntities` is on for the numeric character references it also
* unlocks (`&#233;`), which this library otherwise leaves undecoded. Entities
* declared in a DOCTYPE never reach it: a document carrying one is refused
* before parsing.
*/
const reader = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
parseTagValue: false,
parseAttributeValue: false,
htmlEntities: true,
isArray: (name, _path, _leaf, isAttribute) => !isAttribute && REPEATED.has(localName(name)),
});
type XmlNode = Record<string, unknown>;
function isNode(value: unknown): value is XmlNode {
@@ -410,7 +367,7 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti
const xml = refuseBeforeParsing(source);
let doc: unknown;
try {
doc = reader.parse(xml, true);
doc = gpxTextParser.parse(xml, true);
} catch {
throw new CollectionGpxError('unreadable', 'That GPX file is not well-formed XML');
}
@@ -481,3 +438,15 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti
if (!file) throw new CollectionGpxError('unreadable', 'That GPX file could not be read as a list');
return { file, skipped, track_points: trackPoints };
}
/** The GPX source for lists: the list file out as GPX, a GPX file in as a list file. */
@Injectable()
export class CollectionGpxProvider {
write(file: ExportedCollectionFile): CollectionGpxExport {
return collectionFileToGpx(file);
}
read(source: string, fileName?: string): CollectionGpxReadResult {
return gpxToCollectionFile(source, fileName);
}
}
@@ -0,0 +1,59 @@
/**
* The stops of a shared Google Maps directions link, read off the link itself.
*
* The other half of the request the list import answers: people share a drive
* far more often than they share a list. No API key is involved and no call is
* made to Google for the link itself — the stops are in the URL, which is the
* whole reason this is possible at all. A short link is followed hop by hop
* through the SSRF guard first, and the host that counts is the one the link
* lands on.
*
* A stop the link spells out in coordinates is taken as it stands; one that is
* only a name comes back without coordinates for the caller to geocode. This
* provider asks no geocoder: choosing which source answers is MapsService's
* job, reached through PlaceImportService.
*/
import { Injectable } from '@nestjs/common';
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../../common/google-maps-hosts';
import { parseDirectionsUrl, type DirWaypoint } from '../directions-url.helpers';
import type { ListImportError } from '../place-import.types';
@Injectable()
export class GoogleDirectionsProvider {
/** The stops in driving order (two at least), or the refusal the route answers. */
async read(url: string): Promise<DirWaypoint[] | ListImportError> {
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
let parsed: URL;
try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
// Short links are resolved hop by hop through the guard, exactly as the list import
// does it: a maps.app.goo.gl that 302s to an internal address is still blocked.
let resolvedUrl = url;
if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
// Checked after resolving, not before: the host that counts is the one the link lands
// on, and `/maps/dir/` is a path anybody could serve.
let host = '';
try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ }
if (!isGoogleMapsHost(host)) {
return { error: 'That link is not a Google Maps link.', status: 400 };
}
const waypoints = parseDirectionsUrl(resolvedUrl);
if (waypoints.length < 2) {
return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 };
}
return waypoints;
}
}
@@ -0,0 +1,177 @@
/**
* A shared Google Maps list, read from Google's own list endpoint.
*
* The link is user input, so it is checked by the SSRF guard first and a short
* link is followed hop by hop through it. A short link that lands on a route
* (the Share sheet on a phone produces exactly that) is handed back as a
* directions link for the directions source; everything else must carry a list
* id. The answer is capped before and after it is read, because its size is
* attacker-influenced through the list id. Every refusal is the exact message
* and status the import route has always answered.
*/
import { Injectable } from '@nestjs/common';
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
import { isDirectionsUrl } from '../directions-url.helpers';
import type { DirectionsRedirect, GoogleListPlace, ListImportError, ListRead } from '../place-import.types';
/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */
export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB
/** The browser user agent the list endpoints answer. */
export const LIST_USER_AGENT =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
export function googleMapsHexId(value: unknown): string | null {
if (typeof value !== 'string' && typeof value !== 'number') return null;
const raw = String(value).trim();
if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase();
if (!/^-?\d+$/.test(raw)) return null;
try {
const parsed = BigInt(raw);
const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed;
return `0x${unsigned.toString(16)}`;
} catch {
return null;
}
}
export function googleMapsFeatureIdFromItem(item: unknown): string | null {
if (!Array.isArray(item)) return null;
const candidates = [
Array.isArray(item[1]) ? item[1][6] : null,
Array.isArray(item[7]) ? item[7][1] : null,
];
for (const ids of candidates) {
if (!Array.isArray(ids) || ids.length < 2) continue;
const first = googleMapsHexId(ids[0]);
const second = googleMapsHexId(ids[1]);
if (first && second) return `${first}:${second}`;
}
return null;
}
@Injectable()
export class GoogleListProvider {
async read(url: string): Promise<ListRead<GoogleListPlace> | DirectionsRedirect | ListImportError> {
let listId: string | null = null;
let resolvedUrl = url;
// SSRF guard: validate user-supplied URL before fetching
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
// Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are
// followed manually so every hop is re-checked against the SSRF guard — a
// short link that 302s to an internal IP is blocked even though the initial
// host is public.
if (url.includes('goo.gl') || url.includes('maps.app')) {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
// A route, once the redirect is followed. The dispatch upstream decides on
// the raw URL, and a short link's path is `/<code>` — it matches nothing, so
// every route shared from the Google Maps app arrived here and was answered
// with "could not extract list ID", which is the complaint the directions
// import was written to remove. The Share sheet on a phone produces exactly
// this shape, and the box says a directions link works.
//
// Handed on with the resolved URL, so the hop is not made twice.
if (isDirectionsUrl(resolvedUrl)) return { directions: resolvedUrl };
// Pattern: /placelists/list/{ID}
const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/);
if (plMatch) listId = plMatch[1];
// Pattern: !2s{ID} in data URL params
if (!listId) {
const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/);
if (dataMatch) listId = dataMatch[1];
}
if (!listId) {
// A single-place share link (…/maps/place/…) carries no list id — point the user at
// the place search box instead of a cryptic "could not extract list ID" (#1304).
if (resolvedUrl.includes('/maps/place/')) {
return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 };
}
return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 };
}
// Fetch list data from Google Maps internal API
const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`;
const apiRes = await fetch(apiUrl, {
headers: { 'User-Agent': LIST_USER_AGENT },
signal: AbortSignal.timeout(15000),
});
if (!apiRes.ok) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
// Cap the declared body before reading it (transit.service precedent): the
// response is attacker-influenced via the list id, and buffering it whole
// used to be unbounded.
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
if (declared > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
const rawText = await apiRes.text();
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
const jsonStr = rawText.substring(rawText.indexOf('\n') + 1);
// The provider hands back a JS-prefixed array; a malformed body is a
// provider problem, not a crash — surface the same 400 an unreadable
// payload already produced instead of throwing a SyntaxError.
let listData: unknown;
try {
listData = JSON.parse(jsonStr);
} catch {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
if (!Array.isArray(listData)) {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
const meta = listData[0];
if (!meta) {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
const listName = meta[4] || 'Google Maps List';
const items = meta[8];
if (!Array.isArray(items) || items.length === 0) {
return { error: 'List is empty or could not be read', status: 400 };
}
// Parse place data from items
const places: GoogleListPlace[] = [];
for (const item of items) {
const coords = item?.[1]?.[5];
const lat = coords?.[2];
const lng = coords?.[3];
const name = item?.[2];
const note = item?.[3] || null;
if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) {
places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) });
}
}
if (places.length === 0) {
return { error: 'No places with coordinates found in list', status: 400 };
}
return { listName, places };
}
}
@@ -1,8 +1,11 @@
import { XMLBuilder } from 'fast-xml-parser';
import { Injectable } from '@nestjs/common';
import { coord, gpxGeometryParser, writeGpx } from '../gpx.codec';
import type { GpxImportOptions, PreparedGpxPlace } from '../place-import.types';
/**
* GPX writer, the mirror of the importer in places.helpers.ts. Same library, the
* builder half rather than the parser half, so nothing new is pulled in.
* A trip as GPX and back: the reader that turns a GPX file into the places an
* import would create, and the writer that is its mirror. Both go through
* gpx.codec (the geometry read mode, the shared document writer).
*
* The import decides the shape here: a `<wpt>` becomes a place with coordinates, a
* `<rte>` or `<trk>` becomes a place carrying `route_geometry`, so writing back is
@@ -46,12 +49,6 @@ export interface GpxExportOptions {
type Pt = { lat: number; lng: number; ele: number | null };
/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer
* device resolves and keeps the file from carrying float noise. */
export function coord(n: number): string {
return Number(n.toFixed(7)).toString();
}
function parseGeometry(raw: string | null): Pt[] {
if (!raw) return [];
let parsed: unknown;
@@ -81,15 +78,6 @@ function describe(place: GpxExportPlace): string | undefined {
return parts.length ? parts.join(', ') : undefined;
}
/** Shared with the list writer in collections/, which writes the same dialect. */
export const gpxBuilder = new XMLBuilder({
ignoreAttributes: false,
attributeNamePrefix: '@_',
format: true,
indentBy: ' ',
suppressEmptyNode: true,
});
/**
* Build a GPX 1.1 document. Returns null when the selection produced nothing at all,
* so the caller can answer 404 rather than hand over an empty file that silently
@@ -154,22 +142,7 @@ export function buildGpx(input: GpxExportInput, opts: GpxExportOptions = {}): st
if (wpt.length === 0 && trk.length === 0 && rte.length === 0) return null;
const doc = {
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
gpx: {
'@_version': '1.1',
'@_creator': 'TREK',
'@_xmlns': 'http://www.topografix.com/GPX/1/1',
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'@_xsi:schemaLocation': 'http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd',
metadata: { name: input.tripTitle },
...(wpt.length ? { wpt } : {}),
...(rte.length ? { rte } : {}),
...(trk.length ? { trk } : {}),
},
};
return gpxBuilder.build(doc);
return writeGpx({ metadata: { name: input.tripTitle }, wpt, rte, trk });
}
/** Filenames land on the receiving filesystem, so its reserved characters are
@@ -187,3 +160,96 @@ export function gpxFilename(tripTitle: string): string {
// lone surrogate is exactly what URI-encoding chokes on.
return `${[...base].slice(0, 60).join('') || 'trip'}.gpx`;
}
/**
* Parses a GPX file into the places an import would create, without writing
* anything. Waypoints become places, each route and each track becomes one
* place carrying its geometry.
*/
export function prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts;
const parsed = gpxGeometryParser.parse(fileBuffer.toString('utf-8'));
const gpx = parsed?.gpx;
if (!gpx) return [];
const str = (v: unknown) => (v != null ? String(v).trim() : null);
const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; };
// Routes and tracks rarely carry their own <name>. Without one they all fall back to the
// same generic label, so name-based dedup drops every import after the first. Derive a
// base from the source filename (the requested behaviour) and suffix an index so multiple
// geometries from one file stay distinct.
const rawName = str(defaultName);
const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null;
let geoSeq = 0;
const geoName = (explicit: string | null, fallback: string): string => {
if (explicit) return explicit;
geoSeq++;
const base = baseName || fallback;
return geoSeq === 1 ? base : `${base} ${geoSeq}`;
};
const waypoints: PreparedGpxPlace[] = [];
// 1) Parse <wpt> elements (named waypoints / POIs)
if (importWaypoints) {
for (const wpt of gpx.wpt ?? []) {
const lat = num(wpt['@_lat']);
const lng = num(wpt['@_lon']);
if (lat === null || lng === null) continue;
waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) });
}
}
// 2) Parse <rte> routes as polyline-places (one place per route with route_geometry)
if (importRoutes) {
for (const rte of gpx.rte ?? []) {
const pts = (rte.rtept ?? [])
.map((pt: Record<string, unknown>) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) }))
.filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>;
if (pts.length === 0) continue;
const hasAllEle = pts.every(p => p.ele !== null);
const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) });
}
}
// 3) Extract full track geometry from <trk>
if (importTracks) {
for (const trk of gpx.trk ?? []) {
const trackPoints: { lat: number; lng: number; ele: number | null }[] = [];
for (const seg of trk.trkseg ?? []) {
for (const pt of seg.trkpt ?? []) {
const lat = num(pt['@_lat']);
const lng = num(pt['@_lon']);
if (lat === null || lng === null) continue;
trackPoints.push({ lat, lng, ele: num(pt.ele) });
}
}
if (trackPoints.length === 0) continue;
const start = trackPoints[0];
const hasAllEle = trackPoints.every(p => p.ele !== null);
const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) });
}
}
return waypoints;
}
/** The GPX source: a trip file in, a trip file out. */
@Injectable()
export class GpxProvider {
read(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
return prepareGpxRows(fileBuffer, opts);
}
write(input: GpxExportInput, opts: GpxExportOptions = {}): string | null {
return buildGpx(input, opts);
}
filename(tripTitle: string): string {
return gpxFilename(tripTitle);
}
}
@@ -0,0 +1,26 @@
/**
* A KML or KMZ map file (Google My Maps, Google Earth), read through kml.codec
* into its placemarks. The file type is the upload's extension, as it has
* always been; anything else is refused with the route's own message.
*/
import { Injectable } from '@nestjs/common';
import { readKmlDocument, unpackKmzToKml } from '../kml.codec';
import type { KmlDocumentRead } from '../place-import.types';
@Injectable()
export class KmlProvider {
async readMapFile(fileBuffer: Buffer, filename: string): Promise<KmlDocumentRead> {
const ext = filename.toLowerCase().split('.').pop();
if (ext === 'kmz') return this.readKmz(fileBuffer);
if (ext === 'kml') return this.readKml(fileBuffer);
throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`);
}
async readKmz(kmzBuffer: Buffer): Promise<KmlDocumentRead> {
return this.readKml(await unpackKmzToKml(kmzBuffer));
}
readKml(fileBuffer: Buffer): KmlDocumentRead {
return readKmlDocument(fileBuffer);
}
}
@@ -0,0 +1,137 @@
/**
* A shared Naver Maps list (a bookmark folder), read page by page from Naver's
* share endpoint.
*
* The link is checked by the SSRF guard and a naver.me short link is followed
* hop by hop through it. Every page is capped before and after it is read: the
* folder id is attacker-influenced, and the pager buffers a fresh body on every
* iteration. Every refusal is the exact message and status the import route
* has always answered.
*/
import { Injectable } from '@nestjs/common';
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
import { LIST_USER_AGENT, MAX_LIST_RESPONSE_BYTES } from './google-list.provider';
import type { ListImportError, ListRead, NaverListPlace } from '../place-import.types';
/** Bookmarks asked for per page. */
const PAGE_SIZE = 20;
type NaverPage = {
folder?: { bookmarkCount?: number; name?: string };
bookmarkList?: Record<string, unknown>[];
};
@Injectable()
export class NaverListProvider {
async read(url: string): Promise<ListRead<NaverListPlace> | ListImportError> {
let resolvedUrl = url;
const limit = PAGE_SIZE;
// SSRF guard: validate user-supplied URL before fetching
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
// Resolve naver.me short links to the canonical map.naver.com folder URL.
// Redirects are followed manually so each hop is re-validated against the
// SSRF guard (a short link could otherwise 302 to an internal address).
let parsedUrl: URL;
try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
if (parsedUrl.hostname === 'naver.me') {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i);
const folderId = folderMatch?.[1] || null;
if (!folderId) {
return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 };
}
const fetchPage = async (start: number) => {
const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`;
const apiRes = await fetch(apiUrl, {
headers: {
Accept: 'application/json',
'User-Agent': LIST_USER_AGENT,
},
signal: AbortSignal.timeout(15000),
});
if (!apiRes.ok) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
// Same cap as the Google import: the URL is attacker-influenced via the
// folder id, and this pager buffers a fresh body on every iteration, so an
// uncapped read is worse here than there. The declared length is checked
// before the read; the post-read check covers a chunked response that
// carries no content-length at all.
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
if (declared > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
try {
const rawText = await apiRes.text();
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
const data = JSON.parse(rawText) as NaverPage;
return { data } as const;
} catch {
return { error: 'Invalid list data received from Naver Maps', status: 400 } as const;
}
};
const firstPage = await fetchPage(0);
if ('error' in firstPage) {
return { error: firstPage.error, status: firstPage.status };
}
const listName = firstPage.data.folder?.name || 'Naver Maps List';
const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number'
? firstPage.data.folder.bookmarkCount
: (firstPage.data.bookmarkList?.length || 0);
const allItems: Record<string, unknown>[] = [...(firstPage.data.bookmarkList || [])];
for (let start = limit; start < totalCount; start += limit) {
const page = await fetchPage(start);
if ('error' in page) {
return { error: page.error, status: page.status };
}
const pageItems = page.data.bookmarkList || [];
if (!Array.isArray(pageItems) || pageItems.length === 0) break;
allItems.push(...pageItems);
}
if (allItems.length === 0) {
return { error: 'List is empty or could not be read', status: 400 };
}
const places: NaverListPlace[] = [];
for (const item of allItems) {
const lat = Number(item?.py);
const lng = Number(item?.px);
const name = typeof item?.name === 'string' && item.name.trim()
? item.name.trim()
: (typeof item?.displayName === 'string' ? item.displayName.trim() : '');
const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null;
const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null;
if (name && Number.isFinite(lat) && Number.isFinite(lng)) {
places.push({ name, lat, lng, notes: note, address });
}
}
if (places.length === 0) {
return { error: 'No places with coordinates found in list', status: 400 };
}
return { listName, places };
}
}
+1 -1
View File
@@ -24,7 +24,7 @@ import { memoryStorage } from 'multer';
import { hexColorSchema, placeImageUrlSchema, placeWebsiteSchema } from '@trek/shared';
import type { User } from '../../types';
import { PlacesService } from './places.service';
import { isDirectionsUrl } from './maps-dir.helpers';
import { isDirectionsUrl } from '../place-import/place-import.service';
import { isUpdateConflict } from '../common/conflictResult';
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
import { CurrentUser } from '../auth-core/current-user.decorator';
+21 -103
View File
@@ -1,5 +1,3 @@
import { XMLParser } from 'fast-xml-parser';
import unzipper from 'unzipper';
import {
externalIdsOf,
normalizePlaceName,
@@ -8,7 +6,7 @@ import {
} from '@trek/shared';
import type { Place } from '../../types';
import type { PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository';
import type { KmlImportSummary } from './kml-import.helpers';
import type { KmlImportSummary } from '../place-import/place-import.types';
import type { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
import { haversineMetres } from '../common/geo';
@@ -17,36 +15,15 @@ import { haversineMetres } from '../common/geo';
* out of the legacy services/placeService.ts when it went DI-native. Same
* split as maps.helpers.ts / transit.helpers.ts / files.constants.ts: nothing
* here touches the DB, so it stays plain exports rather than becoming methods
* on PlacesService — which also keeps the frozen-at-import XML parsers a
* single shared instance and lets the KMZ unpacker be unit-tested on its own.
* on PlacesService. Reading files and shared lists (the XML parsers, the KMZ
* unpacker, the list fetchers) lives in place-import/.
*/
const gpxParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
isArray: (name) => ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'].includes(name),
});
const kmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
removeNSPrefix: true,
isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name),
// Treat <description> as raw text so mixed-content HTML (e.g. <br/>, <i>)
// is returned as a string instead of a parsed object.
stopNodes: ['*.description'],
});
export { gpxParser, kmlParser };
export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB
// Re-exported so the importers that already read these from here keep working,
// while the values themselves live in @trek/shared with the rule that uses them.
export { COORD_DEDUP_TOLERANCE, externalIdsOf } from '@trek/shared';
/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */
export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB
export type { GpxImportOptions, KmlImportOptions, ListImportError } from '../place-import/place-import.types';
/**
* Escape the LIKE metacharacters in a user-supplied search term so `%` and `_`
@@ -88,24 +65,6 @@ export interface ListImportResult {
skipped: number;
}
export interface ListImportError {
error: string;
status: number;
}
export interface GpxImportOptions {
importWaypoints?: boolean;
importRoutes?: boolean;
importTracks?: boolean;
/** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */
defaultName?: string;
}
export interface KmlImportOptions {
importPoints?: boolean;
importPaths?: boolean;
}
// Reclaim a deleted place's cached marker photo if nothing else references it.
// The cache key is the Google place_id, or — for coordinate-only places — the
// pseudo-id embedded in the stored proxy URL (/api/maps/place-photo/{id}/bytes).
@@ -173,41 +132,6 @@ export function trackInsertedInDedupSet(place: PlaceMatchCandidate, dedup: Dedup
}
}
// ---------------------------------------------------------------------------
// Google Maps list id parsing
// ---------------------------------------------------------------------------
export function googleMapsHexId(value: unknown): string | null {
if (typeof value !== 'string' && typeof value !== 'number') return null;
const raw = String(value).trim();
if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase();
if (!/^-?\d+$/.test(raw)) return null;
try {
const parsed = BigInt(raw);
const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed;
return `0x${unsigned.toString(16)}`;
} catch {
return null;
}
}
export function googleMapsFeatureIdFromItem(item: unknown): string | null {
if (!Array.isArray(item)) return null;
const candidates = [
Array.isArray(item[1]) ? item[1][6] : null,
Array.isArray(item[7]) ? item[7][1] : null,
];
for (const ids of candidates) {
if (!Array.isArray(ids) || ids.length < 2) continue;
const first = googleMapsHexId(ids[0]);
const second = googleMapsHexId(ids[1]);
if (first && second) return `${first}:${second}`;
}
return null;
}
// ---------------------------------------------------------------------------
// Import enrichment (#886) — the pure half of the former
// services/placeEnrichment.ts; the DB/websocket/Maps half is PlacesService.
@@ -283,30 +207,24 @@ export async function mapWithConcurrency<T>(items: T[], limit: number, fn: (item
export const trimOrNull = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null);
// ---------------------------------------------------------------------------
// KMZ unpacking
// KML folder → trip category
// ---------------------------------------------------------------------------
export async function unpackKmzToKml(
kmzBuffer: Buffer,
decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT,
): Promise<Buffer> {
let zip;
try {
zip = await unzipper.Open.buffer(kmzBuffer);
} catch {
throw new Error('Invalid KMZ archive.');
export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map<string, number> {
const lookup = new Map<string, number>();
for (const category of categories) {
const normalizedName = category.name.trim().toLowerCase();
if (!normalizedName) continue;
if (!lookup.has(normalizedName)) {
lookup.set(normalizedName, category.id);
}
}
const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml'));
if (kmlEntries.length === 0) {
throw new Error('KMZ archive does not contain a KML file.');
}
const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0];
if (preferredEntry.uncompressedSize > decompressedSizeLimit) {
throw new Error('KMZ archive exceeds the maximum allowed decompressed size.');
}
return preferredEntry.buffer();
return lookup;
}
export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map<string, number>): number | null {
if (!folderName) return null;
const normalizedFolder = folderName.trim().toLowerCase();
if (!normalizedFolder) return null;
return lookup.get(normalizedFolder) ?? null;
}
+1 -1
View File
@@ -26,7 +26,7 @@ import { UnitOfWork } from '../database/unit-of-work';
import { MapsService } from '../maps/maps.service';
import { PlacesService } from './places.service';
import { isUpdateConflict } from '../common/conflictResult';
import { isDirectionsUrl } from './maps-dir.helpers';
import { isDirectionsUrl } from '../place-import/place-import.service';
import { TripAccessService } from '../trip-membership/trip-access.service';
function parseId(value: string | string[]): number | null {
+3
View File
@@ -24,6 +24,7 @@ import { UnsplashModule } from '../unsplash/unsplash.module';
import { PlacePhotosModule } from '../place-photos/place-photos.module';
import { QueryHelpersModule } from '../query-helpers/query-helpers.module';
import { MapsModule } from '../maps/maps.module';
import { PlaceImportModule } from '../place-import/place-import.module';
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
import { MulterModule } from '@nestjs/platform-express';
import { StorageModule } from '../storage/storage.module';
@@ -55,6 +56,8 @@ import { TripMembershipModule } from '../trip-membership/trip-membership.module'
StorageModule,
// AccommodationsDomainModule: deleting a place takes the nights booked at it with
// it, and the cascade behind a stay belongs to the domain that owns it.
// PlaceImportModule reads the GPX/KML files and shared lists this domain imports.
PlaceImportModule,
McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule, AccommodationsDomainModule,
// Plan 3c Task 4: `PlacesService`'s and `PlacesMcp`'s own `@InjectRepository`
// constructor params (`PlacesRepository`, `TagsRepository`,
+37 -447
View File
@@ -2,30 +2,20 @@ import path from 'node:path';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@mikro-orm/nestjs';
import { resolveCountryCodeSync } from '../atlas/atlas-geo';
import { XMLValidator } from 'fast-xml-parser';
import { TRACK_COLORS, placeMatchStrategies, type PlaceMatchCandidate } from '@trek/shared';
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
import { RealtimeService } from '../realtime/realtime.service';
import type { TripAccess } from '../../db/repositories/Trips.repository';
import { UnitOfWork } from '../database/unit-of-work';
import { PermissionsService } from '../permissions/permissions.service';
import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service';
import { isDirectionsUrl, parseDirectionsUrl } from './maps-dir.helpers';
import { MapsService } from '../maps/maps.service';
import { PlaceImportService } from '../place-import/place-import.service';
import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from '../place-import/place-import.service';
import type { GoogleListPlace, KmlDocumentRead, PreparedGpxPlace } from '../place-import/place-import.types';
import { toRowId } from '../common/row-id';
import type { User } from '../../types';
import { QueryHelpersService } from '../query-helpers/query-helpers.service';
import { ratingAggregate } from '../common/rowShape';
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../utils/ssrfGuard';
import {
buildCategoryNameLookup,
createKmlImportSummary,
decodeUtf8WithWarning,
extractKmlPlacemarkNodes,
parsePlacemarkNode,
resolveCategoryIdForFolder,
} from './kml-import.helpers';
import { buildGpx, gpxFilename } from './gpx-export.helpers';
import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from './gpx-export.helpers';
import { UnsplashService } from '../unsplash/unsplash.service';
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
import { type UpdateConflict, isUpdateConflict } from '../common/conflictResult';
@@ -64,20 +54,17 @@ const noCancelledStays = (): CancelledStays => ({ reservationIds: [], budgetItem
import {
ENRICH_CONCURRENCY,
ADDRESS_BACKFILL_MAX_PLACES,
buildCategoryNameLookup,
escapeLikePattern,
MAX_LIST_RESPONSE_BYTES,
googleMapsFeatureIdFromItem,
gpxParser,
externalIdsOf,
isPlaceDuplicate,
kmlParser,
mapWithConcurrency,
pickEnrichmentMatch,
reclaimPhotoCache,
SEARCH_BIAS_RADIUS_METERS,
resolveCategoryIdForFolder,
trackInsertedInDedupSet,
trimOrNull,
unpackKmzToKml,
type DedupSet,
type EnrichablePlace,
type GpxImportOptions,
@@ -93,14 +80,7 @@ type Trip = TripAccess;
type ImportedPlace = { id: number; route_geometry?: string | null; route_color?: string | null };
/** One place parsed out of a GPX file, not yet persisted. */
export interface PreparedGpxPlace {
name: string;
lat: number;
lng: number;
description: string | null;
routeGeometry?: string;
}
export type { PreparedGpxPlace } from '../place-import/place-import.types';
/** Fields accepted when creating a place. */
export interface PlaceCreateInput {
@@ -187,6 +167,8 @@ export class PlacesService {
@InjectRepository(BudgetItems) private readonly budgetItemsRepo: BudgetItemsRepository,
// Plan 3h Task 6 (survivors) — additive, SV-PI2's `reclaimPlaceImage` only.
@InjectRepository(CollectionPlaces) private readonly collectionPlacesRepo: CollectionPlacesRepository,
// Reading files and shared lists; this service only persists what it reads.
private readonly placeImport: PlaceImportService,
) {}
/**
@@ -903,95 +885,18 @@ export class PlacesService {
day.points.push({ name: stop.name, lat: stop.lat, lng: stop.lng });
}
const gpx = buildGpx({ tripTitle: title, places, days: [...days.values()] }, opts);
return gpx ? { gpx, filename: gpxFilename(title) } : null;
const gpx = this.placeImport.writeTripGpx({ tripTitle: title, places, days: [...days.values()] }, opts);
return gpx ? { gpx, filename: this.placeImport.tripGpxFilename(title) } : null;
}
private async importGpxRows(tripId: string, fileBuffer: Buffer, opts: GpxImportOptions = {}): Promise<GpxImportResult | null> {
const rows = this.prepareGpxRows(fileBuffer, opts);
const rows = this.placeImport.readGpx(fileBuffer, opts);
if (!rows.length) return null;
return await this.uow.transactional(() => this.persistGpxRows(tripId, rows));
}
/**
* Parses a GPX file into the places an import would create, without writing
* anything. The Tours import reuses it to build its own selection before
* handing the rows to {@link importPreparedGpx}.
*/
prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts;
const parsed = gpxParser.parse(fileBuffer.toString('utf-8'));
const gpx = parsed?.gpx;
if (!gpx) return [];
const str = (v: unknown) => (v != null ? String(v).trim() : null);
const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; };
// Routes and tracks rarely carry their own <name>. Without one they all fall back to the
// same generic label, so name-based dedup drops every import after the first. Derive a
// base from the source filename (the requested behaviour) and suffix an index so multiple
// geometries from one file stay distinct.
const rawName = str(defaultName);
const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null;
let geoSeq = 0;
const geoName = (explicit: string | null, fallback: string): string => {
if (explicit) return explicit;
geoSeq++;
const base = baseName || fallback;
return geoSeq === 1 ? base : `${base} ${geoSeq}`;
};
const waypoints: PreparedGpxPlace[] = [];
// 1) Parse <wpt> elements (named waypoints / POIs)
if (importWaypoints) {
for (const wpt of gpx.wpt ?? []) {
const lat = num(wpt['@_lat']);
const lng = num(wpt['@_lon']);
if (lat === null || lng === null) continue;
waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) });
}
}
// 2) Parse <rte> routes as polyline-places (one place per route with route_geometry)
if (importRoutes) {
for (const rte of gpx.rte ?? []) {
const pts = (rte.rtept ?? [])
.map((pt: Record<string, unknown>) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) }))
.filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>;
if (pts.length === 0) continue;
const hasAllEle = pts.every(p => p.ele !== null);
const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) });
}
}
// 3) Extract full track geometry from <trk>
if (importTracks) {
for (const trk of gpx.trk ?? []) {
const trackPoints: { lat: number; lng: number; ele: number | null }[] = [];
for (const seg of trk.trkseg ?? []) {
for (const pt of seg.trkpt ?? []) {
const lat = num(pt['@_lat']);
const lng = num(pt['@_lon']);
if (lat === null || lng === null) continue;
trackPoints.push({ lat, lng, ele: num(pt.ele) });
}
}
if (trackPoints.length === 0) continue;
const start = trackPoints[0];
const hasAllEle = trackPoints.every(p => p.ele !== null);
const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) });
}
}
return waypoints;
}
/**
* Persists rows from {@link prepareGpxRows} and colours their tracks, both in
* Persists rows PlaceImportService.readGpx prepared and colours their tracks, both in
* one transaction. Called from inside the Tours import's own transaction,
* where this becomes a savepoint, so the places and their tour facets
* commit or roll back together.
@@ -1080,39 +985,21 @@ export class PlacesService {
}
private async importMapFileRows(tripId: string, fileBuffer: Buffer, filename: string, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
const ext = filename.toLowerCase().split('.').pop();
if (ext === 'kmz') return this.importKmzPlaces(tripId, fileBuffer, opts);
if (ext === 'kml') return this.importKmlPlaces(tripId, fileBuffer, opts);
throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`);
return this.persistKmlPlaces(tripId, await this.placeImport.readMapFile(fileBuffer, filename), opts);
}
async importKmzPlaces(tripId: string, kmzBuffer: Buffer, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
const kmlBuffer = await unpackKmzToKml(kmzBuffer);
return this.importKmlPlaces(tripId, kmlBuffer, opts);
return this.persistKmlPlaces(tripId, await this.placeImport.readKmz(kmzBuffer), opts);
}
async importKmlPlaces(tripId: string, fileBuffer: Buffer, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
return this.persistKmlPlaces(tripId, this.placeImport.readKml(fileBuffer), opts);
}
/** The placemarks of a read KML document as places, skipping duplicates. */
private async persistKmlPlaces(tripId: string, read: KmlDocumentRead, opts: KmlImportOptions): Promise<PlaceImportResult> {
const { importPoints = true, importPaths = true } = opts;
const decoded = decodeUtf8WithWarning(fileBuffer);
const validationResult = XMLValidator.validate(decoded.text);
if (validationResult !== true) {
throw new Error('Malformed KML: invalid XML structure');
}
const parsed = kmlParser.parse(decoded.text);
const kmlRoot = parsed?.kml ?? parsed;
if (!kmlRoot || typeof kmlRoot !== 'object') {
throw new Error('Malformed KML: could not parse XML');
}
const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot);
const summary = createKmlImportSummary(placemarkNodes.length);
if (decoded.warning) {
summary.warnings.push(decoded.warning);
}
const { placemarks, summary } = read;
// PL33 — `CategoriesRepository.listIdName` (Plan 3a's repository).
const categories = await this.categoriesRepo.listIdName();
@@ -1126,8 +1013,7 @@ export class PlacesService {
await this.uow.transactional(async () => {
let fallbackIndex = 1;
for (const node of placemarkNodes) {
const parsedPlacemark = parsePlacemarkNode(node);
for (const parsedPlacemark of placemarks) {
const isPath = parsedPlacemark.routeGeometry !== null;
// Unsupported geometry type (polygon, multi-geometry, no geometry, etc.)
@@ -1259,124 +1145,12 @@ export class PlacesService {
// -------------------------------------------------------------------------
async importGoogleList(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
let listId: string | null = null;
let resolvedUrl = url;
// SSRF guard: validate user-supplied URL before fetching
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
// Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are
// followed manually so every hop is re-checked against the SSRF guard — a
// short link that 302s to an internal IP is blocked even though the initial
// host is public.
if (url.includes('goo.gl') || url.includes('maps.app')) {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
// A route, once the redirect is followed. The dispatch upstream decides on
// the raw URL, and a short link's path is `/<code>` — it matches nothing, so
// every route shared from the Google Maps app arrived here and was answered
// with "could not extract list ID", which is the complaint the directions
// import was written to remove. The Share sheet on a phone produces exactly
// this shape, and the box says a directions link works.
//
// Handed on with the resolved URL, so the hop is not made twice.
if (isDirectionsUrl(resolvedUrl)) {
return this.importGoogleDirections(tripId, resolvedUrl, opts);
}
// Pattern: /placelists/list/{ID}
const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/);
if (plMatch) listId = plMatch[1];
// Pattern: !2s{ID} in data URL params
if (!listId) {
const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/);
if (dataMatch) listId = dataMatch[1];
}
if (!listId) {
// A single-place share link (…/maps/place/…) carries no list id — point the user at
// the place search box instead of a cryptic "could not extract list ID" (#1304).
if (resolvedUrl.includes('/maps/place/')) {
return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 };
}
return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 };
}
// Fetch list data from Google Maps internal API
const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`;
const apiRes = await fetch(apiUrl, {
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' },
signal: AbortSignal.timeout(15000),
});
if (!apiRes.ok) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
// Cap the declared body before reading it (transit.service precedent): the
// response is attacker-influenced via the list id, and buffering it whole
// used to be unbounded.
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
if (declared > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
const rawText = await apiRes.text();
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Google Maps', status: 502 };
}
const jsonStr = rawText.substring(rawText.indexOf('\n') + 1);
// The provider hands back a JS-prefixed array; a malformed body is a
// provider problem, not a crash — surface the same 400 an unreadable
// payload already produced instead of throwing a SyntaxError.
let listData: unknown;
try {
listData = JSON.parse(jsonStr);
} catch {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
if (!Array.isArray(listData)) {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
const meta = listData[0];
if (!meta) {
return { error: 'Invalid list data received from Google Maps', status: 400 };
}
const listName = meta[4] || 'Google Maps List';
const items = meta[8];
if (!Array.isArray(items) || items.length === 0) {
return { error: 'List is empty or could not be read', status: 400 };
}
// Parse place data from items
const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = [];
for (const item of items) {
const coords = item?.[1]?.[5];
const lat = coords?.[2];
const lng = coords?.[3];
const name = item?.[2];
const note = item?.[3] || null;
if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) {
places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) });
}
}
if (places.length === 0) {
return { error: 'No places with coordinates found in list', status: 400 };
}
const read = await this.placeImport.readGoogleList(url);
if ('error' in read) return read;
// A short link that lands on a route is a directions link: imported as one,
// with the resolved URL, so the hop is not made twice.
if ('directions' in read) return this.importGoogleDirections(tripId, read.directions, opts);
const { listName, places } = read;
const { created, skipped } = await this.storeGooglePlaces(tripId, places);
@@ -1396,7 +1170,7 @@ export class PlacesService {
*/
private async storeGooglePlaces(
tripId: string,
places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[],
places: GoogleListPlace[],
): Promise<{ created: PlaceWithTags[]; skipped: number }> {
// Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the
// caller's `requireTrip` gate already `toRowId`-parsed this `tripId`.
@@ -1469,90 +1243,13 @@ export class PlacesService {
// -------------------------------------------------------------------------
/**
* The stops of a route somebody else planned.
*
* The other half of the request the list import answers: people share a drive far more
* often than they share a list, and until now a pasted `/maps/dir/` link came back as a
* cryptic "could not extract list ID". No API key is involved and no call is made to
* Google for the link itself — the stops are in the URL, which is the whole reason
* this is possible at all.
*
* A stop the link spells out in coordinates is taken as it stands; one that is only a
* name is geocoded, one request each. A name nobody can place is left out rather than
* failing the import, because a route of six stops with five findable is five stops
* more than the traveller had.
* The stops of a route somebody else planned, read and placed by
* PlaceImportService (no API key, the stops are in the link), stored here.
*/
async importGoogleDirections(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
let parsed: URL;
try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
// Short links are resolved hop by hop through the guard, exactly as the list import
// does it: a maps.app.goo.gl that 302s to an internal address is still blocked.
let resolvedUrl = url;
if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
// Checked after resolving, not before: the host that counts is the one the link lands
// on, and `/maps/dir/` is a path anybody could serve.
let host = '';
try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ }
if (!isGoogleMapsHost(host)) {
return { error: 'That link is not a Google Maps link.', status: 400 };
}
const waypoints = parseDirectionsUrl(resolvedUrl);
if (waypoints.length < 2) {
return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 };
}
const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = [];
let unplaceable = 0;
for (const wp of waypoints) {
if (wp.lat !== null && wp.lng !== null) {
// A stop written as coordinates has no name of its own, and the coordinates are a
// better label than "Stop 3": they are what the traveller can look up.
places.push({
name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`,
lat: wp.lat,
lng: wp.lng,
notes: null,
googleFtid: null,
});
continue;
}
if (!wp.name) continue;
try {
// Through geocodeQuery, which asks the TREK index first and only falls
// through to Nominatim for what it does not know — and does so on the
// BACKGROUND lane. That matters here more than anywhere: this loop runs
// up to thirty times in one request, each Nominatim call taking the next
// slot on a 1.1 s process-wide throttle, so on the interactive lane one
// pasted link made everybody else's place search queue behind it for
// half a minute. An index hit costs no slot at all.
const hit = await this.maps.geocodeQuery(wp.name);
// The name from the link, not the one the geocoder answers with: somebody who
// typed a nickname into Google should not find a street address on their trip.
if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null });
else unplaceable++;
} catch {
// A geocoder that is down or rate-limited costs this one stop, not the import.
unplaceable++;
}
}
if (places.length < 2) {
return { error: 'None of the stops in that link could be placed on the map.', status: 400 };
}
const read = await this.placeImport.readGoogleDirections(url, (query) => this.maps.geocodeQuery(query));
if ('error' in read) return read;
const { places, unplaceable } = read;
const { created, skipped } = await this.storeGooglePlaces(tripId, places);
if (created.length) {
@@ -1571,116 +1268,9 @@ export class PlacesService {
// -------------------------------------------------------------------------
async importNaverList(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
let resolvedUrl = url;
const limit = 20;
// SSRF guard: validate user-supplied URL before fetching
const ssrf = await checkSsrf(url);
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
// Resolve naver.me short links to the canonical map.naver.com folder URL.
// Redirects are followed manually so each hop is re-validated against the
// SSRF guard (a short link could otherwise 302 to an internal address).
let parsedUrl: URL;
try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
if (parsedUrl.hostname === 'naver.me') {
try {
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
resolvedUrl = redirectRes.url;
} catch (err) {
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
throw err;
}
}
const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i);
const folderId = folderMatch?.[1] || null;
if (!folderId) {
return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 };
}
const fetchPage = async (start: number) => {
const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`;
const apiRes = await fetch(apiUrl, {
headers: {
Accept: 'application/json',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
signal: AbortSignal.timeout(15000),
});
if (!apiRes.ok) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
// Same cap as the Google import: the URL is attacker-influenced via the
// folder id, and this pager buffers a fresh body on every iteration, so an
// uncapped read is worse here than there. The declared length is checked
// before the read; the post-read check covers a chunked response that
// carries no content-length at all.
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
if (declared > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
try {
const rawText = await apiRes.text();
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
}
const data = JSON.parse(rawText) as {
folder?: { bookmarkCount?: number; name?: string };
bookmarkList?: Record<string, unknown>[];
};
return { data } as const;
} catch {
return { error: 'Invalid list data received from Naver Maps', status: 400 } as const;
}
};
const firstPage = await fetchPage(0);
if ('error' in firstPage) {
return { error: firstPage.error, status: firstPage.status };
}
const listName = firstPage.data.folder?.name || 'Naver Maps List';
const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number'
? firstPage.data.folder.bookmarkCount
: (firstPage.data.bookmarkList?.length || 0);
const allItems: Record<string, unknown>[] = [...(firstPage.data.bookmarkList || [])];
for (let start = limit; start < totalCount; start += limit) {
const page = await fetchPage(start);
if ('error' in page) {
return { error: page.error, status: page.status };
}
const pageItems = page.data.bookmarkList || [];
if (!Array.isArray(pageItems) || pageItems.length === 0) break;
allItems.push(...pageItems);
}
if (allItems.length === 0) {
return { error: 'List is empty or could not be read', status: 400 };
}
const places: { name: string; lat: number; lng: number; notes: string | null; address: string | null }[] = [];
for (const item of allItems) {
const lat = Number(item?.py);
const lng = Number(item?.px);
const name = typeof item?.name === 'string' && item.name.trim()
? item.name.trim()
: (typeof item?.displayName === 'string' ? item.displayName.trim() : '');
const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null;
const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null;
if (name && Number.isFinite(lat) && Number.isFinite(lng)) {
places.push({ name, lat, lng, notes: note, address });
}
}
if (places.length === 0) {
return { error: 'No places with coordinates found in list', status: 400 };
}
const read = await this.placeImport.readNaverList(url);
if ('error' in read) return read;
const { listName, places } = read;
// Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the
// caller's `requireTrip` gate already `toRowId`-parsed this `tripId`.
@@ -2,7 +2,7 @@ import { HttpException, Injectable } from '@nestjs/common';
import { InjectRepository } from '@mikro-orm/nestjs';
import type { GoogleRouteImport, GoogleRoutePreview } from '@trek/shared';
import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service';
import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../places/maps-dir.helpers';
import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../place-import/place-import.service';
import { safeFetchFollow } from '../../utils/ssrfGuard';
import { UnitOfWork } from '../database/unit-of-work';
import { PlacesService } from '../places/places.service';
+4 -2
View File
@@ -8,12 +8,13 @@ import { ToursController } from './tours.controller';
import { ToursImportController } from './tours-import.controller';
import { ToursService } from './tours.service';
import { PlacesModule } from '../places/places.module';
import { PlaceImportModule } from '../place-import/place-import.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { AddonsModule } from '../addons/addons.module';
/**
* Tours domain: an isolated bounded context that imports PlacesModule to reuse
* GPX preparation and persistence through PlacesService. The tour SQL lives in
* Tours domain: an isolated bounded context that reads GPX through PlaceImportModule and
* persists the places through PlacesService (PlacesModule). The tour SQL lives in
* the Tours and TourWaypoints repositories; the owning place's route columns go
* through PlacesRepository, registered here because forFeature only reaches
* this module's own providers.
@@ -21,6 +22,7 @@ import { AddonsModule } from '../addons/addons.module';
@Module({
imports: [
PlacesModule,
PlaceImportModule,
PermissionsModule,
AddonsModule,
+6 -4
View File
@@ -12,6 +12,7 @@ import type { TourWaypointsRepository } from '../../db/repositories/TourWaypoint
import { toRowId } from '../common/row-id';
import { UnitOfWork } from '../database/unit-of-work';
import { PlacesService } from '../places/places.service';
import { PlaceImportService } from '../place-import/place-import.service';
import { computeTourMetrics, parseRouteGeometry, LOW_CONFIDENCE_THRESHOLD, type GeometryPoint } from './tours.helpers';
export interface ImportGpxAsTourResult {
@@ -23,8 +24,8 @@ export interface ImportGpxAsTourResult {
/**
* Tours domain: the `tours` facet table (place_id PK/FK) carries tour-specific
* metadata while existing places and day assignments are reused.
* GPX parsing and persistence are reused through PlacesService.prepareGpxRows
* and importPreparedGpx. Places and facet rows are persisted in one transaction,
* GPX parsing goes through PlaceImportService.readGpx, persistence through
* PlacesService.importPreparedGpx. Places and facet rows are persisted in one transaction,
* with tour metrics derived from each place's route_geometry.
*/
@Injectable()
@@ -38,6 +39,7 @@ export class ToursService {
@InjectRepository(TourTypes) private readonly tourTypesRepo: TourTypesRepository,
@InjectRepository(TourWaypoints) private readonly waypointsRepo: TourWaypointsRepository,
@InjectRepository(Places) private readonly placesRepo: PlacesRepository,
private readonly placeImport: PlaceImportService,
) {}
private toItem(r: TourListRow): TourListItem {
@@ -196,7 +198,7 @@ export class ToursService {
}
/**
* The tours-mode GPX import prepares rows through PlacesService.prepareGpxRows
* The tours-mode GPX import reads rows through PlaceImportService.readGpx
* with waypoints excluded. PlacesService.importPreparedGpx persists the places
* in the same transaction as their `tours` facet rows, whose metrics are
* derived from route_geometry.
@@ -206,7 +208,7 @@ export class ToursService {
* match_confidence so the client can surface a "with caution" toast.
*/
async importGpxAsTour(tripId: string, fileBuffer: Buffer, defaultName?: string, socketId?: string): Promise<ImportGpxAsTourResult | null> {
const rows = this.places.prepareGpxRows(fileBuffer, {
const rows = this.placeImport.readGpx(fileBuffer, {
importWaypoints: false, importRoutes: true, importTracks: true, defaultName,
});
if (rows.length === 0) return null;