mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
refactor(place-import): file and shared-list readers move into their own domain over one gpx and one kml codec
This commit is contained in:
@@ -48,6 +48,7 @@ import { JourneyModule } from './journey/journey.module';
|
||||
import { LlmParseModule } from './llm-parse/llm-parse.module';
|
||||
import { ManagedExtModule } from './managed/managed-ext.module';
|
||||
import { MapsModule } from './maps/maps.module';
|
||||
import { PlaceImportModule } from './place-import/place-import.module';
|
||||
import { McpTransportModule } from './mcp-transport/mcp-transport.module';
|
||||
import { MemoriesModule } from './memories/memories.module';
|
||||
import { NotificationsModule } from './notifications/notifications.module';
|
||||
@@ -119,6 +120,7 @@ import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
|
||||
SystemNoticesModule,
|
||||
GeoModule,
|
||||
MapsModule,
|
||||
PlaceImportModule,
|
||||
GoogleQuotaModule,
|
||||
PlaceEnrichmentModule,
|
||||
PlaceShadowModule,
|
||||
|
||||
@@ -56,7 +56,7 @@ import {
|
||||
CollectionGpxReadDto,
|
||||
} from './collections.dto';
|
||||
import { PlaceRatingDto } from '../places/places.dto';
|
||||
import { CollectionGpxError } from './collection-gpx.helpers';
|
||||
import { CollectionGpxError } from '../place-import/place-import.types';
|
||||
|
||||
export const MAX_COVER_SIZE = 20 * 1024 * 1024;
|
||||
// Duplicated on purpose from trips.controller.ts (historical parity — no
|
||||
|
||||
@@ -26,6 +26,7 @@ import { StorageModule } from '../storage/storage.module';
|
||||
import { StorageService } from '../storage/storage.service';
|
||||
import { buildStorageUploadOptions } from '../storage/storage-upload.factory';
|
||||
import { MAX_COVER_SIZE } from './collections.controller';
|
||||
import { PlaceImportModule } from '../place-import/place-import.module';
|
||||
|
||||
/** Collections domain (saved-places library). Registered in AppModule.
|
||||
* Exports CollectionsService for in-container consumers (PluginsModule's
|
||||
@@ -47,7 +48,7 @@ import { MAX_COVER_SIZE } from './collections.controller';
|
||||
}),
|
||||
}),
|
||||
StorageModule,
|
||||
NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule,
|
||||
NotificationsModule, AddonsModule, PermissionsModule, AppConfigModule, PluginGuardsModule, PlaceImportModule,
|
||||
MikroOrmModule.forFeature([
|
||||
Collections, CollectionMembers, CollectionLabels, CollectionPlaces, CollectionPlaceRatings,
|
||||
Categories, Users, Trips, TripMembers, Places, PlaceRatings, Tags,
|
||||
|
||||
@@ -73,7 +73,7 @@ import type {
|
||||
CollectionImportablesResponse,
|
||||
} from '@trek/shared';
|
||||
import { NotificationsService } from '../notifications/notifications.service';
|
||||
import { collectionFileToGpx, gpxToCollectionFile, type ExportedCollectionFile } from './collection-gpx.helpers';
|
||||
import { PlaceImportService, type ExportedCollectionFile } from '../place-import/place-import.service';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
|
||||
/** Links are stored as a JSON TEXT column; parse on read, stringify on write. */
|
||||
@@ -143,6 +143,7 @@ export class CollectionsService {
|
||||
@InjectRepository(PlaceRatings) private readonly tripPlaceRatings: PlaceRatingsRepository,
|
||||
@InjectRepository(Tags) private readonly tags: TagsRepository,
|
||||
@InjectRepository(Users) private readonly users: UsersRepository,
|
||||
private readonly placeImport: PlaceImportService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
@@ -398,17 +399,16 @@ export class CollectionsService {
|
||||
* only ever carry less of it. Same access rule too, since it is the same read.
|
||||
*/
|
||||
async exportCollectionGpx(userId: number, id: number): Promise<CollectionGpxExport> {
|
||||
return collectionFileToGpx(await this.exportCollection(userId, id));
|
||||
return this.placeImport.writeCollectionGpx(await this.exportCollection(userId, id));
|
||||
}
|
||||
|
||||
/**
|
||||
* A GPX document read into the list file it amounts to. Touches no table:
|
||||
* the file goes back to the browser to be shown, and comes back through
|
||||
* importCollection like any other, so there is one import and one
|
||||
* transaction whatever the format was.
|
||||
* A GPX document read (by place-import) into the list file it amounts to. Touches no table: the file
|
||||
* goes back to the browser to be shown, and comes back through importCollection like any other, so
|
||||
* there is one import and one transaction whatever the format was.
|
||||
*/
|
||||
readCollectionGpx(body: CollectionGpxReadRequest): CollectionGpxReadResult {
|
||||
return gpxToCollectionFile(body.gpx, body.file_name);
|
||||
return this.placeImport.readCollectionGpx(body.gpx, body.file_name);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Which hosts a pasted Google Maps link may come from. Shared kernel because
|
||||
* the maps link resolver and the directions import (place-import) both follow a
|
||||
* short link through the SSRF guard and then require the landing host to be
|
||||
* Google's own.
|
||||
*/
|
||||
|
||||
export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl'];
|
||||
|
||||
/**
|
||||
* Google Maps lives on every country domain — google.de, maps.google.co.uk,
|
||||
* google.com.au — so the host is matched by shape. A fixed list of .com hosts
|
||||
* would quietly stop resolving the ccTLD links people actually paste. The TLD
|
||||
* labels stay short (2-3 letters, optionally two of them) so that
|
||||
* `google.evil.com` is not a Google host.
|
||||
*/
|
||||
export function isGoogleMapsHost(hostname: string): boolean {
|
||||
return GOOGLE_SHORT_HOSTS.includes(hostname)
|
||||
|| /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname);
|
||||
}
|
||||
@@ -14,26 +14,13 @@ import { discardBody, exceedsDeclaredLength, readCappedText } from '../../utils/
|
||||
import { UA, googleFtidFromMapsUrl } from './maps.helpers';
|
||||
import { AMAP_SHORT_HOSTS, isAmapHost, parseAmapUrl } from './providers/amap.provider';
|
||||
import { OsmClient } from './providers/osm.client';
|
||||
import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts';
|
||||
|
||||
// A Google Maps place page is a few hundred KB; the coordinates sit in the
|
||||
// embedded map data near the top, so two megabytes is plenty and keeps an
|
||||
// unbounded body out of memory.
|
||||
const MAX_MAPS_PAGE_BYTES = 2_000_000;
|
||||
|
||||
export const GOOGLE_SHORT_HOSTS = ['goo.gl', 'maps.app.goo.gl'];
|
||||
|
||||
/**
|
||||
* Google Maps lives on every country domain — google.de, maps.google.co.uk,
|
||||
* google.com.au — so the host is matched by shape. A fixed list of .com hosts
|
||||
* would quietly stop resolving the ccTLD links people actually paste. The TLD
|
||||
* labels stay short (2-3 letters, optionally two of them) so that
|
||||
* `google.evil.com` is not a Google host.
|
||||
*/
|
||||
export function isGoogleMapsHost(hostname: string): boolean {
|
||||
return GOOGLE_SHORT_HOSTS.includes(hostname)
|
||||
|| /^(www\.|maps\.)?google\.[a-z]{2,3}(\.[a-z]{2})?$/.test(hostname);
|
||||
}
|
||||
|
||||
export interface ResolvedMapsUrl {
|
||||
lat: number;
|
||||
lng: number;
|
||||
|
||||
@@ -27,7 +27,7 @@ import { NEARBY_DEFAULT_LIMIT, NEARBY_DEFAULT_RADIUS_M, nearbyCacheKey, nearestF
|
||||
export { readBrandIdentity, readWikiIdentity, type WikiIdentity } from './providers/wiki-identity';
|
||||
export { withPhotoFetchSlot } from './photo-fetch-slot';
|
||||
export type { BrandLogo, CommonsCandidate } from './providers/wikimedia.client';
|
||||
export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from './maps-url.resolver';
|
||||
export { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../common/google-maps-hosts';
|
||||
export { PLACES_PROVIDER_SETTING, PLACES_GOOGLE_ONLY_SETTING, type KeyedProvider } from './places-provider.selector';
|
||||
|
||||
// Places near a point (#976): cached longer than the POI boxes, because the
|
||||
|
||||
@@ -29,6 +29,14 @@ export class PlacePhotoResolver {
|
||||
private readonly selector: PlacesProviderSelector,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* The photo for one place: disk hit, negative cache, in-flight dedupe, then
|
||||
* Google (for a Google id) and Wikimedia, under the shared photo-fetch slots.
|
||||
*
|
||||
* @txIndependent cache bookkeeping around network fetches: the stored photo,
|
||||
* the negative-cache entry and the image_url backfill are each complete on
|
||||
* their own, and a failed one costs the cache, never the answer.
|
||||
*/
|
||||
async resolve(
|
||||
userId: number,
|
||||
placeId: string,
|
||||
|
||||
@@ -286,6 +286,9 @@ export class GooglePlacesClient {
|
||||
* download. The outcome tells a place with no photo apart from a call that
|
||||
* went wrong, because the negative cache keeps the first for a day and the
|
||||
* second for minutes. Throws only when the transport itself does.
|
||||
*
|
||||
* @txIndependent two billed calls with network I/O between them: each counts
|
||||
* against the daily ceiling on its own, as it is made.
|
||||
*/
|
||||
async firstPhoto(placeId: string, apiKey: string): Promise<GooglePhotoOutcome> {
|
||||
// Fetch details to get the photo name
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* GPX in and out: the one place the XML library is configured for GPX.
|
||||
*
|
||||
* Reading comes in two modes, chosen by the caller, because the two GPX
|
||||
* importers want different things from the same document and have shipped
|
||||
* that way:
|
||||
*
|
||||
* - `geometry` (the trip and Tours importers): tag values are parsed, so
|
||||
* `<ele>12.5</ele>` arrives as a number, and the repeating elements are the
|
||||
* six that carry places and geometry, matched on the name as written.
|
||||
* `<name>007</name>` becomes the number 7 and character references stay
|
||||
* undecoded, which is fine for track geometry.
|
||||
* - `text` (the list importer in collections): text stays text, so a
|
||||
* waypoint called 007 keeps its zeros, numeric character references
|
||||
* (`é`) are decoded, and `link` / `label` repeat as well, matched on
|
||||
* the local name so a namespaced `<trek:label>` counts.
|
||||
*
|
||||
* Writing goes through one builder and one document envelope (`writeGpx`),
|
||||
* shared by the trip export and the list export: GPX 1.1, creator TREK, the
|
||||
* schema location, then the caller's namespaces, metadata and elements.
|
||||
*/
|
||||
import { XMLBuilder, XMLParser } from 'fast-xml-parser';
|
||||
|
||||
export type GpxReadMode = 'geometry' | 'text';
|
||||
|
||||
/** Repeating elements in geometry mode: the places and the geometry, by name as written. */
|
||||
const GEOMETRY_ARRAYS = ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'];
|
||||
|
||||
/** Repeating elements in text mode, matched on the local name. */
|
||||
const TEXT_ARRAYS = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']);
|
||||
|
||||
const localName = (name: string): string => name.slice(name.indexOf(':') + 1);
|
||||
|
||||
/** The GPX parser for one read mode. Built once per mode below; exported for the tests. */
|
||||
export function createGpxParser(mode: GpxReadMode): XMLParser {
|
||||
if (mode === 'geometry') {
|
||||
return new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
isArray: (name) => GEOMETRY_ARRAYS.includes(name),
|
||||
});
|
||||
}
|
||||
// Entities declared in a DOCTYPE never reach the text parser: the list reader
|
||||
// refuses a document carrying one before parsing.
|
||||
return new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
parseTagValue: false,
|
||||
parseAttributeValue: false,
|
||||
htmlEntities: true,
|
||||
isArray: (name, _path, _leaf, isAttribute) => !isAttribute && TEXT_ARRAYS.has(localName(name)),
|
||||
});
|
||||
}
|
||||
|
||||
/** Frozen at import, one shared instance per mode. */
|
||||
export const gpxGeometryParser = createGpxParser('geometry');
|
||||
export const gpxTextParser = createGpxParser('text');
|
||||
|
||||
export const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1';
|
||||
|
||||
/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer
|
||||
* device resolves and keeps the file from carrying float noise. */
|
||||
export function coord(n: number): string {
|
||||
return Number(n.toFixed(7)).toString();
|
||||
}
|
||||
|
||||
const gpxBuilder = new XMLBuilder({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
format: true,
|
||||
indentBy: ' ',
|
||||
suppressEmptyNode: true,
|
||||
});
|
||||
|
||||
/** What goes inside `<gpx>`; an empty element list is left out entirely. */
|
||||
export interface GpxDocument {
|
||||
/** Extra `xmlns:<prefix>` bindings, written after the default namespace. */
|
||||
namespaces?: Record<string, string>;
|
||||
metadata: Record<string, unknown>;
|
||||
wpt?: unknown[];
|
||||
rte?: unknown[];
|
||||
trk?: unknown[];
|
||||
}
|
||||
|
||||
/** A GPX 1.1 document, elements in the schema's order (metadata, wpt, rte, trk). */
|
||||
export function writeGpx(doc: GpxDocument): string {
|
||||
const namespaces = Object.fromEntries(Object.entries(doc.namespaces ?? {}).map(([prefix, uri]) => [`@_xmlns:${prefix}`, uri]));
|
||||
return gpxBuilder.build({
|
||||
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
|
||||
gpx: {
|
||||
'@_version': '1.1',
|
||||
'@_creator': 'TREK',
|
||||
'@_xmlns': GPX_NAMESPACE,
|
||||
...namespaces,
|
||||
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
|
||||
'@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`,
|
||||
metadata: doc.metadata,
|
||||
...(doc.wpt?.length ? { wpt: doc.wpt } : {}),
|
||||
...(doc.rte?.length ? { rte: doc.rte } : {}),
|
||||
...(doc.trk?.length ? { trk: doc.trk } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
+85
-34
@@ -1,27 +1,38 @@
|
||||
/**
|
||||
* KML and KMZ in: the one place the XML library is configured for KML.
|
||||
*
|
||||
* A KMZ is unpacked to its KML (doc.kml preferred, the decompressed size
|
||||
* capped), the bytes are decoded as UTF-8 (loosely, with a warning, when they
|
||||
* are not), the document is validated and parsed, and every Placemark comes
|
||||
* back with the folder it sat in, its name, a plain-text description, and
|
||||
* either a point or a path's first point plus the path as route geometry.
|
||||
* Which placemarks become places, and under which category, is the importing
|
||||
* domain's decision.
|
||||
*/
|
||||
import { TextDecoder } from 'util';
|
||||
import { XMLParser, XMLValidator } from 'fast-xml-parser';
|
||||
import unzipper from 'unzipper';
|
||||
import { stripHtmlTags } from '../common/stripHtmlTags';
|
||||
import type { KmlDocumentRead, KmlImportSummary, ParsedKmlPlacemark } from './place-import.types';
|
||||
|
||||
export interface ParsedKmlPlacemark {
|
||||
name: string | null;
|
||||
description: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
folderName: string | null;
|
||||
routeGeometry: string | null;
|
||||
}
|
||||
export type { KmlImportSummary, ParsedKmlPlacemark } from './place-import.types';
|
||||
|
||||
export interface KmlPlacemarkNode {
|
||||
placemark: any;
|
||||
folderName: string | null;
|
||||
}
|
||||
|
||||
export interface KmlImportSummary {
|
||||
totalPlacemarks: number;
|
||||
createdCount: number;
|
||||
skippedCount: number;
|
||||
warnings: string[];
|
||||
errors: string[];
|
||||
}
|
||||
const kmlParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
removeNSPrefix: true,
|
||||
isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name),
|
||||
// Treat <description> as raw text so mixed-content HTML (e.g. <br/>, <i>)
|
||||
// is returned as a string instead of a parsed object.
|
||||
stopNodes: ['*.description'],
|
||||
});
|
||||
|
||||
export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB
|
||||
|
||||
const UTF8_DECODER_FATAL = new TextDecoder('utf-8', { fatal: true });
|
||||
const UTF8_DECODER_LOOSE = new TextDecoder('utf-8');
|
||||
@@ -144,25 +155,6 @@ export function createKmlImportSummary(totalPlacemarks: number): KmlImportSummar
|
||||
};
|
||||
}
|
||||
|
||||
export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map<string, number> {
|
||||
const lookup = new Map<string, number>();
|
||||
for (const category of categories) {
|
||||
const normalizedName = category.name.trim().toLowerCase();
|
||||
if (!normalizedName) continue;
|
||||
if (!lookup.has(normalizedName)) {
|
||||
lookup.set(normalizedName, category.id);
|
||||
}
|
||||
}
|
||||
return lookup;
|
||||
}
|
||||
|
||||
export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map<string, number>): number | null {
|
||||
if (!folderName) return null;
|
||||
const normalizedFolder = folderName.trim().toLowerCase();
|
||||
if (!normalizedFolder) return null;
|
||||
return lookup.get(normalizedFolder) ?? null;
|
||||
}
|
||||
|
||||
export function extractKmlPlacemarkNodes(kmlRoot: any): KmlPlacemarkNode[] {
|
||||
const nodes: KmlPlacemarkNode[] = [];
|
||||
|
||||
@@ -211,3 +203,62 @@ export function parsePlacemarkNode(node: KmlPlacemarkNode): ParsedKmlPlacemark {
|
||||
routeGeometry,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A KML document as its placemarks (each level's own before its folders'), with the summary the
|
||||
* import fills in. Throws the two messages the import route has always
|
||||
* answered for a broken file.
|
||||
*/
|
||||
export function readKmlDocument(fileBuffer: Buffer): KmlDocumentRead {
|
||||
const decoded = decodeUtf8WithWarning(fileBuffer);
|
||||
|
||||
const validationResult = XMLValidator.validate(decoded.text);
|
||||
if (validationResult !== true) {
|
||||
throw new Error('Malformed KML: invalid XML structure');
|
||||
}
|
||||
|
||||
const parsed = kmlParser.parse(decoded.text);
|
||||
const kmlRoot = parsed?.kml ?? parsed;
|
||||
|
||||
if (!kmlRoot || typeof kmlRoot !== 'object') {
|
||||
throw new Error('Malformed KML: could not parse XML');
|
||||
}
|
||||
|
||||
const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot);
|
||||
const summary = createKmlImportSummary(placemarkNodes.length);
|
||||
|
||||
if (decoded.warning) {
|
||||
summary.warnings.push(decoded.warning);
|
||||
}
|
||||
|
||||
return { placemarks: placemarkNodes.map(parsePlacemarkNode), summary };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// KMZ unpacking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function unpackKmzToKml(
|
||||
kmzBuffer: Buffer,
|
||||
decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT,
|
||||
): Promise<Buffer> {
|
||||
let zip;
|
||||
try {
|
||||
zip = await unzipper.Open.buffer(kmzBuffer);
|
||||
} catch {
|
||||
throw new Error('Invalid KMZ archive.');
|
||||
}
|
||||
|
||||
const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml'));
|
||||
if (kmlEntries.length === 0) {
|
||||
throw new Error('KMZ archive does not contain a KML file.');
|
||||
}
|
||||
|
||||
const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0];
|
||||
|
||||
if (preferredEntry.uncompressedSize > decompressedSizeLimit) {
|
||||
throw new Error('KMZ archive exceeds the maximum allowed decompressed size.');
|
||||
}
|
||||
|
||||
return preferredEntry.buffer();
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { PlaceImportService } from './place-import.service';
|
||||
import { GpxProvider } from './providers/gpx.provider';
|
||||
import { CollectionGpxProvider } from './providers/collection-gpx.provider';
|
||||
import { KmlProvider } from './providers/kml.provider';
|
||||
import { GoogleListProvider } from './providers/google-list.provider';
|
||||
import { GoogleDirectionsProvider } from './providers/google-directions.provider';
|
||||
import { NaverListProvider } from './providers/naver-list.provider';
|
||||
|
||||
/**
|
||||
* Place import (leaf module): reads places out of GPX, KML/KMZ, Google lists,
|
||||
* Google directions links and Naver lists, and writes GPX. No controller, no
|
||||
* table of its own: PlacesModule, CollectionsModule and ToursModule import it
|
||||
* and persist what it reads. It imports no other
|
||||
* domain: the stops of a directions link that carry only a name are geocoded
|
||||
* through a function the importer hands in (MapsService.geocodeQuery).
|
||||
*/
|
||||
@Module({
|
||||
providers: [
|
||||
PlaceImportService,
|
||||
GpxProvider,
|
||||
CollectionGpxProvider,
|
||||
KmlProvider,
|
||||
GoogleListProvider,
|
||||
GoogleDirectionsProvider,
|
||||
NaverListProvider,
|
||||
],
|
||||
exports: [PlaceImportService],
|
||||
})
|
||||
export class PlaceImportModule {}
|
||||
@@ -0,0 +1,159 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { CollectionGpxExport, CollectionGpxReadResult } from '@trek/shared';
|
||||
import { GpxProvider, type GpxExportInput, type GpxExportOptions } from './providers/gpx.provider';
|
||||
import { CollectionGpxProvider, type ExportedCollectionFile } from './providers/collection-gpx.provider';
|
||||
import { KmlProvider } from './providers/kml.provider';
|
||||
import { GoogleListProvider } from './providers/google-list.provider';
|
||||
import { GoogleDirectionsProvider } from './providers/google-directions.provider';
|
||||
import { NaverListProvider } from './providers/naver-list.provider';
|
||||
import type {
|
||||
DirectionsRead,
|
||||
DirectionsRedirect,
|
||||
GoogleListPlace,
|
||||
GpxImportOptions,
|
||||
KmlDocumentRead,
|
||||
ListImportError,
|
||||
ListRead,
|
||||
NaverListPlace,
|
||||
PreparedGpxPlace,
|
||||
} from './place-import.types';
|
||||
|
||||
// The directions-link reading is also what the road trip planner reads a
|
||||
// shared route with, and what the import routes dispatch on.
|
||||
export { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS, type DirWaypoint } from './directions-url.helpers';
|
||||
export type { GpxExportDay, GpxExportInput, GpxExportOptions, GpxExportPlace } from './providers/gpx.provider';
|
||||
export type { ExportedCollectionFile } from './providers/collection-gpx.provider';
|
||||
|
||||
/** Places a name, or says nobody can. */
|
||||
export type Geocoder = (query: string) => Promise<{ lat: number; lng: number } | null>;
|
||||
|
||||
/**
|
||||
* Places out of files and shared lists: the one layer that reads GPX, KML/KMZ,
|
||||
* Google lists, Google directions links and Naver lists, and writes GPX.
|
||||
*
|
||||
* One provider per source behind it, over one GPX codec and one KML codec.
|
||||
* Nothing here persists: a trip import (PlacesService), the Tours import
|
||||
* (ToursService) and a list import or export (CollectionsService) each take
|
||||
* what comes back and write their own rows, so the dedup, the transactions and
|
||||
* the broadcasts stay with the domain that owns them.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PlaceImportService {
|
||||
constructor(
|
||||
private readonly gpx: GpxProvider,
|
||||
private readonly collectionGpx: CollectionGpxProvider,
|
||||
private readonly kml: KmlProvider,
|
||||
private readonly googleList: GoogleListProvider,
|
||||
private readonly googleDirections: GoogleDirectionsProvider,
|
||||
private readonly naverList: NaverListProvider,
|
||||
) {}
|
||||
|
||||
// ── GPX ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/** The places a trip GPX import would create, nothing written. */
|
||||
readGpx(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
|
||||
return this.gpx.read(fileBuffer, opts);
|
||||
}
|
||||
|
||||
/** A trip as a GPX document, or null when the selection yields nothing. */
|
||||
writeTripGpx(input: GpxExportInput, opts: GpxExportOptions = {}): string | null {
|
||||
return this.gpx.write(input, opts);
|
||||
}
|
||||
|
||||
tripGpxFilename(tripTitle: string): string {
|
||||
return this.gpx.filename(tripTitle);
|
||||
}
|
||||
|
||||
/** A list file as GPX (#2301). */
|
||||
writeCollectionGpx(file: ExportedCollectionFile): CollectionGpxExport {
|
||||
return this.collectionGpx.write(file);
|
||||
}
|
||||
|
||||
/** A GPX document as the list file it amounts to. Throws CollectionGpxError on a refusal. */
|
||||
readCollectionGpx(source: string, fileName?: string): CollectionGpxReadResult {
|
||||
return this.collectionGpx.read(source, fileName);
|
||||
}
|
||||
|
||||
// ── KML / KMZ ──────────────────────────────────────────────────────────────
|
||||
|
||||
/** A .kml or .kmz upload, by its file name. */
|
||||
readMapFile(fileBuffer: Buffer, filename: string): Promise<KmlDocumentRead> {
|
||||
return this.kml.readMapFile(fileBuffer, filename);
|
||||
}
|
||||
|
||||
readKmz(kmzBuffer: Buffer): Promise<KmlDocumentRead> {
|
||||
return this.kml.readKmz(kmzBuffer);
|
||||
}
|
||||
|
||||
readKml(fileBuffer: Buffer): KmlDocumentRead {
|
||||
return this.kml.readKml(fileBuffer);
|
||||
}
|
||||
|
||||
// ── Shared lists ───────────────────────────────────────────────────────────
|
||||
|
||||
/** A Google Maps list, or the directions link a short link turned out to be. */
|
||||
readGoogleList(url: string): Promise<ListRead<GoogleListPlace> | DirectionsRedirect | ListImportError> {
|
||||
return this.googleList.read(url);
|
||||
}
|
||||
|
||||
readNaverList(url: string): Promise<ListRead<NaverListPlace> | ListImportError> {
|
||||
return this.naverList.read(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* The stops of a shared directions link, placed on the map.
|
||||
*
|
||||
* `geocode` places a stop the link names without coordinates. It is handed
|
||||
* in rather than injected so this layer stays below the maps domain: the
|
||||
* importer passes MapsService.geocodeQuery.
|
||||
*
|
||||
* A stop the link spells out in coordinates is taken as it stands; one that
|
||||
* is only a name is geocoded, one request each. A name nobody can place is
|
||||
* left out rather than failing the import, because a route of six stops with
|
||||
* five findable is five stops more than the traveller had.
|
||||
*/
|
||||
async readGoogleDirections(url: string, geocode: Geocoder): Promise<DirectionsRead | ListImportError> {
|
||||
const waypoints = await this.googleDirections.read(url);
|
||||
if (!Array.isArray(waypoints)) return waypoints;
|
||||
|
||||
const places: GoogleListPlace[] = [];
|
||||
let unplaceable = 0;
|
||||
for (const wp of waypoints) {
|
||||
if (wp.lat !== null && wp.lng !== null) {
|
||||
// A stop written as coordinates has no name of its own, and the coordinates are a
|
||||
// better label than "Stop 3": they are what the traveller can look up.
|
||||
places.push({
|
||||
name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`,
|
||||
lat: wp.lat,
|
||||
lng: wp.lng,
|
||||
notes: null,
|
||||
googleFtid: null,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!wp.name) continue;
|
||||
try {
|
||||
// Through geocodeQuery, which asks the TREK index first and only falls
|
||||
// through to Nominatim for what it does not know — and does so on the
|
||||
// BACKGROUND lane. That matters here more than anywhere: this loop runs
|
||||
// up to thirty times in one request, each Nominatim call taking the next
|
||||
// slot on a 1.1 s process-wide throttle, so on the interactive lane one
|
||||
// pasted link made everybody else's place search queue behind it for
|
||||
// half a minute. An index hit costs no slot at all.
|
||||
const hit = await geocode(wp.name);
|
||||
// The name from the link, not the one the geocoder answers with: somebody who
|
||||
// typed a nickname into Google should not find a street address on their trip.
|
||||
if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null });
|
||||
else unplaceable++;
|
||||
} catch {
|
||||
// A geocoder that is down or rate-limited costs this one stop, not the import.
|
||||
unplaceable++;
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length < 2) {
|
||||
return { error: 'None of the stops in that link could be placed on the map.', status: 400 };
|
||||
}
|
||||
return { places, unplaceable };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* What the place-import layer hands its callers: places read out of a file or
|
||||
* a shared list, not yet persisted. Persisting them (dedup against the trip,
|
||||
* insert, track colours, enrichment) stays with the domain that owns the rows,
|
||||
* PlacesService for a trip and CollectionsService for a list.
|
||||
*/
|
||||
import type { CollectionGpxProblem } from '@trek/shared';
|
||||
|
||||
/** One place parsed out of a GPX file, not yet persisted. */
|
||||
export interface PreparedGpxPlace {
|
||||
name: string;
|
||||
lat: number;
|
||||
lng: number;
|
||||
description: string | null;
|
||||
routeGeometry?: string;
|
||||
}
|
||||
|
||||
export interface GpxImportOptions {
|
||||
importWaypoints?: boolean;
|
||||
importRoutes?: boolean;
|
||||
importTracks?: boolean;
|
||||
/** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */
|
||||
defaultName?: string;
|
||||
}
|
||||
|
||||
export interface KmlImportOptions {
|
||||
importPoints?: boolean;
|
||||
importPaths?: boolean;
|
||||
}
|
||||
|
||||
export interface KmlImportSummary {
|
||||
totalPlacemarks: number;
|
||||
createdCount: number;
|
||||
skippedCount: number;
|
||||
warnings: string[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
export interface ParsedKmlPlacemark {
|
||||
name: string | null;
|
||||
description: string | null;
|
||||
lat: number | null;
|
||||
lng: number | null;
|
||||
folderName: string | null;
|
||||
routeGeometry: string | null;
|
||||
}
|
||||
|
||||
/** A KML document read: every placemark, and the summary the import fills in. */
|
||||
export interface KmlDocumentRead {
|
||||
placemarks: ParsedKmlPlacemark[];
|
||||
summary: KmlImportSummary;
|
||||
}
|
||||
|
||||
/** An import that could not happen, with the status and the exact message the route answers. */
|
||||
export interface ListImportError {
|
||||
error: string;
|
||||
status: number;
|
||||
}
|
||||
|
||||
/** A place out of a Google list or a directions link. */
|
||||
export interface GoogleListPlace {
|
||||
name: string;
|
||||
lat: number;
|
||||
lng: number;
|
||||
notes: string | null;
|
||||
googleFtid: string | null;
|
||||
}
|
||||
|
||||
/** A place out of a Naver list. */
|
||||
export interface NaverListPlace {
|
||||
name: string;
|
||||
lat: number;
|
||||
lng: number;
|
||||
notes: string | null;
|
||||
address: string | null;
|
||||
}
|
||||
|
||||
/** A shared list read: its name and the places that carried coordinates. */
|
||||
export interface ListRead<P> {
|
||||
listName: string;
|
||||
places: P[];
|
||||
}
|
||||
|
||||
/** A Google link that turned out to be a route once its redirect was followed. */
|
||||
export interface DirectionsRedirect {
|
||||
directions: string;
|
||||
}
|
||||
|
||||
/** The stops of a directions link, placed on the map; `unplaceable` counts the ones nobody could find. */
|
||||
export interface DirectionsRead {
|
||||
places: GoogleListPlace[];
|
||||
unplaceable: number;
|
||||
}
|
||||
|
||||
/** Why a GPX document was refused for a list, carried to the controller as the response's `code`. */
|
||||
export class CollectionGpxError extends Error {
|
||||
constructor(
|
||||
readonly code: CollectionGpxProblem,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'CollectionGpxError';
|
||||
}
|
||||
}
|
||||
+23
-54
@@ -1,4 +1,4 @@
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import type { ZodType } from 'zod';
|
||||
import {
|
||||
COLLECTION_FILE_FORMAT,
|
||||
@@ -16,11 +16,13 @@ import {
|
||||
type CollectionFileLabel,
|
||||
type CollectionFilePlace,
|
||||
type CollectionGpxExport,
|
||||
type CollectionGpxProblem,
|
||||
type CollectionGpxReadResult,
|
||||
type CollectionLink,
|
||||
} from '@trek/shared';
|
||||
import { coord, gpxBuilder } from '../places/gpx-export.helpers';
|
||||
import { coord, gpxTextParser, writeGpx } from '../gpx.codec';
|
||||
import { CollectionGpxError } from '../place-import.types';
|
||||
|
||||
export { CollectionGpxError } from '../place-import.types';
|
||||
|
||||
/**
|
||||
* A list as GPX and back (#2301).
|
||||
@@ -32,16 +34,15 @@ import { coord, gpxBuilder } from '../places/gpx-export.helpers';
|
||||
* fields of a place map onto a waypoint, and that table lives in the contract
|
||||
* (`COLLECTION_GPX_PLACE_FIELDS`).
|
||||
*
|
||||
* Same library as the trip importer and exporter in places/, with a reader of
|
||||
* its own: the trip parser turns `<name>007</name>` into the number 7 and does
|
||||
* not decode character references, which is fine for track geometry and wrong
|
||||
* for the names and notes a list is made of.
|
||||
* Same codec as the trip importer and exporter (gpx.codec), in its `text`
|
||||
* read mode: the trip parser turns `<name>007</name>` into the number 7 and
|
||||
* does not decode character references, which is fine for track geometry and
|
||||
* wrong for the names and notes a list is made of.
|
||||
*/
|
||||
|
||||
/** A list file as the exporter builds it, its places already in the file's shape. */
|
||||
export type ExportedCollectionFile = Omit<CollectionFile, 'places'> & { places: CollectionFilePlace[] };
|
||||
|
||||
const GPX_NAMESPACE = 'http://www.topografix.com/GPX/1/1';
|
||||
/** The prefix this writer binds; the reader looks the namespace up instead. */
|
||||
const TREK = 'trek';
|
||||
/** OsmAnd writes its favourites with the address in its own namespace. */
|
||||
@@ -148,19 +149,7 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx
|
||||
wpt.push(waypoint(place, lat, lng));
|
||||
}
|
||||
|
||||
const gpx: string = gpxBuilder.build({
|
||||
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
|
||||
gpx: {
|
||||
'@_version': '1.1',
|
||||
'@_creator': 'TREK',
|
||||
'@_xmlns': GPX_NAMESPACE,
|
||||
[`@_xmlns:${TREK}`]: COLLECTION_GPX_NAMESPACE,
|
||||
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
|
||||
'@_xsi:schemaLocation': `${GPX_NAMESPACE} ${GPX_NAMESPACE}/gpx.xsd`,
|
||||
metadata: metadata(file),
|
||||
...(wpt.length ? { wpt } : {}),
|
||||
},
|
||||
});
|
||||
const gpx = writeGpx({ namespaces: { [TREK]: COLLECTION_GPX_NAMESPACE }, metadata: metadata(file), wpt });
|
||||
return { name: file.name, gpx, waypoints: wpt.length, omitted };
|
||||
}
|
||||
|
||||
@@ -168,17 +157,6 @@ export function collectionFileToGpx(file: ExportedCollectionFile): CollectionGpx
|
||||
// Reading
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Why a document was refused, carried to the controller as the response's `code`. */
|
||||
export class CollectionGpxError extends Error {
|
||||
constructor(
|
||||
readonly code: CollectionGpxProblem,
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'CollectionGpxError';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The most elements a document may open before it is parsed at all.
|
||||
*
|
||||
@@ -189,27 +167,6 @@ export class CollectionGpxError extends Error {
|
||||
*/
|
||||
export const MAX_GPX_ELEMENTS = 100_000;
|
||||
|
||||
/** Elements that may repeat, so they always parse as arrays. Matched on the local name. */
|
||||
const REPEATED = new Set(['wpt', 'rte', 'rtept', 'trk', 'trkseg', 'trkpt', 'link', 'label']);
|
||||
|
||||
const localName = (name: string): string => name.slice(name.indexOf(':') + 1);
|
||||
|
||||
/**
|
||||
* Text stays text (`parseTagValue: false`), so a waypoint called 007 keeps its
|
||||
* zeros. `htmlEntities` is on for the numeric character references it also
|
||||
* unlocks (`é`), which this library otherwise leaves undecoded. Entities
|
||||
* declared in a DOCTYPE never reach it: a document carrying one is refused
|
||||
* before parsing.
|
||||
*/
|
||||
const reader = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
parseTagValue: false,
|
||||
parseAttributeValue: false,
|
||||
htmlEntities: true,
|
||||
isArray: (name, _path, _leaf, isAttribute) => !isAttribute && REPEATED.has(localName(name)),
|
||||
});
|
||||
|
||||
type XmlNode = Record<string, unknown>;
|
||||
|
||||
function isNode(value: unknown): value is XmlNode {
|
||||
@@ -410,7 +367,7 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti
|
||||
const xml = refuseBeforeParsing(source);
|
||||
let doc: unknown;
|
||||
try {
|
||||
doc = reader.parse(xml, true);
|
||||
doc = gpxTextParser.parse(xml, true);
|
||||
} catch {
|
||||
throw new CollectionGpxError('unreadable', 'That GPX file is not well-formed XML');
|
||||
}
|
||||
@@ -481,3 +438,15 @@ export function gpxToCollectionFile(source: string, fileName?: string): Collecti
|
||||
if (!file) throw new CollectionGpxError('unreadable', 'That GPX file could not be read as a list');
|
||||
return { file, skipped, track_points: trackPoints };
|
||||
}
|
||||
|
||||
/** The GPX source for lists: the list file out as GPX, a GPX file in as a list file. */
|
||||
@Injectable()
|
||||
export class CollectionGpxProvider {
|
||||
write(file: ExportedCollectionFile): CollectionGpxExport {
|
||||
return collectionFileToGpx(file);
|
||||
}
|
||||
|
||||
read(source: string, fileName?: string): CollectionGpxReadResult {
|
||||
return gpxToCollectionFile(source, fileName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* The stops of a shared Google Maps directions link, read off the link itself.
|
||||
*
|
||||
* The other half of the request the list import answers: people share a drive
|
||||
* far more often than they share a list. No API key is involved and no call is
|
||||
* made to Google for the link itself — the stops are in the URL, which is the
|
||||
* whole reason this is possible at all. A short link is followed hop by hop
|
||||
* through the SSRF guard first, and the host that counts is the one the link
|
||||
* lands on.
|
||||
*
|
||||
* A stop the link spells out in coordinates is taken as it stands; one that is
|
||||
* only a name comes back without coordinates for the caller to geocode. This
|
||||
* provider asks no geocoder: choosing which source answers is MapsService's
|
||||
* job, reached through PlaceImportService.
|
||||
*/
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
|
||||
import { GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../../common/google-maps-hosts';
|
||||
import { parseDirectionsUrl, type DirWaypoint } from '../directions-url.helpers';
|
||||
import type { ListImportError } from '../place-import.types';
|
||||
|
||||
@Injectable()
|
||||
export class GoogleDirectionsProvider {
|
||||
/** The stops in driving order (two at least), or the refusal the route answers. */
|
||||
async read(url: string): Promise<DirWaypoint[] | ListImportError> {
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
let parsed: URL;
|
||||
try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
|
||||
|
||||
// Short links are resolved hop by hop through the guard, exactly as the list import
|
||||
// does it: a maps.app.goo.gl that 302s to an internal address is still blocked.
|
||||
let resolvedUrl = url;
|
||||
if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Checked after resolving, not before: the host that counts is the one the link lands
|
||||
// on, and `/maps/dir/` is a path anybody could serve.
|
||||
let host = '';
|
||||
try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ }
|
||||
if (!isGoogleMapsHost(host)) {
|
||||
return { error: 'That link is not a Google Maps link.', status: 400 };
|
||||
}
|
||||
|
||||
const waypoints = parseDirectionsUrl(resolvedUrl);
|
||||
if (waypoints.length < 2) {
|
||||
return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 };
|
||||
}
|
||||
return waypoints;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* A shared Google Maps list, read from Google's own list endpoint.
|
||||
*
|
||||
* The link is user input, so it is checked by the SSRF guard first and a short
|
||||
* link is followed hop by hop through it. A short link that lands on a route
|
||||
* (the Share sheet on a phone produces exactly that) is handed back as a
|
||||
* directions link for the directions source; everything else must carry a list
|
||||
* id. The answer is capped before and after it is read, because its size is
|
||||
* attacker-influenced through the list id. Every refusal is the exact message
|
||||
* and status the import route has always answered.
|
||||
*/
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
|
||||
import { isDirectionsUrl } from '../directions-url.helpers';
|
||||
import type { DirectionsRedirect, GoogleListPlace, ListImportError, ListRead } from '../place-import.types';
|
||||
|
||||
/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */
|
||||
export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
|
||||
/** The browser user agent the list endpoints answer. */
|
||||
export const LIST_USER_AGENT =
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
||||
|
||||
export function googleMapsHexId(value: unknown): string | null {
|
||||
if (typeof value !== 'string' && typeof value !== 'number') return null;
|
||||
const raw = String(value).trim();
|
||||
if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase();
|
||||
if (!/^-?\d+$/.test(raw)) return null;
|
||||
try {
|
||||
const parsed = BigInt(raw);
|
||||
const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed;
|
||||
return `0x${unsigned.toString(16)}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function googleMapsFeatureIdFromItem(item: unknown): string | null {
|
||||
if (!Array.isArray(item)) return null;
|
||||
const candidates = [
|
||||
Array.isArray(item[1]) ? item[1][6] : null,
|
||||
Array.isArray(item[7]) ? item[7][1] : null,
|
||||
];
|
||||
|
||||
for (const ids of candidates) {
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
const first = googleMapsHexId(ids[0]);
|
||||
const second = googleMapsHexId(ids[1]);
|
||||
if (first && second) return `${first}:${second}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GoogleListProvider {
|
||||
async read(url: string): Promise<ListRead<GoogleListPlace> | DirectionsRedirect | ListImportError> {
|
||||
let listId: string | null = null;
|
||||
let resolvedUrl = url;
|
||||
|
||||
// SSRF guard: validate user-supplied URL before fetching
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
// Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are
|
||||
// followed manually so every hop is re-checked against the SSRF guard — a
|
||||
// short link that 302s to an internal IP is blocked even though the initial
|
||||
// host is public.
|
||||
if (url.includes('goo.gl') || url.includes('maps.app')) {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// A route, once the redirect is followed. The dispatch upstream decides on
|
||||
// the raw URL, and a short link's path is `/<code>` — it matches nothing, so
|
||||
// every route shared from the Google Maps app arrived here and was answered
|
||||
// with "could not extract list ID", which is the complaint the directions
|
||||
// import was written to remove. The Share sheet on a phone produces exactly
|
||||
// this shape, and the box says a directions link works.
|
||||
//
|
||||
// Handed on with the resolved URL, so the hop is not made twice.
|
||||
if (isDirectionsUrl(resolvedUrl)) return { directions: resolvedUrl };
|
||||
|
||||
// Pattern: /placelists/list/{ID}
|
||||
const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/);
|
||||
if (plMatch) listId = plMatch[1];
|
||||
|
||||
// Pattern: !2s{ID} in data URL params
|
||||
if (!listId) {
|
||||
const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/);
|
||||
if (dataMatch) listId = dataMatch[1];
|
||||
}
|
||||
|
||||
if (!listId) {
|
||||
// A single-place share link (…/maps/place/…) carries no list id — point the user at
|
||||
// the place search box instead of a cryptic "could not extract list ID" (#1304).
|
||||
if (resolvedUrl.includes('/maps/place/')) {
|
||||
return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 };
|
||||
}
|
||||
return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 };
|
||||
}
|
||||
|
||||
// Fetch list data from Google Maps internal API
|
||||
const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`;
|
||||
const apiRes = await fetch(apiUrl, {
|
||||
headers: { 'User-Agent': LIST_USER_AGENT },
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!apiRes.ok) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
|
||||
// Cap the declared body before reading it (transit.service precedent): the
|
||||
// response is attacker-influenced via the list id, and buffering it whole
|
||||
// used to be unbounded.
|
||||
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
|
||||
if (declared > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
|
||||
const rawText = await apiRes.text();
|
||||
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
const jsonStr = rawText.substring(rawText.indexOf('\n') + 1);
|
||||
// The provider hands back a JS-prefixed array; a malformed body is a
|
||||
// provider problem, not a crash — surface the same 400 an unreadable
|
||||
// payload already produced instead of throwing a SyntaxError.
|
||||
let listData: unknown;
|
||||
try {
|
||||
listData = JSON.parse(jsonStr);
|
||||
} catch {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
if (!Array.isArray(listData)) {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
|
||||
const meta = listData[0];
|
||||
if (!meta) {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
|
||||
const listName = meta[4] || 'Google Maps List';
|
||||
const items = meta[8];
|
||||
|
||||
if (!Array.isArray(items) || items.length === 0) {
|
||||
return { error: 'List is empty or could not be read', status: 400 };
|
||||
}
|
||||
|
||||
// Parse place data from items
|
||||
const places: GoogleListPlace[] = [];
|
||||
for (const item of items) {
|
||||
const coords = item?.[1]?.[5];
|
||||
const lat = coords?.[2];
|
||||
const lng = coords?.[3];
|
||||
const name = item?.[2];
|
||||
const note = item?.[3] || null;
|
||||
|
||||
if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) {
|
||||
places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) });
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length === 0) {
|
||||
return { error: 'No places with coordinates found in list', status: 400 };
|
||||
}
|
||||
|
||||
return { listName, places };
|
||||
}
|
||||
}
|
||||
+100
-34
@@ -1,8 +1,11 @@
|
||||
import { XMLBuilder } from 'fast-xml-parser';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { coord, gpxGeometryParser, writeGpx } from '../gpx.codec';
|
||||
import type { GpxImportOptions, PreparedGpxPlace } from '../place-import.types';
|
||||
|
||||
/**
|
||||
* GPX writer, the mirror of the importer in places.helpers.ts. Same library, the
|
||||
* builder half rather than the parser half, so nothing new is pulled in.
|
||||
* A trip as GPX and back: the reader that turns a GPX file into the places an
|
||||
* import would create, and the writer that is its mirror. Both go through
|
||||
* gpx.codec (the geometry read mode, the shared document writer).
|
||||
*
|
||||
* The import decides the shape here: a `<wpt>` becomes a place with coordinates, a
|
||||
* `<rte>` or `<trk>` becomes a place carrying `route_geometry`, so writing back is
|
||||
@@ -46,12 +49,6 @@ export interface GpxExportOptions {
|
||||
|
||||
type Pt = { lat: number; lng: number; ele: number | null };
|
||||
|
||||
/** Coordinates are written with 7 decimals, ~11 mm, which is past what any consumer
|
||||
* device resolves and keeps the file from carrying float noise. */
|
||||
export function coord(n: number): string {
|
||||
return Number(n.toFixed(7)).toString();
|
||||
}
|
||||
|
||||
function parseGeometry(raw: string | null): Pt[] {
|
||||
if (!raw) return [];
|
||||
let parsed: unknown;
|
||||
@@ -81,15 +78,6 @@ function describe(place: GpxExportPlace): string | undefined {
|
||||
return parts.length ? parts.join(', ') : undefined;
|
||||
}
|
||||
|
||||
/** Shared with the list writer in collections/, which writes the same dialect. */
|
||||
export const gpxBuilder = new XMLBuilder({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
format: true,
|
||||
indentBy: ' ',
|
||||
suppressEmptyNode: true,
|
||||
});
|
||||
|
||||
/**
|
||||
* Build a GPX 1.1 document. Returns null when the selection produced nothing at all,
|
||||
* so the caller can answer 404 rather than hand over an empty file that silently
|
||||
@@ -154,22 +142,7 @@ export function buildGpx(input: GpxExportInput, opts: GpxExportOptions = {}): st
|
||||
|
||||
if (wpt.length === 0 && trk.length === 0 && rte.length === 0) return null;
|
||||
|
||||
const doc = {
|
||||
'?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' },
|
||||
gpx: {
|
||||
'@_version': '1.1',
|
||||
'@_creator': 'TREK',
|
||||
'@_xmlns': 'http://www.topografix.com/GPX/1/1',
|
||||
'@_xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
|
||||
'@_xsi:schemaLocation': 'http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd',
|
||||
metadata: { name: input.tripTitle },
|
||||
...(wpt.length ? { wpt } : {}),
|
||||
...(rte.length ? { rte } : {}),
|
||||
...(trk.length ? { trk } : {}),
|
||||
},
|
||||
};
|
||||
|
||||
return gpxBuilder.build(doc);
|
||||
return writeGpx({ metadata: { name: input.tripTitle }, wpt, rte, trk });
|
||||
}
|
||||
|
||||
/** Filenames land on the receiving filesystem, so its reserved characters are
|
||||
@@ -187,3 +160,96 @@ export function gpxFilename(tripTitle: string): string {
|
||||
// lone surrogate is exactly what URI-encoding chokes on.
|
||||
return `${[...base].slice(0, 60).join('') || 'trip'}.gpx`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a GPX file into the places an import would create, without writing
|
||||
* anything. Waypoints become places, each route and each track becomes one
|
||||
* place carrying its geometry.
|
||||
*/
|
||||
export function prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
|
||||
const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts;
|
||||
|
||||
const parsed = gpxGeometryParser.parse(fileBuffer.toString('utf-8'));
|
||||
const gpx = parsed?.gpx;
|
||||
if (!gpx) return [];
|
||||
|
||||
const str = (v: unknown) => (v != null ? String(v).trim() : null);
|
||||
const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; };
|
||||
|
||||
// Routes and tracks rarely carry their own <name>. Without one they all fall back to the
|
||||
// same generic label, so name-based dedup drops every import after the first. Derive a
|
||||
// base from the source filename (the requested behaviour) and suffix an index so multiple
|
||||
// geometries from one file stay distinct.
|
||||
const rawName = str(defaultName);
|
||||
const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null;
|
||||
let geoSeq = 0;
|
||||
const geoName = (explicit: string | null, fallback: string): string => {
|
||||
if (explicit) return explicit;
|
||||
geoSeq++;
|
||||
const base = baseName || fallback;
|
||||
return geoSeq === 1 ? base : `${base} ${geoSeq}`;
|
||||
};
|
||||
|
||||
const waypoints: PreparedGpxPlace[] = [];
|
||||
|
||||
// 1) Parse <wpt> elements (named waypoints / POIs)
|
||||
if (importWaypoints) {
|
||||
for (const wpt of gpx.wpt ?? []) {
|
||||
const lat = num(wpt['@_lat']);
|
||||
const lng = num(wpt['@_lon']);
|
||||
if (lat === null || lng === null) continue;
|
||||
waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) });
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Parse <rte> routes as polyline-places (one place per route with route_geometry)
|
||||
if (importRoutes) {
|
||||
for (const rte of gpx.rte ?? []) {
|
||||
const pts = (rte.rtept ?? [])
|
||||
.map((pt: Record<string, unknown>) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) }))
|
||||
.filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>;
|
||||
if (pts.length === 0) continue;
|
||||
const hasAllEle = pts.every(p => p.ele !== null);
|
||||
const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
|
||||
waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) });
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Extract full track geometry from <trk>
|
||||
if (importTracks) {
|
||||
for (const trk of gpx.trk ?? []) {
|
||||
const trackPoints: { lat: number; lng: number; ele: number | null }[] = [];
|
||||
for (const seg of trk.trkseg ?? []) {
|
||||
for (const pt of seg.trkpt ?? []) {
|
||||
const lat = num(pt['@_lat']);
|
||||
const lng = num(pt['@_lon']);
|
||||
if (lat === null || lng === null) continue;
|
||||
trackPoints.push({ lat, lng, ele: num(pt.ele) });
|
||||
}
|
||||
}
|
||||
if (trackPoints.length === 0) continue;
|
||||
const start = trackPoints[0];
|
||||
const hasAllEle = trackPoints.every(p => p.ele !== null);
|
||||
const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
|
||||
waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) });
|
||||
}
|
||||
}
|
||||
|
||||
return waypoints;
|
||||
}
|
||||
|
||||
/** The GPX source: a trip file in, a trip file out. */
|
||||
@Injectable()
|
||||
export class GpxProvider {
|
||||
read(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
|
||||
return prepareGpxRows(fileBuffer, opts);
|
||||
}
|
||||
|
||||
write(input: GpxExportInput, opts: GpxExportOptions = {}): string | null {
|
||||
return buildGpx(input, opts);
|
||||
}
|
||||
|
||||
filename(tripTitle: string): string {
|
||||
return gpxFilename(tripTitle);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* A KML or KMZ map file (Google My Maps, Google Earth), read through kml.codec
|
||||
* into its placemarks. The file type is the upload's extension, as it has
|
||||
* always been; anything else is refused with the route's own message.
|
||||
*/
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { readKmlDocument, unpackKmzToKml } from '../kml.codec';
|
||||
import type { KmlDocumentRead } from '../place-import.types';
|
||||
|
||||
@Injectable()
|
||||
export class KmlProvider {
|
||||
async readMapFile(fileBuffer: Buffer, filename: string): Promise<KmlDocumentRead> {
|
||||
const ext = filename.toLowerCase().split('.').pop();
|
||||
if (ext === 'kmz') return this.readKmz(fileBuffer);
|
||||
if (ext === 'kml') return this.readKml(fileBuffer);
|
||||
throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`);
|
||||
}
|
||||
|
||||
async readKmz(kmzBuffer: Buffer): Promise<KmlDocumentRead> {
|
||||
return this.readKml(await unpackKmzToKml(kmzBuffer));
|
||||
}
|
||||
|
||||
readKml(fileBuffer: Buffer): KmlDocumentRead {
|
||||
return readKmlDocument(fileBuffer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* A shared Naver Maps list (a bookmark folder), read page by page from Naver's
|
||||
* share endpoint.
|
||||
*
|
||||
* The link is checked by the SSRF guard and a naver.me short link is followed
|
||||
* hop by hop through it. Every page is capped before and after it is read: the
|
||||
* folder id is attacker-influenced, and the pager buffers a fresh body on every
|
||||
* iteration. Every refusal is the exact message and status the import route
|
||||
* has always answered.
|
||||
*/
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../../utils/ssrfGuard';
|
||||
import { LIST_USER_AGENT, MAX_LIST_RESPONSE_BYTES } from './google-list.provider';
|
||||
import type { ListImportError, ListRead, NaverListPlace } from '../place-import.types';
|
||||
|
||||
/** Bookmarks asked for per page. */
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
type NaverPage = {
|
||||
folder?: { bookmarkCount?: number; name?: string };
|
||||
bookmarkList?: Record<string, unknown>[];
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class NaverListProvider {
|
||||
async read(url: string): Promise<ListRead<NaverListPlace> | ListImportError> {
|
||||
let resolvedUrl = url;
|
||||
const limit = PAGE_SIZE;
|
||||
|
||||
// SSRF guard: validate user-supplied URL before fetching
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
// Resolve naver.me short links to the canonical map.naver.com folder URL.
|
||||
// Redirects are followed manually so each hop is re-validated against the
|
||||
// SSRF guard (a short link could otherwise 302 to an internal address).
|
||||
let parsedUrl: URL;
|
||||
try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
|
||||
if (parsedUrl.hostname === 'naver.me') {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i);
|
||||
const folderId = folderMatch?.[1] || null;
|
||||
if (!folderId) {
|
||||
return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 };
|
||||
}
|
||||
|
||||
const fetchPage = async (start: number) => {
|
||||
const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`;
|
||||
const apiRes = await fetch(apiUrl, {
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': LIST_USER_AGENT,
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!apiRes.ok) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
|
||||
// Same cap as the Google import: the URL is attacker-influenced via the
|
||||
// folder id, and this pager buffers a fresh body on every iteration, so an
|
||||
// uncapped read is worse here than there. The declared length is checked
|
||||
// before the read; the post-read check covers a chunked response that
|
||||
// carries no content-length at all.
|
||||
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
|
||||
if (declared > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawText = await apiRes.text();
|
||||
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
const data = JSON.parse(rawText) as NaverPage;
|
||||
return { data } as const;
|
||||
} catch {
|
||||
return { error: 'Invalid list data received from Naver Maps', status: 400 } as const;
|
||||
}
|
||||
};
|
||||
|
||||
const firstPage = await fetchPage(0);
|
||||
if ('error' in firstPage) {
|
||||
return { error: firstPage.error, status: firstPage.status };
|
||||
}
|
||||
|
||||
const listName = firstPage.data.folder?.name || 'Naver Maps List';
|
||||
const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number'
|
||||
? firstPage.data.folder.bookmarkCount
|
||||
: (firstPage.data.bookmarkList?.length || 0);
|
||||
|
||||
const allItems: Record<string, unknown>[] = [...(firstPage.data.bookmarkList || [])];
|
||||
for (let start = limit; start < totalCount; start += limit) {
|
||||
const page = await fetchPage(start);
|
||||
if ('error' in page) {
|
||||
return { error: page.error, status: page.status };
|
||||
}
|
||||
const pageItems = page.data.bookmarkList || [];
|
||||
if (!Array.isArray(pageItems) || pageItems.length === 0) break;
|
||||
allItems.push(...pageItems);
|
||||
}
|
||||
|
||||
if (allItems.length === 0) {
|
||||
return { error: 'List is empty or could not be read', status: 400 };
|
||||
}
|
||||
|
||||
const places: NaverListPlace[] = [];
|
||||
for (const item of allItems) {
|
||||
const lat = Number(item?.py);
|
||||
const lng = Number(item?.px);
|
||||
const name = typeof item?.name === 'string' && item.name.trim()
|
||||
? item.name.trim()
|
||||
: (typeof item?.displayName === 'string' ? item.displayName.trim() : '');
|
||||
const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null;
|
||||
const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null;
|
||||
|
||||
if (name && Number.isFinite(lat) && Number.isFinite(lng)) {
|
||||
places.push({ name, lat, lng, notes: note, address });
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length === 0) {
|
||||
return { error: 'No places with coordinates found in list', status: 400 };
|
||||
}
|
||||
|
||||
return { listName, places };
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ import { memoryStorage } from 'multer';
|
||||
import { hexColorSchema, placeImageUrlSchema, placeWebsiteSchema } from '@trek/shared';
|
||||
import type { User } from '../../types';
|
||||
import { PlacesService } from './places.service';
|
||||
import { isDirectionsUrl } from './maps-dir.helpers';
|
||||
import { isDirectionsUrl } from '../place-import/place-import.service';
|
||||
import { isUpdateConflict } from '../common/conflictResult';
|
||||
import { JwtAuthGuard } from '../auth-core/jwt-auth.guard';
|
||||
import { CurrentUser } from '../auth-core/current-user.decorator';
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import unzipper from 'unzipper';
|
||||
import {
|
||||
externalIdsOf,
|
||||
normalizePlaceName,
|
||||
@@ -8,7 +6,7 @@ import {
|
||||
} from '@trek/shared';
|
||||
import type { Place } from '../../types';
|
||||
import type { PlaceWithTagsRow as PlaceWithTags } from '../../db/repositories/Places.repository';
|
||||
import type { KmlImportSummary } from './kml-import.helpers';
|
||||
import type { KmlImportSummary } from '../place-import/place-import.types';
|
||||
import type { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
|
||||
import { haversineMetres } from '../common/geo';
|
||||
|
||||
@@ -17,36 +15,15 @@ import { haversineMetres } from '../common/geo';
|
||||
* out of the legacy services/placeService.ts when it went DI-native. Same
|
||||
* split as maps.helpers.ts / transit.helpers.ts / files.constants.ts: nothing
|
||||
* here touches the DB, so it stays plain exports rather than becoming methods
|
||||
* on PlacesService — which also keeps the frozen-at-import XML parsers a
|
||||
* single shared instance and lets the KMZ unpacker be unit-tested on its own.
|
||||
* on PlacesService. Reading files and shared lists (the XML parsers, the KMZ
|
||||
* unpacker, the list fetchers) lives in place-import/.
|
||||
*/
|
||||
|
||||
const gpxParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
isArray: (name) => ['wpt', 'trkpt', 'rtept', 'trk', 'trkseg', 'rte'].includes(name),
|
||||
});
|
||||
|
||||
const kmlParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
removeNSPrefix: true,
|
||||
isArray: (name) => ['Placemark', 'Folder', 'Document'].includes(name),
|
||||
// Treat <description> as raw text so mixed-content HTML (e.g. <br/>, <i>)
|
||||
// is returned as a string instead of a parsed object.
|
||||
stopNodes: ['*.description'],
|
||||
});
|
||||
|
||||
export { gpxParser, kmlParser };
|
||||
|
||||
export const KMZ_DECOMPRESSED_SIZE_LIMIT = 50 * 1024 * 1024; // 50 MB
|
||||
|
||||
// Re-exported so the importers that already read these from here keep working,
|
||||
// while the values themselves live in @trek/shared with the rule that uses them.
|
||||
export { COORD_DEDUP_TOLERANCE, externalIdsOf } from '@trek/shared';
|
||||
|
||||
/** Cap on a provider list-import response body — the payload is attacker-influenced via the list id. */
|
||||
export const MAX_LIST_RESPONSE_BYTES = 8 * 1024 * 1024; // 8 MB
|
||||
export type { GpxImportOptions, KmlImportOptions, ListImportError } from '../place-import/place-import.types';
|
||||
|
||||
/**
|
||||
* Escape the LIKE metacharacters in a user-supplied search term so `%` and `_`
|
||||
@@ -88,24 +65,6 @@ export interface ListImportResult {
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export interface ListImportError {
|
||||
error: string;
|
||||
status: number;
|
||||
}
|
||||
|
||||
export interface GpxImportOptions {
|
||||
importWaypoints?: boolean;
|
||||
importRoutes?: boolean;
|
||||
importTracks?: boolean;
|
||||
/** Source filename used to name unnamed routes/tracks (keeps multiple imports distinct). */
|
||||
defaultName?: string;
|
||||
}
|
||||
|
||||
export interface KmlImportOptions {
|
||||
importPoints?: boolean;
|
||||
importPaths?: boolean;
|
||||
}
|
||||
|
||||
// Reclaim a deleted place's cached marker photo if nothing else references it.
|
||||
// The cache key is the Google place_id, or — for coordinate-only places — the
|
||||
// pseudo-id embedded in the stored proxy URL (/api/maps/place-photo/{id}/bytes).
|
||||
@@ -173,41 +132,6 @@ export function trackInsertedInDedupSet(place: PlaceMatchCandidate, dedup: Dedup
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Google Maps list id parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function googleMapsHexId(value: unknown): string | null {
|
||||
if (typeof value !== 'string' && typeof value !== 'number') return null;
|
||||
const raw = String(value).trim();
|
||||
if (/^0x[0-9a-f]+$/i.test(raw)) return raw.toLowerCase();
|
||||
if (!/^-?\d+$/.test(raw)) return null;
|
||||
try {
|
||||
const parsed = BigInt(raw);
|
||||
const unsigned = parsed < 0n ? (1n << 64n) + parsed : parsed;
|
||||
return `0x${unsigned.toString(16)}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function googleMapsFeatureIdFromItem(item: unknown): string | null {
|
||||
if (!Array.isArray(item)) return null;
|
||||
const candidates = [
|
||||
Array.isArray(item[1]) ? item[1][6] : null,
|
||||
Array.isArray(item[7]) ? item[7][1] : null,
|
||||
];
|
||||
|
||||
for (const ids of candidates) {
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
const first = googleMapsHexId(ids[0]);
|
||||
const second = googleMapsHexId(ids[1]);
|
||||
if (first && second) return `${first}:${second}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Import enrichment (#886) — the pure half of the former
|
||||
// services/placeEnrichment.ts; the DB/websocket/Maps half is PlacesService.
|
||||
@@ -283,30 +207,24 @@ export async function mapWithConcurrency<T>(items: T[], limit: number, fn: (item
|
||||
export const trimOrNull = (v: unknown): string | null => (typeof v === 'string' && v.trim() ? v.trim() : null);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// KMZ unpacking
|
||||
// KML folder → trip category
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function unpackKmzToKml(
|
||||
kmzBuffer: Buffer,
|
||||
decompressedSizeLimit = KMZ_DECOMPRESSED_SIZE_LIMIT,
|
||||
): Promise<Buffer> {
|
||||
let zip;
|
||||
try {
|
||||
zip = await unzipper.Open.buffer(kmzBuffer);
|
||||
} catch {
|
||||
throw new Error('Invalid KMZ archive.');
|
||||
export function buildCategoryNameLookup(categories: { id: number; name: string }[]): Map<string, number> {
|
||||
const lookup = new Map<string, number>();
|
||||
for (const category of categories) {
|
||||
const normalizedName = category.name.trim().toLowerCase();
|
||||
if (!normalizedName) continue;
|
||||
if (!lookup.has(normalizedName)) {
|
||||
lookup.set(normalizedName, category.id);
|
||||
}
|
||||
}
|
||||
|
||||
const kmlEntries = zip.files.filter((entry) => !entry.path.endsWith('/') && entry.path.toLowerCase().endsWith('.kml'));
|
||||
if (kmlEntries.length === 0) {
|
||||
throw new Error('KMZ archive does not contain a KML file.');
|
||||
}
|
||||
|
||||
const preferredEntry = kmlEntries.find((entry) => entry.path.toLowerCase().endsWith('doc.kml')) || kmlEntries[0];
|
||||
|
||||
if (preferredEntry.uncompressedSize > decompressedSizeLimit) {
|
||||
throw new Error('KMZ archive exceeds the maximum allowed decompressed size.');
|
||||
}
|
||||
|
||||
return preferredEntry.buffer();
|
||||
return lookup;
|
||||
}
|
||||
|
||||
export function resolveCategoryIdForFolder(folderName: string | null, lookup: Map<string, number>): number | null {
|
||||
if (!folderName) return null;
|
||||
const normalizedFolder = folderName.trim().toLowerCase();
|
||||
if (!normalizedFolder) return null;
|
||||
return lookup.get(normalizedFolder) ?? null;
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { MapsService } from '../maps/maps.service';
|
||||
import { PlacesService } from './places.service';
|
||||
import { isUpdateConflict } from '../common/conflictResult';
|
||||
import { isDirectionsUrl } from './maps-dir.helpers';
|
||||
import { isDirectionsUrl } from '../place-import/place-import.service';
|
||||
import { TripAccessService } from '../trip-membership/trip-access.service';
|
||||
|
||||
function parseId(value: string | string[]): number | null {
|
||||
|
||||
@@ -24,6 +24,7 @@ import { UnsplashModule } from '../unsplash/unsplash.module';
|
||||
import { PlacePhotosModule } from '../place-photos/place-photos.module';
|
||||
import { QueryHelpersModule } from '../query-helpers/query-helpers.module';
|
||||
import { MapsModule } from '../maps/maps.module';
|
||||
import { PlaceImportModule } from '../place-import/place-import.module';
|
||||
import { McpSharedModule } from '../mcp-shared/mcp-shared.module';
|
||||
import { MulterModule } from '@nestjs/platform-express';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
@@ -55,6 +56,8 @@ import { TripMembershipModule } from '../trip-membership/trip-membership.module'
|
||||
StorageModule,
|
||||
// AccommodationsDomainModule: deleting a place takes the nights booked at it with
|
||||
// it, and the cascade behind a stay belongs to the domain that owns it.
|
||||
// PlaceImportModule reads the GPX/KML files and shared lists this domain imports.
|
||||
PlaceImportModule,
|
||||
McpSharedModule, PermissionsModule, QueryHelpersModule, MapsModule, AppConfigModule, UnsplashModule, PlacePhotosModule, JourneyDomainModule, RealtimeModule, PluginGuardsModule, AssignmentsDomainModule, AccommodationsDomainModule,
|
||||
// Plan 3c Task 4: `PlacesService`'s and `PlacesMcp`'s own `@InjectRepository`
|
||||
// constructor params (`PlacesRepository`, `TagsRepository`,
|
||||
|
||||
@@ -2,30 +2,20 @@ import path from 'node:path';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import { resolveCountryCodeSync } from '../atlas/atlas-geo';
|
||||
import { XMLValidator } from 'fast-xml-parser';
|
||||
import { TRACK_COLORS, placeMatchStrategies, type PlaceMatchCandidate } from '@trek/shared';
|
||||
import type { TrekWsPayload, TrekWsTripEventName } from '@trek/shared';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import type { TripAccess } from '../../db/repositories/Trips.repository';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { PermissionsService } from '../permissions/permissions.service';
|
||||
import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service';
|
||||
import { isDirectionsUrl, parseDirectionsUrl } from './maps-dir.helpers';
|
||||
import { MapsService } from '../maps/maps.service';
|
||||
import { PlaceImportService } from '../place-import/place-import.service';
|
||||
import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from '../place-import/place-import.service';
|
||||
import type { GoogleListPlace, KmlDocumentRead, PreparedGpxPlace } from '../place-import/place-import.types';
|
||||
import { toRowId } from '../common/row-id';
|
||||
import type { User } from '../../types';
|
||||
import { QueryHelpersService } from '../query-helpers/query-helpers.service';
|
||||
import { ratingAggregate } from '../common/rowShape';
|
||||
import { checkSsrf, safeFetchFollow, SsrfBlockedError } from '../../utils/ssrfGuard';
|
||||
import {
|
||||
buildCategoryNameLookup,
|
||||
createKmlImportSummary,
|
||||
decodeUtf8WithWarning,
|
||||
extractKmlPlacemarkNodes,
|
||||
parsePlacemarkNode,
|
||||
resolveCategoryIdForFolder,
|
||||
} from './kml-import.helpers';
|
||||
import { buildGpx, gpxFilename } from './gpx-export.helpers';
|
||||
import type { GpxExportDay, GpxExportOptions, GpxExportPlace } from './gpx-export.helpers';
|
||||
import { UnsplashService } from '../unsplash/unsplash.service';
|
||||
import { PlacePhotoCacheService } from '../place-photos/place-photo-cache.service';
|
||||
import { type UpdateConflict, isUpdateConflict } from '../common/conflictResult';
|
||||
@@ -64,20 +54,17 @@ const noCancelledStays = (): CancelledStays => ({ reservationIds: [], budgetItem
|
||||
import {
|
||||
ENRICH_CONCURRENCY,
|
||||
ADDRESS_BACKFILL_MAX_PLACES,
|
||||
buildCategoryNameLookup,
|
||||
escapeLikePattern,
|
||||
MAX_LIST_RESPONSE_BYTES,
|
||||
googleMapsFeatureIdFromItem,
|
||||
gpxParser,
|
||||
externalIdsOf,
|
||||
isPlaceDuplicate,
|
||||
kmlParser,
|
||||
mapWithConcurrency,
|
||||
pickEnrichmentMatch,
|
||||
reclaimPhotoCache,
|
||||
SEARCH_BIAS_RADIUS_METERS,
|
||||
resolveCategoryIdForFolder,
|
||||
trackInsertedInDedupSet,
|
||||
trimOrNull,
|
||||
unpackKmzToKml,
|
||||
type DedupSet,
|
||||
type EnrichablePlace,
|
||||
type GpxImportOptions,
|
||||
@@ -93,14 +80,7 @@ type Trip = TripAccess;
|
||||
|
||||
type ImportedPlace = { id: number; route_geometry?: string | null; route_color?: string | null };
|
||||
|
||||
/** One place parsed out of a GPX file, not yet persisted. */
|
||||
export interface PreparedGpxPlace {
|
||||
name: string;
|
||||
lat: number;
|
||||
lng: number;
|
||||
description: string | null;
|
||||
routeGeometry?: string;
|
||||
}
|
||||
export type { PreparedGpxPlace } from '../place-import/place-import.types';
|
||||
|
||||
/** Fields accepted when creating a place. */
|
||||
export interface PlaceCreateInput {
|
||||
@@ -187,6 +167,8 @@ export class PlacesService {
|
||||
@InjectRepository(BudgetItems) private readonly budgetItemsRepo: BudgetItemsRepository,
|
||||
// Plan 3h Task 6 (survivors) — additive, SV-PI2's `reclaimPlaceImage` only.
|
||||
@InjectRepository(CollectionPlaces) private readonly collectionPlacesRepo: CollectionPlacesRepository,
|
||||
// Reading files and shared lists; this service only persists what it reads.
|
||||
private readonly placeImport: PlaceImportService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
@@ -903,95 +885,18 @@ export class PlacesService {
|
||||
day.points.push({ name: stop.name, lat: stop.lat, lng: stop.lng });
|
||||
}
|
||||
|
||||
const gpx = buildGpx({ tripTitle: title, places, days: [...days.values()] }, opts);
|
||||
return gpx ? { gpx, filename: gpxFilename(title) } : null;
|
||||
const gpx = this.placeImport.writeTripGpx({ tripTitle: title, places, days: [...days.values()] }, opts);
|
||||
return gpx ? { gpx, filename: this.placeImport.tripGpxFilename(title) } : null;
|
||||
}
|
||||
|
||||
private async importGpxRows(tripId: string, fileBuffer: Buffer, opts: GpxImportOptions = {}): Promise<GpxImportResult | null> {
|
||||
const rows = this.prepareGpxRows(fileBuffer, opts);
|
||||
const rows = this.placeImport.readGpx(fileBuffer, opts);
|
||||
if (!rows.length) return null;
|
||||
return await this.uow.transactional(() => this.persistGpxRows(tripId, rows));
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a GPX file into the places an import would create, without writing
|
||||
* anything. The Tours import reuses it to build its own selection before
|
||||
* handing the rows to {@link importPreparedGpx}.
|
||||
*/
|
||||
prepareGpxRows(fileBuffer: Buffer, opts: GpxImportOptions = {}): PreparedGpxPlace[] {
|
||||
const { importWaypoints = true, importRoutes = true, importTracks = true, defaultName } = opts;
|
||||
|
||||
const parsed = gpxParser.parse(fileBuffer.toString('utf-8'));
|
||||
const gpx = parsed?.gpx;
|
||||
if (!gpx) return [];
|
||||
|
||||
const str = (v: unknown) => (v != null ? String(v).trim() : null);
|
||||
const num = (v: unknown) => { const n = Number.parseFloat(String(v)); return Number.isNaN(n) ? null : n; };
|
||||
|
||||
// Routes and tracks rarely carry their own <name>. Without one they all fall back to the
|
||||
// same generic label, so name-based dedup drops every import after the first. Derive a
|
||||
// base from the source filename (the requested behaviour) and suffix an index so multiple
|
||||
// geometries from one file stay distinct.
|
||||
const rawName = str(defaultName);
|
||||
const baseName = rawName ? rawName.replace(/\.[^.]+$/, '').trim() || rawName : null;
|
||||
let geoSeq = 0;
|
||||
const geoName = (explicit: string | null, fallback: string): string => {
|
||||
if (explicit) return explicit;
|
||||
geoSeq++;
|
||||
const base = baseName || fallback;
|
||||
return geoSeq === 1 ? base : `${base} ${geoSeq}`;
|
||||
};
|
||||
|
||||
const waypoints: PreparedGpxPlace[] = [];
|
||||
|
||||
// 1) Parse <wpt> elements (named waypoints / POIs)
|
||||
if (importWaypoints) {
|
||||
for (const wpt of gpx.wpt ?? []) {
|
||||
const lat = num(wpt['@_lat']);
|
||||
const lng = num(wpt['@_lon']);
|
||||
if (lat === null || lng === null) continue;
|
||||
waypoints.push({ lat, lng, name: str(wpt.name) || `Waypoint ${waypoints.length + 1}`, description: str(wpt.desc) });
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Parse <rte> routes as polyline-places (one place per route with route_geometry)
|
||||
if (importRoutes) {
|
||||
for (const rte of gpx.rte ?? []) {
|
||||
const pts = (rte.rtept ?? [])
|
||||
.map((pt: Record<string, unknown>) => ({ lat: num(pt['@_lat']), lng: num(pt['@_lon']), ele: num(pt['ele']) }))
|
||||
.filter((p: { lat: number | null; lng: number | null; ele: number | null }) => p.lat !== null && p.lng !== null) as Array<{ lat: number; lng: number; ele: number | null }>;
|
||||
if (pts.length === 0) continue;
|
||||
const hasAllEle = pts.every(p => p.ele !== null);
|
||||
const routeGeometry = pts.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
|
||||
waypoints.push({ lat: pts[0].lat, lng: pts[0].lng, name: geoName(str(rte.name), 'GPX Route'), description: str(rte.desc), routeGeometry: JSON.stringify(routeGeometry) });
|
||||
}
|
||||
}
|
||||
|
||||
// 3) Extract full track geometry from <trk>
|
||||
if (importTracks) {
|
||||
for (const trk of gpx.trk ?? []) {
|
||||
const trackPoints: { lat: number; lng: number; ele: number | null }[] = [];
|
||||
for (const seg of trk.trkseg ?? []) {
|
||||
for (const pt of seg.trkpt ?? []) {
|
||||
const lat = num(pt['@_lat']);
|
||||
const lng = num(pt['@_lon']);
|
||||
if (lat === null || lng === null) continue;
|
||||
trackPoints.push({ lat, lng, ele: num(pt.ele) });
|
||||
}
|
||||
}
|
||||
if (trackPoints.length === 0) continue;
|
||||
const start = trackPoints[0];
|
||||
const hasAllEle = trackPoints.every(p => p.ele !== null);
|
||||
const routeGeometry = trackPoints.map(p => hasAllEle ? [p.lat, p.lng, p.ele] : [p.lat, p.lng]);
|
||||
waypoints.push({ lat: start.lat, lng: start.lng, name: geoName(str(trk.name), 'GPX Track'), description: str(trk.desc), routeGeometry: JSON.stringify(routeGeometry) });
|
||||
}
|
||||
}
|
||||
|
||||
return waypoints;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persists rows from {@link prepareGpxRows} and colours their tracks, both in
|
||||
* Persists rows PlaceImportService.readGpx prepared and colours their tracks, both in
|
||||
* one transaction. Called from inside the Tours import's own transaction,
|
||||
* where this becomes a savepoint, so the places and their tour facets
|
||||
* commit or roll back together.
|
||||
@@ -1080,39 +985,21 @@ export class PlacesService {
|
||||
}
|
||||
|
||||
private async importMapFileRows(tripId: string, fileBuffer: Buffer, filename: string, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
|
||||
const ext = filename.toLowerCase().split('.').pop();
|
||||
if (ext === 'kmz') return this.importKmzPlaces(tripId, fileBuffer, opts);
|
||||
if (ext === 'kml') return this.importKmlPlaces(tripId, fileBuffer, opts);
|
||||
throw new Error(`Unsupported map file format: .${ext}. Please upload a .kml or .kmz file.`);
|
||||
return this.persistKmlPlaces(tripId, await this.placeImport.readMapFile(fileBuffer, filename), opts);
|
||||
}
|
||||
|
||||
async importKmzPlaces(tripId: string, kmzBuffer: Buffer, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
|
||||
const kmlBuffer = await unpackKmzToKml(kmzBuffer);
|
||||
return this.importKmlPlaces(tripId, kmlBuffer, opts);
|
||||
return this.persistKmlPlaces(tripId, await this.placeImport.readKmz(kmzBuffer), opts);
|
||||
}
|
||||
|
||||
async importKmlPlaces(tripId: string, fileBuffer: Buffer, opts: KmlImportOptions = {}): Promise<PlaceImportResult> {
|
||||
return this.persistKmlPlaces(tripId, this.placeImport.readKml(fileBuffer), opts);
|
||||
}
|
||||
|
||||
/** The placemarks of a read KML document as places, skipping duplicates. */
|
||||
private async persistKmlPlaces(tripId: string, read: KmlDocumentRead, opts: KmlImportOptions): Promise<PlaceImportResult> {
|
||||
const { importPoints = true, importPaths = true } = opts;
|
||||
const decoded = decodeUtf8WithWarning(fileBuffer);
|
||||
|
||||
const validationResult = XMLValidator.validate(decoded.text);
|
||||
if (validationResult !== true) {
|
||||
throw new Error('Malformed KML: invalid XML structure');
|
||||
}
|
||||
|
||||
const parsed = kmlParser.parse(decoded.text);
|
||||
const kmlRoot = parsed?.kml ?? parsed;
|
||||
|
||||
if (!kmlRoot || typeof kmlRoot !== 'object') {
|
||||
throw new Error('Malformed KML: could not parse XML');
|
||||
}
|
||||
|
||||
const placemarkNodes = extractKmlPlacemarkNodes(kmlRoot);
|
||||
const summary = createKmlImportSummary(placemarkNodes.length);
|
||||
|
||||
if (decoded.warning) {
|
||||
summary.warnings.push(decoded.warning);
|
||||
}
|
||||
const { placemarks, summary } = read;
|
||||
|
||||
// PL33 — `CategoriesRepository.listIdName` (Plan 3a's repository).
|
||||
const categories = await this.categoriesRepo.listIdName();
|
||||
@@ -1126,8 +1013,7 @@ export class PlacesService {
|
||||
|
||||
await this.uow.transactional(async () => {
|
||||
let fallbackIndex = 1;
|
||||
for (const node of placemarkNodes) {
|
||||
const parsedPlacemark = parsePlacemarkNode(node);
|
||||
for (const parsedPlacemark of placemarks) {
|
||||
const isPath = parsedPlacemark.routeGeometry !== null;
|
||||
|
||||
// Unsupported geometry type (polygon, multi-geometry, no geometry, etc.)
|
||||
@@ -1259,124 +1145,12 @@ export class PlacesService {
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
async importGoogleList(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
|
||||
let listId: string | null = null;
|
||||
let resolvedUrl = url;
|
||||
|
||||
// SSRF guard: validate user-supplied URL before fetching
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
// Follow redirects for short URLs (maps.app.goo.gl, goo.gl). Redirects are
|
||||
// followed manually so every hop is re-checked against the SSRF guard — a
|
||||
// short link that 302s to an internal IP is blocked even though the initial
|
||||
// host is public.
|
||||
if (url.includes('goo.gl') || url.includes('maps.app')) {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// A route, once the redirect is followed. The dispatch upstream decides on
|
||||
// the raw URL, and a short link's path is `/<code>` — it matches nothing, so
|
||||
// every route shared from the Google Maps app arrived here and was answered
|
||||
// with "could not extract list ID", which is the complaint the directions
|
||||
// import was written to remove. The Share sheet on a phone produces exactly
|
||||
// this shape, and the box says a directions link works.
|
||||
//
|
||||
// Handed on with the resolved URL, so the hop is not made twice.
|
||||
if (isDirectionsUrl(resolvedUrl)) {
|
||||
return this.importGoogleDirections(tripId, resolvedUrl, opts);
|
||||
}
|
||||
|
||||
// Pattern: /placelists/list/{ID}
|
||||
const plMatch = resolvedUrl.match(/placelists\/list\/([A-Za-z0-9_-]+)/);
|
||||
if (plMatch) listId = plMatch[1];
|
||||
|
||||
// Pattern: !2s{ID} in data URL params
|
||||
if (!listId) {
|
||||
const dataMatch = resolvedUrl.match(/!2s([A-Za-z0-9_-]{15,})/);
|
||||
if (dataMatch) listId = dataMatch[1];
|
||||
}
|
||||
|
||||
if (!listId) {
|
||||
// A single-place share link (…/maps/place/…) carries no list id — point the user at
|
||||
// the place search box instead of a cryptic "could not extract list ID" (#1304).
|
||||
if (resolvedUrl.includes('/maps/place/')) {
|
||||
return { error: 'That link points to a single place, not a list. To add it, paste the link into the place search box instead of using the list import.', status: 400 };
|
||||
}
|
||||
return { error: 'Could not extract list ID from URL. Please use a shared Google Maps list link.', status: 400 };
|
||||
}
|
||||
|
||||
// Fetch list data from Google Maps internal API
|
||||
const apiUrl = `https://www.google.com/maps/preview/entitylist/getlist?authuser=0&hl=en&gl=us&pb=!1m1!1s${encodeURIComponent(listId)}!2e2!3e2!4i500!16b1`;
|
||||
const apiRes = await fetch(apiUrl, {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' },
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!apiRes.ok) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
|
||||
// Cap the declared body before reading it (transit.service precedent): the
|
||||
// response is attacker-influenced via the list id, and buffering it whole
|
||||
// used to be unbounded.
|
||||
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
|
||||
if (declared > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
|
||||
const rawText = await apiRes.text();
|
||||
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Google Maps', status: 502 };
|
||||
}
|
||||
const jsonStr = rawText.substring(rawText.indexOf('\n') + 1);
|
||||
// The provider hands back a JS-prefixed array; a malformed body is a
|
||||
// provider problem, not a crash — surface the same 400 an unreadable
|
||||
// payload already produced instead of throwing a SyntaxError.
|
||||
let listData: unknown;
|
||||
try {
|
||||
listData = JSON.parse(jsonStr);
|
||||
} catch {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
if (!Array.isArray(listData)) {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
|
||||
const meta = listData[0];
|
||||
if (!meta) {
|
||||
return { error: 'Invalid list data received from Google Maps', status: 400 };
|
||||
}
|
||||
|
||||
const listName = meta[4] || 'Google Maps List';
|
||||
const items = meta[8];
|
||||
|
||||
if (!Array.isArray(items) || items.length === 0) {
|
||||
return { error: 'List is empty or could not be read', status: 400 };
|
||||
}
|
||||
|
||||
// Parse place data from items
|
||||
const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = [];
|
||||
for (const item of items) {
|
||||
const coords = item?.[1]?.[5];
|
||||
const lat = coords?.[2];
|
||||
const lng = coords?.[3];
|
||||
const name = item?.[2];
|
||||
const note = item?.[3] || null;
|
||||
|
||||
if (name && typeof lat === 'number' && typeof lng === 'number' && !Number.isNaN(lat) && !Number.isNaN(lng)) {
|
||||
places.push({ name, lat, lng, notes: note || null, googleFtid: googleMapsFeatureIdFromItem(item) });
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length === 0) {
|
||||
return { error: 'No places with coordinates found in list', status: 400 };
|
||||
}
|
||||
const read = await this.placeImport.readGoogleList(url);
|
||||
if ('error' in read) return read;
|
||||
// A short link that lands on a route is a directions link: imported as one,
|
||||
// with the resolved URL, so the hop is not made twice.
|
||||
if ('directions' in read) return this.importGoogleDirections(tripId, read.directions, opts);
|
||||
const { listName, places } = read;
|
||||
|
||||
const { created, skipped } = await this.storeGooglePlaces(tripId, places);
|
||||
|
||||
@@ -1396,7 +1170,7 @@ export class PlacesService {
|
||||
*/
|
||||
private async storeGooglePlaces(
|
||||
tripId: string,
|
||||
places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[],
|
||||
places: GoogleListPlace[],
|
||||
): Promise<{ created: PlaceWithTags[]; skipped: number }> {
|
||||
// Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the
|
||||
// caller's `requireTrip` gate already `toRowId`-parsed this `tripId`.
|
||||
@@ -1469,90 +1243,13 @@ export class PlacesService {
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* The stops of a route somebody else planned.
|
||||
*
|
||||
* The other half of the request the list import answers: people share a drive far more
|
||||
* often than they share a list, and until now a pasted `/maps/dir/` link came back as a
|
||||
* cryptic "could not extract list ID". No API key is involved and no call is made to
|
||||
* Google for the link itself — the stops are in the URL, which is the whole reason
|
||||
* this is possible at all.
|
||||
*
|
||||
* A stop the link spells out in coordinates is taken as it stands; one that is only a
|
||||
* name is geocoded, one request each. A name nobody can place is left out rather than
|
||||
* failing the import, because a route of six stops with five findable is five stops
|
||||
* more than the traveller had.
|
||||
* The stops of a route somebody else planned, read and placed by
|
||||
* PlaceImportService (no API key, the stops are in the link), stored here.
|
||||
*/
|
||||
async importGoogleDirections(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
let parsed: URL;
|
||||
try { parsed = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
|
||||
|
||||
// Short links are resolved hop by hop through the guard, exactly as the list import
|
||||
// does it: a maps.app.goo.gl that 302s to an internal address is still blocked.
|
||||
let resolvedUrl = url;
|
||||
if (GOOGLE_SHORT_HOSTS.includes(parsed.hostname)) {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Checked after resolving, not before: the host that counts is the one the link lands
|
||||
// on, and `/maps/dir/` is a path anybody could serve.
|
||||
let host = '';
|
||||
try { host = new URL(resolvedUrl).hostname; } catch { /* an unparseable hop fails the check below */ }
|
||||
if (!isGoogleMapsHost(host)) {
|
||||
return { error: 'That link is not a Google Maps link.', status: 400 };
|
||||
}
|
||||
|
||||
const waypoints = parseDirectionsUrl(resolvedUrl);
|
||||
if (waypoints.length < 2) {
|
||||
return { error: 'Could not read any stops from that directions link. Open the route in Google Maps and use its Share button.', status: 400 };
|
||||
}
|
||||
|
||||
const places: { name: string; lat: number; lng: number; notes: string | null; googleFtid: string | null }[] = [];
|
||||
let unplaceable = 0;
|
||||
for (const wp of waypoints) {
|
||||
if (wp.lat !== null && wp.lng !== null) {
|
||||
// A stop written as coordinates has no name of its own, and the coordinates are a
|
||||
// better label than "Stop 3": they are what the traveller can look up.
|
||||
places.push({
|
||||
name: wp.name || `${wp.lat.toFixed(5)}, ${wp.lng.toFixed(5)}`,
|
||||
lat: wp.lat,
|
||||
lng: wp.lng,
|
||||
notes: null,
|
||||
googleFtid: null,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!wp.name) continue;
|
||||
try {
|
||||
// Through geocodeQuery, which asks the TREK index first and only falls
|
||||
// through to Nominatim for what it does not know — and does so on the
|
||||
// BACKGROUND lane. That matters here more than anywhere: this loop runs
|
||||
// up to thirty times in one request, each Nominatim call taking the next
|
||||
// slot on a 1.1 s process-wide throttle, so on the interactive lane one
|
||||
// pasted link made everybody else's place search queue behind it for
|
||||
// half a minute. An index hit costs no slot at all.
|
||||
const hit = await this.maps.geocodeQuery(wp.name);
|
||||
// The name from the link, not the one the geocoder answers with: somebody who
|
||||
// typed a nickname into Google should not find a street address on their trip.
|
||||
if (hit) places.push({ name: wp.name, lat: hit.lat, lng: hit.lng, notes: null, googleFtid: null });
|
||||
else unplaceable++;
|
||||
} catch {
|
||||
// A geocoder that is down or rate-limited costs this one stop, not the import.
|
||||
unplaceable++;
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length < 2) {
|
||||
return { error: 'None of the stops in that link could be placed on the map.', status: 400 };
|
||||
}
|
||||
const read = await this.placeImport.readGoogleDirections(url, (query) => this.maps.geocodeQuery(query));
|
||||
if ('error' in read) return read;
|
||||
const { places, unplaceable } = read;
|
||||
|
||||
const { created, skipped } = await this.storeGooglePlaces(tripId, places);
|
||||
if (created.length) {
|
||||
@@ -1571,116 +1268,9 @@ export class PlacesService {
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
async importNaverList(tripId: string, url: string, opts?: ListImportOptions): Promise<ListImportResult | ListImportError> {
|
||||
let resolvedUrl = url;
|
||||
const limit = 20;
|
||||
|
||||
// SSRF guard: validate user-supplied URL before fetching
|
||||
const ssrf = await checkSsrf(url);
|
||||
if (!ssrf.allowed) return { error: 'URL is not allowed', status: 400 };
|
||||
|
||||
// Resolve naver.me short links to the canonical map.naver.com folder URL.
|
||||
// Redirects are followed manually so each hop is re-validated against the
|
||||
// SSRF guard (a short link could otherwise 302 to an internal address).
|
||||
let parsedUrl: URL;
|
||||
try { parsedUrl = new URL(url); } catch { return { error: 'Invalid URL', status: 400 }; }
|
||||
if (parsedUrl.hostname === 'naver.me') {
|
||||
try {
|
||||
const redirectRes = await safeFetchFollow(url, { signal: AbortSignal.timeout(10000) });
|
||||
resolvedUrl = redirectRes.url;
|
||||
} catch (err) {
|
||||
if (err instanceof SsrfBlockedError) return { error: 'URL is not allowed', status: 400 };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const folderMatch = resolvedUrl.match(/favorite\/myPlace\/folder\/([A-Za-z0-9_-]+)/i);
|
||||
const folderId = folderMatch?.[1] || null;
|
||||
if (!folderId) {
|
||||
return { error: 'Could not extract folder ID from URL. Please use a shared Naver Maps list link.', status: 400 };
|
||||
}
|
||||
|
||||
const fetchPage = async (start: number) => {
|
||||
const apiUrl = `https://pages.map.naver.com/save-pages/api/maps-bookmark/v3/shares/${encodeURIComponent(folderId)}/bookmarks?placeInfo=true&start=${start}&limit=${limit}&sort=lastUseTime&mcids=ALL&createIdNo=true`;
|
||||
const apiRes = await fetch(apiUrl, {
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
|
||||
if (!apiRes.ok) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
|
||||
// Same cap as the Google import: the URL is attacker-influenced via the
|
||||
// folder id, and this pager buffers a fresh body on every iteration, so an
|
||||
// uncapped read is worse here than there. The declared length is checked
|
||||
// before the read; the post-read check covers a chunked response that
|
||||
// carries no content-length at all.
|
||||
const declared = Number(apiRes.headers?.get('content-length') ?? 0);
|
||||
if (declared > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
|
||||
try {
|
||||
const rawText = await apiRes.text();
|
||||
if (rawText.length > MAX_LIST_RESPONSE_BYTES) {
|
||||
return { error: 'Failed to fetch list from Naver Maps', status: 502 } as const;
|
||||
}
|
||||
const data = JSON.parse(rawText) as {
|
||||
folder?: { bookmarkCount?: number; name?: string };
|
||||
bookmarkList?: Record<string, unknown>[];
|
||||
};
|
||||
return { data } as const;
|
||||
} catch {
|
||||
return { error: 'Invalid list data received from Naver Maps', status: 400 } as const;
|
||||
}
|
||||
};
|
||||
|
||||
const firstPage = await fetchPage(0);
|
||||
if ('error' in firstPage) {
|
||||
return { error: firstPage.error, status: firstPage.status };
|
||||
}
|
||||
|
||||
const listName = firstPage.data.folder?.name || 'Naver Maps List';
|
||||
const totalCount = typeof firstPage.data.folder?.bookmarkCount === 'number'
|
||||
? firstPage.data.folder.bookmarkCount
|
||||
: (firstPage.data.bookmarkList?.length || 0);
|
||||
|
||||
const allItems: Record<string, unknown>[] = [...(firstPage.data.bookmarkList || [])];
|
||||
for (let start = limit; start < totalCount; start += limit) {
|
||||
const page = await fetchPage(start);
|
||||
if ('error' in page) {
|
||||
return { error: page.error, status: page.status };
|
||||
}
|
||||
const pageItems = page.data.bookmarkList || [];
|
||||
if (!Array.isArray(pageItems) || pageItems.length === 0) break;
|
||||
allItems.push(...pageItems);
|
||||
}
|
||||
|
||||
if (allItems.length === 0) {
|
||||
return { error: 'List is empty or could not be read', status: 400 };
|
||||
}
|
||||
|
||||
const places: { name: string; lat: number; lng: number; notes: string | null; address: string | null }[] = [];
|
||||
for (const item of allItems) {
|
||||
const lat = Number(item?.py);
|
||||
const lng = Number(item?.px);
|
||||
const name = typeof item?.name === 'string' && item.name.trim()
|
||||
? item.name.trim()
|
||||
: (typeof item?.displayName === 'string' ? item.displayName.trim() : '');
|
||||
const note = typeof item?.memo === 'string' && item.memo.trim() ? item.memo.trim() : null;
|
||||
const address = typeof item?.address === 'string' && item.address.trim() ? item.address.trim() : null;
|
||||
|
||||
if (name && Number.isFinite(lat) && Number.isFinite(lng)) {
|
||||
places.push({ name, lat, lng, notes: note, address });
|
||||
}
|
||||
}
|
||||
|
||||
if (places.length === 0) {
|
||||
return { error: 'No places with coordinates found in list', status: 400 };
|
||||
}
|
||||
const read = await this.placeImport.readNaverList(url);
|
||||
if ('error' in read) return read;
|
||||
const { listName, places } = read;
|
||||
|
||||
// Rule 21 / M1 (Task 9 fix wave) — same ruling as `importGpxRows`: the
|
||||
// caller's `requireTrip` gate already `toRowId`-parsed this `tripId`.
|
||||
|
||||
@@ -2,7 +2,7 @@ import { HttpException, Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@mikro-orm/nestjs';
|
||||
import type { GoogleRouteImport, GoogleRoutePreview } from '@trek/shared';
|
||||
import { MapsService, GOOGLE_SHORT_HOSTS, isGoogleMapsHost } from '../maps/maps.service';
|
||||
import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../places/maps-dir.helpers';
|
||||
import { isDirectionsUrl, parseDirectionsUrl, MAX_DIR_WAYPOINTS } from '../place-import/place-import.service';
|
||||
import { safeFetchFollow } from '../../utils/ssrfGuard';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { PlacesService } from '../places/places.service';
|
||||
|
||||
@@ -8,12 +8,13 @@ import { ToursController } from './tours.controller';
|
||||
import { ToursImportController } from './tours-import.controller';
|
||||
import { ToursService } from './tours.service';
|
||||
import { PlacesModule } from '../places/places.module';
|
||||
import { PlaceImportModule } from '../place-import/place-import.module';
|
||||
import { PermissionsModule } from '../permissions/permissions.module';
|
||||
import { AddonsModule } from '../addons/addons.module';
|
||||
|
||||
/**
|
||||
* Tours domain: an isolated bounded context that imports PlacesModule to reuse
|
||||
* GPX preparation and persistence through PlacesService. The tour SQL lives in
|
||||
* Tours domain: an isolated bounded context that reads GPX through PlaceImportModule and
|
||||
* persists the places through PlacesService (PlacesModule). The tour SQL lives in
|
||||
* the Tours and TourWaypoints repositories; the owning place's route columns go
|
||||
* through PlacesRepository, registered here because forFeature only reaches
|
||||
* this module's own providers.
|
||||
@@ -21,6 +22,7 @@ import { AddonsModule } from '../addons/addons.module';
|
||||
@Module({
|
||||
imports: [
|
||||
PlacesModule,
|
||||
PlaceImportModule,
|
||||
PermissionsModule,
|
||||
|
||||
AddonsModule,
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { TourWaypointsRepository } from '../../db/repositories/TourWaypoint
|
||||
import { toRowId } from '../common/row-id';
|
||||
import { UnitOfWork } from '../database/unit-of-work';
|
||||
import { PlacesService } from '../places/places.service';
|
||||
import { PlaceImportService } from '../place-import/place-import.service';
|
||||
import { computeTourMetrics, parseRouteGeometry, LOW_CONFIDENCE_THRESHOLD, type GeometryPoint } from './tours.helpers';
|
||||
|
||||
export interface ImportGpxAsTourResult {
|
||||
@@ -23,8 +24,8 @@ export interface ImportGpxAsTourResult {
|
||||
/**
|
||||
* Tours domain: the `tours` facet table (place_id PK/FK) carries tour-specific
|
||||
* metadata while existing places and day assignments are reused.
|
||||
* GPX parsing and persistence are reused through PlacesService.prepareGpxRows
|
||||
* and importPreparedGpx. Places and facet rows are persisted in one transaction,
|
||||
* GPX parsing goes through PlaceImportService.readGpx, persistence through
|
||||
* PlacesService.importPreparedGpx. Places and facet rows are persisted in one transaction,
|
||||
* with tour metrics derived from each place's route_geometry.
|
||||
*/
|
||||
@Injectable()
|
||||
@@ -38,6 +39,7 @@ export class ToursService {
|
||||
@InjectRepository(TourTypes) private readonly tourTypesRepo: TourTypesRepository,
|
||||
@InjectRepository(TourWaypoints) private readonly waypointsRepo: TourWaypointsRepository,
|
||||
@InjectRepository(Places) private readonly placesRepo: PlacesRepository,
|
||||
private readonly placeImport: PlaceImportService,
|
||||
) {}
|
||||
|
||||
private toItem(r: TourListRow): TourListItem {
|
||||
@@ -196,7 +198,7 @@ export class ToursService {
|
||||
}
|
||||
|
||||
/**
|
||||
* The tours-mode GPX import prepares rows through PlacesService.prepareGpxRows
|
||||
* The tours-mode GPX import reads rows through PlaceImportService.readGpx
|
||||
* with waypoints excluded. PlacesService.importPreparedGpx persists the places
|
||||
* in the same transaction as their `tours` facet rows, whose metrics are
|
||||
* derived from route_geometry.
|
||||
@@ -206,7 +208,7 @@ export class ToursService {
|
||||
* match_confidence so the client can surface a "with caution" toast.
|
||||
*/
|
||||
async importGpxAsTour(tripId: string, fileBuffer: Buffer, defaultName?: string, socketId?: string): Promise<ImportGpxAsTourResult | null> {
|
||||
const rows = this.places.prepareGpxRows(fileBuffer, {
|
||||
const rows = this.placeImport.readGpx(fileBuffer, {
|
||||
importWaypoints: false, importRoutes: true, importTracks: true, defaultName,
|
||||
});
|
||||
if (rows.length === 0) return null;
|
||||
|
||||
Reference in New Issue
Block a user