diff --git a/server/scripts/build-atlas-geo.mjs b/server/scripts/build-atlas-geo.mjs index 68ac6dd40..01b99a755 100644 --- a/server/scripts/build-atlas-geo.mjs +++ b/server/scripts/build-atlas-geo.mjs @@ -372,7 +372,7 @@ async function pool(items, worker) { // the now-redundant consecutive duplicate points shrinks the bundles ~5-8x with no // visible effect at the atlas' zoom range (3-10). ADM0 fills are viewed zoomed out, so // they tolerate a coarser grid than ADM1 region borders. -const ADM0_DECIMALS = 2; // ~1.1 km +const ADM0_DECIMALS = 2; // ~1.1 km; the server's country index packs ×100 into Int16 (ADMIN0_SCALE in country-borders.ts), raise both const ADM1_DECIMALS = 3; // ~110 m; the server's region cache packs ×1000 (REGION_SCALE in atlas-geo.ts), raise both function quantizeRing(ring, decimals) { diff --git a/server/src/nest/atlas/atlas-geo.ts b/server/src/nest/atlas/atlas-geo.ts index 59566865a..75e611eb5 100644 --- a/server/src/nest/atlas/atlas-geo.ts +++ b/server/src/nest/atlas/atlas-geo.ts @@ -5,9 +5,9 @@ import path from 'path'; import zlib from 'zlib'; import { admin1AssetPath, + getAdmin0File, getCountryFromCoords, isPointInCountryBox, - loadAdmin0Gz, pointInPackedGeometry, type PackedGeom, } from '../../utils/country-borders'; @@ -30,30 +30,30 @@ export { getCountryFromCoords, isPointInCountryBox }; // ── Bundled boundary GeoJSON (admin-0 countries + admin-1 regions) ───────── // -// Where the bundles come from, the path they are read from, the admin-0 gz bytes and -// the country index built on them are in utils/country-borders.ts. The admin-1 half +// Where the bundles come from, the path they are read from, the admin-0 file and +// the country index built on it are in utils/country-borders.ts. The admin-1 half // is below. // // Neither parsed bundle is cached. admin0 (~145MB) and admin1 (~260MB) parsed at once are // what exhausted a 512MB host while using Atlas (#1576). Instead we retain only compact -// derivatives: the raw admin0 .gz bytes for direct serving and a Float64Array poly/box -// index for server-side point-in-polygon (see buildCountryIndexes in -// utils/country-borders.ts). admin1 ships as one gzipped file per country, which is -// served from disk as it is and read one country at a time for region lookups. +// derivatives: the admin0 .gz is sent from disk and an Int16Array poly/box index serves +// server-side point-in-polygon (see buildCountryIndexes in utils/country-borders.ts). +// admin1 ships as one gzipped file per country, which is served from disk as it is and +// read one country at a time for region lookups. -/** admin-0 country borders as gzipped GeoJSON bytes, served to the client map with +/** Path of the gzipped admin-0 country borders, sent to the client map with * Content-Encoding: gzip so the server never holds the parsed FeatureCollection. */ -export function getCountryGeoGz(): Buffer | null { - return loadAdmin0Gz(); +export function getCountryGeoFile(): string | null { + return getAdmin0File(); } /** Parsed admin-0 FeatureCollection, parsed on demand (not cached). Not on the client hot - * path — the map is served the gz bytes via getCountryGeoGz — but kept for internal/test + * path — the map is sent the gz file via getCountryGeoFile — but kept for internal/test * callers that need the objects. */ export function getCountryGeo(): any { - const gz = loadAdmin0Gz(); - if (!gz) return { type: 'FeatureCollection', features: [] }; - return JSON.parse(zlib.gunzipSync(gz).toString('utf8')); + const file = getAdmin0File(); + if (!file) return { type: 'FeatureCollection', features: [] }; + return JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf8')); } // A country's admin-1 file, or null for anything that is not a two-letter code, so @@ -86,7 +86,7 @@ function readAdmin1Json(countryCode: string): Buffer | null { } /** One country's admin-1 regions as gzipped GeoJSON bytes, served to the client map with - * Content-Encoding: gzip like getCountryGeoGz, so the server never parses them. */ + * Content-Encoding: gzip like getCountryGeoFile, so the server never parses them. */ export async function getRegionGeoGz(countryCode: string): Promise { const file = admin1File(countryCode); if (!file) return null; diff --git a/server/src/nest/atlas/atlas.controller.ts b/server/src/nest/atlas/atlas.controller.ts index 84767242d..1f4830e61 100644 --- a/server/src/nest/atlas/atlas.controller.ts +++ b/server/src/nest/atlas/atlas.controller.ts @@ -26,6 +26,7 @@ import { regionGeoParamsSchema } from '@trek/shared'; import type { AtlasLocateResponse, RegionGeoParams } from '@trek/shared'; import type { Response } from 'express'; +import path from 'node:path'; /** * /api/addons/atlas — visited countries/regions, region GeoJSON, bucket list. @@ -123,10 +124,13 @@ export class AtlasController { countryGeo(@Res() res: Response): void { // Serve the pre-gzipped admin-0 bundle straight from disk. The browser decompresses // transparently, so the wire shape is identical to before, but the server never parses - // or holds the ~145MB FeatureCollection (#1576). Content-Encoding is set explicitly, so - // the compression middleware leaves the body untouched. - const gz = this.atlas.countryGeoGz(); - if (!gz) { + // or holds the ~145MB FeatureCollection (#1576), nor the 4MB file itself. Content-Encoding + // is set explicitly, so the compression middleware leaves the body untouched, and + // send() keeps the Content-Type and Cache-Control set here. Without Accept-Ranges and + // Last-Modified the headers are the ones res.send(buffer) gave. Root + basename, not the + // absolute path: see plugin-frame.controller.ts. + const file = this.atlas.countryGeoFile(); + if (!file) { res.setHeader('Cache-Control', 'no-store'); res.json({ type: 'FeatureCollection', features: [] }); return; @@ -134,7 +138,7 @@ export class AtlasController { res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.setHeader('Content-Encoding', 'gzip'); res.setHeader('Cache-Control', 'public, max-age=86400'); - res.send(gz); + res.sendFile(path.basename(file), { root: path.dirname(file), acceptRanges: false, lastModified: false }); } @Get('country/:code') diff --git a/server/src/nest/atlas/atlas.service.ts b/server/src/nest/atlas/atlas.service.ts index 00a0aac7b..fffbf0830 100644 --- a/server/src/nest/atlas/atlas.service.ts +++ b/server/src/nest/atlas/atlas.service.ts @@ -26,7 +26,7 @@ import type { ReservationEndpointsRepository, TravelerOwnedEndpointRow } from '. import { getCountryFromAddress, getCountryFromCoords, - getCountryGeoGz, + getCountryGeoFile, getRegionFromCoords, geocodingInFlight, resolveCountryCodeSync, @@ -876,8 +876,8 @@ export class AtlasService { }; } - countryGeoGz(): Buffer | null { - return getCountryGeoGz(); + countryGeoFile(): string | null { + return getCountryGeoFile(); } // ── Bucket list CRUD ────────────────────────────────────────────────────── diff --git a/server/src/utils/country-borders.ts b/server/src/utils/country-borders.ts index ebe91a0df..1332003e7 100644 --- a/server/src/utils/country-borders.ts +++ b/server/src/utils/country-borders.ts @@ -11,8 +11,8 @@ import zlib from 'zlib'; // for the country an index suggestion names. Importing it from atlas tied maps into // an import cycle with atlas (lint:boundaries). // -// atlas-geo resolves admin-1 regions with the per-country asset path and the ring test -// exported here. The admin0 gz bytes and the country poly/box +// atlas-geo resolves admin-1 regions with the per-country asset path and the packed ring +// test exported here. The admin0 file path and the country poly/box // indexes below are file-scoped for the reason atlas-geo gives: their lifetime stays // process-global however many callers there are, so the indexes are built once per // process, which the #1576 OOM fix depends on. @@ -38,15 +38,17 @@ export function admin1AssetPath(countryCode: string): string { return path.join(__dirname, '..', '..', 'assets', 'atlas', 'admin1', `${countryCode}.geojson.gz`); } -let admin0Gz: Buffer | null | undefined; -export function loadAdmin0Gz(): Buffer | null { - if (admin0Gz !== undefined) return admin0Gz; +// The admin0 bytes are not kept: countries/geo sends the file from disk, and the country +// index below reads it once while it is built. +let admin0File: string | null | undefined; +export function getAdmin0File(): string | null { + if (admin0File !== undefined) return admin0File; const file = assetPath('admin0'); if (!fs.existsSync(file)) { console.warn(`[Atlas] admin0.geojson.gz missing — run \`node scripts/build-atlas-geo.mjs\``); - return (admin0Gz = null); + return (admin0File = null); } - return (admin0Gz = fs.readFileSync(file)); + return (admin0File = file); } // Feed arbitrary gunzip chunks; invokes onFeature(text) once per top-level Feature object @@ -159,53 +161,14 @@ const MICRO_BOX_MAX_AREA = 2.0; // ── Point-in-polygon over the bundled admin0 borders (#1331) ───────────────── -// Ray-casting (even-odd) test of (lng,lat) against a single GeoJSON ring. -// Ray-cast on a flat [lng,lat,lng,lat,…] ring. Same algorithm as the classic number[][] -// version, but the coordinates live in a Float64Array so the parsed admin0 geometry (with -// its millions of tiny [lng,lat] arrays, ~145MB) need not be retained — only these rings. -function pointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean { - let inside = false; - const n = ring.length / 2; - for (let i = 0, j = n - 1; i < n; j = i++) { - const xi = ring[2 * i], - yi = ring[2 * i + 1]; - const xj = ring[2 * j], - yj = ring[2 * j + 1]; - if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) { - inside = !inside; - } - } - return inside; -} - -// True when (lng,lat) falls inside a compact Polygon/MultiPolygon, honouring holes. -// `rings` is every ring flattened; `polyRingCounts[k]` is how many rings polygon k owns -// (its first ring is the outer boundary, the rest are holes). -function pointInGeometry(lng: number, lat: number, geom: CompactGeom): boolean { - let ri = 0; - for (const rc of geom.polyRingCounts) { - if (pointInFlatRing(lng, lat, geom.rings[ri])) { - let inHole = false; - for (let h = 1; h < rc; h++) { - if (pointInFlatRing(lng, lat, geom.rings[ri + h])) { - inHole = true; - break; - } - } - if (!inHole) return true; - } - ri += rc; - } - return false; -} - -// The ray-cast above on ring pairs [from, to) of packed coordinates. Each one is divided back -// by the scale before use: for coordinates with no more decimals than the scale holds that -// gives back the exact double the bundle had, so the answer is the one the Float64 ring gives. +// Ray-casting (even-odd) test of (lng,lat) against ring pairs [from, to) of packed +// coordinates. Each one is divided back by the scale before use: for coordinates with no +// more decimals than the scale holds that gives back the exact double the bundle had, so the +// answer is the one a Float64 ring of the same points gives. function pointInPackedRing( lng: number, lat: number, - coords: Int32Array, + coords: Int16Array | Int32Array, scale: number, from: number, to: number, @@ -225,7 +188,8 @@ function pointInPackedRing( return inside; } -// pointInGeometry for a PackedGeom. +// True when (lng,lat) falls inside a packed Polygon/MultiPolygon, honouring holes: the +// first ring of each polygon is its outer boundary, the rest are holes. export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom, scale: number): boolean { const { coords, ringStarts, polyRingCounts } = geom; let ri = 0; @@ -245,20 +209,23 @@ export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom return false; } -// Compact polygon geometry: rings flattened into Float64Arrays, grouped into polygons by -// polyRingCounts. Replaces the retained parsed GeoJSON geometry. -type CompactGeom = { rings: Float64Array[]; polyRingCounts: number[] }; - -// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one Int32Array, each -// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts -// groups the rings into polygons as in CompactGeom. Three typed arrays per geometry instead -// of one per ring, at half the bytes per point. -export type PackedGeom = { coords: Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array }; +// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one typed array, each +// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts[k] +// is how many rings polygon k owns. Three typed arrays per geometry instead of one per ring, +// at a quarter (Int16, admin-0) or half (Int32, admin-1) of the bytes per point. +export type PackedGeom = { coords: Int16Array | Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array }; type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat] -// ISO_A2 → compact admin0 geometry + bounding boxes, derived from the bundled admin0 +// The admin-0 bundle is quantized to ADM0_DECIMALS = 2 (scripts/build-atlas-geo.mjs) and no +// coordinate passes ±180, so every one is a whole number of hundredths within ±18000 and fits +// an Int16 exactly (ATLAS-BUNDLE-010 guards it). A third decimal needs ×1000 here and an +// Int32Array, since ±180000 does not fit an Int16. +const ADMIN0_SCALE = 100; + +// ISO_A2 → packed admin0 geometry + bounding boxes, derived from the bundled admin0 // borders on first use. The parsed FeatureCollection is dropped after this runs; only the -// Float64Array rings and boxes are retained (≈1MB vs the ≈145MB parsed geometry, #1576). +// Int16 coordinates and the boxes are retained (≈9MB for the 2M points, vs ≈34MB as +// Float64Array rings and ≈145MB as the parsed geometry, #1576). // // The boxes used to be a hand-maintained table, which drifted: 43 countries (NG, BY, // GL, KP, TD, SS, …) had no box at all, so their coordinates fell into a *neighbour's* @@ -269,20 +236,20 @@ type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat] // One box is stored PER GEOMETRY PART, not per country. A single box around a country // that straddles the antimeridian (RU, US, FJ, KI) would span nearly the whole globe; // per-part boxes keep Alaska and Chukotka separate and handle the ±180 wrap for free. -let countryPolyIndex: Map | null = null; +let countryPolyIndex: Map | null = null; let countryBoxIndex: Map | null = null; function buildCountryIndexes(): void { - const polys = new Map(); + const polys = new Map(); const boxes = new Map(); - const gz = loadAdmin0Gz(); - if (gz) { + const file = getAdmin0File(); + if (file) { // Parse ONE feature at a time off the gunzipped string rather than JSON.parse-ing the // whole FeatureCollection: the full parse transiently allocates ~285MB (the 145MB // object graph plus intermediates) and V8 keeps those pages, which alone exhausts a // 512MB host before admin1 even loads (#1576). - const json = zlib.gunzipSync(gz).toString('utf8'); + const json = zlib.gunzipSync(fs.readFileSync(file)).toString('utf8'); const consume = createFeatureSplitter((text) => { const f = JSON.parse(text); const raw = f.properties?.ISO_A2; @@ -292,18 +259,27 @@ function buildCountryIndexes(): void { const parts = ( f.geometry.type === 'Polygon' ? [f.geometry.coordinates] : f.geometry.coordinates ) as number[][][][]; - const rings: Float64Array[] = []; - const polyRingCounts: number[] = []; - const codeBoxes = boxes.get(code) ?? []; + let points = 0; + let ringCount = 0; for (const part of parts) { - polyRingCounts.push(part.length); + ringCount += part.length; + for (const ring of part) points += ring.length; + } + const coords = new Int16Array(points * 2); + const ringStarts = new Uint32Array(ringCount + 1); + const polyRingCounts = new Uint32Array(parts.length); + const codeBoxes = boxes.get(code) ?? []; + let p = 0; + let ri = 0; + for (const [k, part] of parts.entries()) { + polyRingCounts[k] = part.length; for (const ring of part) { - const flat = new Float64Array(ring.length * 2); - for (let i = 0; i < ring.length; i++) { - flat[2 * i] = ring[i][0]; - flat[2 * i + 1] = ring[i][1]; + ringStarts[ri++] = p; + for (const [lng, lat] of ring) { + coords[2 * p] = Math.round(lng * ADMIN0_SCALE); + coords[2 * p + 1] = Math.round(lat * ADMIN0_SCALE); + p++; } - rings.push(flat); } // Bounding box from the part's outer ring (part[0]). let minLng = Infinity, @@ -318,9 +294,10 @@ function buildCountryIndexes(): void { } codeBoxes.push([minLng, minLat, maxLng, maxLat]); } + ringStarts[ri] = p; // Matches the previous index exactly: geometry is overwritten (last feature for a // code wins) while boxes accumulate across a code's features. - polys.set(code, { rings, polyRingCounts }); + polys.set(code, { coords, ringStarts, polyRingCounts }); boxes.set(code, codeBoxes); }); consume(json); @@ -335,7 +312,7 @@ function buildCountryIndexes(): void { countryBoxIndex = boxes; } -function getCountryPolyIndex(): Map { +function getCountryPolyIndex(): Map { if (!countryPolyIndex) buildCountryIndexes(); return countryPolyIndex!; } @@ -393,7 +370,7 @@ export function getCountryFromCoords(lat: number, lng: number): string | null { if (looseBoxFallback === null) looseBoxFallback = code; continue; } - if (pointInGeometry(lng, lat, poly)) return code; + if (pointInPackedGeometry(lng, lat, poly, ADMIN0_SCALE)) return code; } // No tight micro-box and no polygon contained the point — prefer a deferred loose box // (a real PS/XK point), else fall back to the smallest box (coastal slop / data gap). diff --git a/server/tests/e2e/atlas.e2e.test.ts b/server/tests/e2e/atlas.e2e.test.ts index 362e7837b..3d1640f83 100644 --- a/server/tests/e2e/atlas.e2e.test.ts +++ b/server/tests/e2e/atlas.e2e.test.ts @@ -16,6 +16,7 @@ import { VisitedRegions } from '../../src/db/entities/VisitedRegions.entity'; import { AtlasModule } from '../../src/nest/atlas/atlas.module'; import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter'; import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe'; +import { getAdmin0File } from '../../src/utils/country-borders'; import { createUser, createTrip } from '../helpers/factories'; import { markCountryVisited } from '../helpers/factories/atlas'; import { makePlace } from '../helpers/factories/places'; @@ -28,6 +29,7 @@ import { sessionCookie } from './harness'; import { Test } from '@nestjs/testing'; import cookieParser from 'cookie-parser'; +import fs from 'fs'; import type { Server } from 'http'; import request from 'supertest'; import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; @@ -85,11 +87,22 @@ describe('Atlas e2e (real auth guard + real service + temp SQLite)', () => { it('200 countries/geo serves the bundled gzipped admin-0 that the client decompresses to a FeatureCollection', async () => { const res = await request(server).get('/api/addons/atlas/countries/geo').set('Cookie', sessionCookie(userId)); expect(res.status).toBe(200); - expect(res.headers['content-encoding']).toBe('gzip'); + const admin0 = getAdmin0File(); + expect(admin0).not.toBeNull(); + // Sent from disk with the headers res.send(buffer) gave when the bytes were kept in + // memory: send() replaces none of them and adds no Accept-Ranges or Last-Modified. + expect(res.headers).toMatchObject({ + 'content-type': 'application/json; charset=utf-8', + 'content-encoding': 'gzip', + 'cache-control': 'public, max-age=86400', + 'content-length': String(fs.statSync(String(admin0)).size), + }); + expect(res.headers.etag).toMatch(/^W\/"/); + expect(res.headers).not.toHaveProperty('accept-ranges'); + expect(res.headers).not.toHaveProperty('last-modified'); // superagent transparently decompresses, mirroring the browser. expect(res.body.type).toBe('FeatureCollection'); expect(res.body.features.length).toBeGreaterThan(0); - expect(res.headers['cache-control']).toContain('max-age=86400'); }); it('200 stats for an authenticated user — the trip-less branch keeps its bespoke shape', async () => { diff --git a/server/tests/integration/bootstrap.test.ts b/server/tests/integration/bootstrap.test.ts index 787d76569..ae5bbf6d7 100644 --- a/server/tests/integration/bootstrap.test.ts +++ b/server/tests/integration/bootstrap.test.ts @@ -131,6 +131,8 @@ describe('BOOTSTRAP (F6) — unified NestJS app serves the whole surface', () => .set('Cookie', authCookie(user.id)); expect(res.status).toBe(200); expect(res.headers['content-encoding']).toBe('gzip'); + // Gzipped once: the compression middleware leaves the file's own encoding alone. + expect(res.body.type).toBe('FeatureCollection'); }); it('BOOTSTRAP-ORM-001: the ORM unit of work resolves from the container', () => { diff --git a/server/tests/unit/nest/atlas-country-index.test.ts b/server/tests/unit/nest/atlas-country-index.test.ts new file mode 100644 index 000000000..ba6a18e95 --- /dev/null +++ b/server/tests/unit/nest/atlas-country-index.test.ts @@ -0,0 +1,183 @@ +import { getAdmin0File, getCountryFromCoords } from '../../../src/utils/country-borders'; + +import fs from 'fs'; +import { describe, expect, it } from 'vitest'; +import zlib from 'zlib'; + +// The packed admin-0 country index behind getCountryFromCoords: Int16 ×100 coordinates in +// place of Float64Array rings. + +type Geometry = { type: string; coordinates: unknown }; +type Feature = { properties: { ISO_A2?: string | null }; geometry: Geometry | null }; +const admin0 = getAdmin0File(); +if (!admin0) throw new Error('admin0.geojson.gz missing'); +const features = (JSON.parse(zlib.gunzipSync(fs.readFileSync(admin0)).toString('utf8')) as { features: Feature[] }) + .features; + +// ── The lookup as it was before the packing, kept as the reference ───────── +// Float64 rings, the same boxes, micro-territories and ranking: what getCountryFromCoords +// did before this index, copied from country-borders.ts. + +type OldGeom = { rings: Float64Array[]; polyRingCounts: number[] }; +type Box = [number, number, number, number]; + +const MICRO_TERRITORY_BOXES: Record = { + HK: [113.83, 22.15, 114.43, 22.56], + MO: [113.53, 22.1, 113.6, 22.21], + GI: [-5.36, 36.11, -5.33, 36.16], + PR: [-67.3, 17.88, -65.22, 18.53], + PS: [34.2, 29.5, 35.6, 32.6], + XK: [20.0, 41.9, 21.8, 43.3], +}; +const MICRO_BOX_MAX_AREA = 2.0; + +function oldIndexes(): { polys: Map; boxes: Map } { + const polys = new Map(); + const boxes = new Map(); + for (const f of features) { + const raw = f.properties?.ISO_A2; + if (!raw || raw === '-99' || !f.geometry) continue; + const code = String(raw).toUpperCase(); + const g = f.geometry; + const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]; + const rings: Float64Array[] = []; + const polyRingCounts: number[] = []; + const codeBoxes = boxes.get(code) ?? []; + for (const part of parts) { + polyRingCounts.push(part.length); + for (const ring of part) rings.push(Float64Array.from(ring.flat())); + const lngs = part[0].map(([lng]) => lng); + const lats = part[0].map(([, lat]) => lat); + codeBoxes.push([Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)]); + } + polys.set(code, { rings, polyRingCounts }); + boxes.set(code, codeBoxes); + } + for (const [code, box] of Object.entries(MICRO_TERRITORY_BOXES)) { + if (!boxes.has(code)) boxes.set(code, [box]); + } + return { polys, boxes }; +} + +function oldPointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean { + let inside = false; + const n = ring.length / 2; + for (let i = 0, j = n - 1; i < n; j = i++) { + const xi = ring[2 * i], + yi = ring[2 * i + 1]; + const xj = ring[2 * j], + yj = ring[2 * j + 1]; + if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) { + inside = !inside; + } + } + return inside; +} + +function oldPointInGeometry(lng: number, lat: number, geom: OldGeom): boolean { + let ri = 0; + for (const rc of geom.polyRingCounts) { + if (oldPointInFlatRing(lng, lat, geom.rings[ri])) { + let inHole = false; + for (let h = 1; h < rc; h++) { + if (oldPointInFlatRing(lng, lat, geom.rings[ri + h])) { + inHole = true; + break; + } + } + if (!inHole) return true; + } + ri += rc; + } + return false; +} + +function oldLookup(old: ReturnType, lat: number, lng: number): string | null { + const candidates: { code: string; area: number }[] = []; + for (const [code, boxes] of old.boxes) { + for (const [minLng, minLat, maxLng, maxLat] of boxes) { + if (lat >= minLat && lat <= maxLat && lng >= minLng && lng <= maxLng) { + candidates.push({ code, area: (maxLng - minLng) * (maxLat - minLat) }); + break; + } + } + } + if (candidates.length === 0) return null; + candidates.sort((a, b) => a.area - b.area); + let looseBoxFallback: string | null = null; + for (const { code, area } of candidates) { + const poly = old.polys.get(code); + if (!poly) { + if (area <= MICRO_BOX_MAX_AREA) return code; + if (looseBoxFallback === null) looseBoxFallback = code; + continue; + } + if (oldPointInGeometry(lng, lat, poly)) return code; + } + return looseBoxFallback ?? candidates[0].code; +} + +// Deterministic points per country: about `perCountry` vertices spread over all its rings, +// each exactly, 1e-7° to either side of it and of the next edge's middle (where a ray-cast +// is most fragile), plus random points in the country's box. +function testPoints(f: Feature, perCountry: number, random: () => number): [number, number][] { + if (!f.geometry) return []; + const g = f.geometry; + const rings = ((g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]).flat(); + const total = rings.reduce((n, ring) => n + ring.length, 0); + const step = Math.max(1, Math.floor(total / perCountry)); + const points: [number, number][] = []; + let seen = 0; + let minLng = Infinity, + minLat = Infinity, + maxLng = -Infinity, + maxLat = -Infinity; + for (const ring of rings) { + for (let i = 0; i < ring.length - 1; i++) { + const [lng, lat] = ring[i]; + minLng = Math.min(minLng, lng); + maxLng = Math.max(maxLng, lng); + minLat = Math.min(minLat, lat); + maxLat = Math.max(maxLat, lat); + if (seen++ % step !== 0) continue; + const midLng = (lng + ring[i + 1][0]) / 2; + const midLat = (lat + ring[i + 1][1]) / 2; + points.push( + [lat, lng], + [lat + 1e-7, lng + 1e-7], + [lat - 1e-7, lng - 1e-7], + [midLat + 1e-7, midLng], + [midLat - 1e-7, midLng], + ); + } + } + for (let k = 0; k < 20; k++) { + points.push([minLat + random() * (maxLat - minLat), minLng + random() * (maxLng - minLng)]); + } + return points; +} + +describe('getCountryFromCoords on the packed country index', () => { + it('ATLAS-SVC-055: answers every test point the way the Float64 lookup did, in every country', () => { + const old = oldIndexes(); + let seed = 0x2f6b; + const random = () => (seed = (seed * 16807) % 2147483647) / 2147483647; + let checked = 0; + let inCountry = 0; + const different: string[] = []; + for (const f of features) { + // Each point near a border is tested against the big polygons whose boxes cover it + // (RU, CA, US, CN, ...), so 100 vertices per country already take ~15s. + for (const [lat, lng] of testPoints(f, 100, random)) { + const before = oldLookup(old, lat, lng); + const after = getCountryFromCoords(lat, lng); + checked++; + if (before) inCountry++; + if (after !== before) different.push(`${f.properties.ISO_A2} ${lat},${lng}: ${before} → ${after}`); + } + } + expect(checked).toBeGreaterThan(100_000); + expect(inCountry / checked, 'most points are in a country').toBeGreaterThan(0.9); + expect(different.slice(0, 10)).toEqual([]); + }, 120_000); +}); diff --git a/server/tests/unit/nest/atlas.controller.test.ts b/server/tests/unit/nest/atlas.controller.test.ts index 6e3186c7b..d55dc0c27 100644 --- a/server/tests/unit/nest/atlas.controller.test.ts +++ b/server/tests/unit/nest/atlas.controller.test.ts @@ -19,11 +19,12 @@ function makeController(svc: Partial & Partial } function makeRes() { - const res = { setHeader: vi.fn(), send: vi.fn(), json: vi.fn(), type: vi.fn() }; + const res = { setHeader: vi.fn(), send: vi.fn(), sendFile: vi.fn(), json: vi.fn(), type: vi.fn() }; res.type.mockReturnValue(res); return res as unknown as Response & { setHeader: ReturnType; send: ReturnType; + sendFile: ReturnType; json: ReturnType; type: ReturnType; }; @@ -89,20 +90,24 @@ describe('AtlasController (parity with the legacy /api/addons/atlas route)', () }); }); - it('GET /countries/geo serves the gzipped admin-0 bundle with Content-Encoding', () => { - const gz = Buffer.from('gzipped-bytes'); - const countryGeoGz = vi.fn().mockReturnValue(gz); + it('GET /countries/geo sends the gzipped admin-0 file from disk with Content-Encoding', () => { + const countryGeoFile = vi.fn().mockReturnValue('/srv/trek/assets/atlas/admin0.geojson.gz'); const res = makeRes(); - makeController({ countryGeoGz }).countryGeo(res); + makeController({ countryGeoFile }).countryGeo(res); + expect(res.setHeader).toHaveBeenCalledWith('Content-Type', 'application/json; charset=utf-8'); expect(res.setHeader).toHaveBeenCalledWith('Content-Encoding', 'gzip'); expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400'); - expect(res.send).toHaveBeenCalledWith(gz); + expect(res.sendFile).toHaveBeenCalledWith('admin0.geojson.gz', { + root: '/srv/trek/assets/atlas', + acceptRanges: false, + lastModified: false, + }); }); it('GET /countries/geo falls back to an empty FeatureCollection when the bundle is missing', () => { - const countryGeoGz = vi.fn().mockReturnValue(null); + const countryGeoFile = vi.fn().mockReturnValue(null); const res = makeRes(); - makeController({ countryGeoGz }).countryGeo(res); + makeController({ countryGeoFile }).countryGeo(res); expect(res.json).toHaveBeenCalledWith({ type: 'FeatureCollection', features: [] }); expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'no-store'); }); diff --git a/server/tests/unit/nest/atlasBundle.test.ts b/server/tests/unit/nest/atlasBundle.test.ts index 0746e7a31..04d288263 100644 --- a/server/tests/unit/nest/atlasBundle.test.ts +++ b/server/tests/unit/nest/atlasBundle.test.ts @@ -17,6 +17,7 @@ describe('Atlas admin0 country bundle (#1609)', () => { const bundlePath = path.join(__dirname, '..', '..', '..', 'assets', 'atlas', 'admin0.geojson.gz'); const features = JSON.parse(zlib.gunzipSync(fs.readFileSync(bundlePath)).toString()).features as { properties: { ISO_A2: string | null; ADM0_A3: string; NAME?: string }; + geometry: { type: string; coordinates: number[][][] | number[][][][] } | null; }[]; it('ATLAS-BUNDLE-003 — Kosovo ships with a resolvable ISO_A2', () => { @@ -24,6 +25,28 @@ describe('Atlas admin0 country bundle (#1609)', () => { expect(kosovo.length, 'exactly one Kosovo feature').toBe(1); expect(kosovo[0].properties.ISO_A2).toBe('XK'); }); + + // The server's country index keeps every coordinate in an Int16Array ×100 (ADMIN0_SCALE in + // country-borders.ts) and divides it back on use. A value with a third decimal comes back + // changed, and one past ±327.67 wraps around without an error, so a rebuild with more + // decimals (ADM0_DECIMALS) must fail here, not move borders. + it('ATLAS-BUNDLE-010 — every admin0 coordinate round-trips through Int16 ×100 unchanged', () => { + const int16 = new Int16Array(1); + let values = 0; + const lossy: string[] = []; + for (const f of features) { + const g = f.geometry; + if (!g) continue; + const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]; + for (const v of parts.flat(3)) { + values++; + int16[0] = Math.round(v * 100); + if (int16[0] / 100 !== v) lossy.push(`${f.properties.ADM0_A3}: ${v} comes back as ${int16[0] / 100}`); + } + } + expect(values / 2, 'coordinate pairs checked').toBeGreaterThan(2_000_000); + expect(lossy.slice(0, 5)).toEqual([]); + }); }); describe('Atlas admin1 region bundle (#1217)', () => {