perf(atlas): keep the admin0 country index as Int16 arrays and send countries/geo from disk

This commit is contained in:
Vini
2026-10-27 10:28:31 +01:00
committed by Maurice
parent c94ff7421a
commit f56f6ae11e
10 changed files with 321 additions and 114 deletions
+1 -1
View File
@@ -372,7 +372,7 @@ async function pool(items, worker) {
// the now-redundant consecutive duplicate points shrinks the bundles ~5-8x with no
// visible effect at the atlas' zoom range (3-10). ADM0 fills are viewed zoomed out, so
// they tolerate a coarser grid than ADM1 region borders.
const ADM0_DECIMALS = 2; // ~1.1 km
const ADM0_DECIMALS = 2; // ~1.1 km; the server's country index packs ×100 into Int16 (ADMIN0_SCALE in country-borders.ts), raise both
const ADM1_DECIMALS = 3; // ~110 m; the server's region cache packs ×1000 (REGION_SCALE in atlas-geo.ts), raise both
function quantizeRing(ring, decimals) {
+15 -15
View File
@@ -5,9 +5,9 @@ import path from 'path';
import zlib from 'zlib';
import {
admin1AssetPath,
getAdmin0File,
getCountryFromCoords,
isPointInCountryBox,
loadAdmin0Gz,
pointInPackedGeometry,
type PackedGeom,
} from '../../utils/country-borders';
@@ -30,30 +30,30 @@ export { getCountryFromCoords, isPointInCountryBox };
// ── Bundled boundary GeoJSON (admin-0 countries + admin-1 regions) ─────────
//
// Where the bundles come from, the path they are read from, the admin-0 gz bytes and
// the country index built on them are in utils/country-borders.ts. The admin-1 half
// Where the bundles come from, the path they are read from, the admin-0 file and
// the country index built on it are in utils/country-borders.ts. The admin-1 half
// is below.
//
// Neither parsed bundle is cached. admin0 (~145MB) and admin1 (~260MB) parsed at once are
// what exhausted a 512MB host while using Atlas (#1576). Instead we retain only compact
// derivatives: the raw admin0 .gz bytes for direct serving and a Float64Array poly/box
// index for server-side point-in-polygon (see buildCountryIndexes in
// utils/country-borders.ts). admin1 ships as one gzipped file per country, which is
// served from disk as it is and read one country at a time for region lookups.
// derivatives: the admin0 .gz is sent from disk and an Int16Array poly/box index serves
// server-side point-in-polygon (see buildCountryIndexes in utils/country-borders.ts).
// admin1 ships as one gzipped file per country, which is served from disk as it is and
// read one country at a time for region lookups.
/** admin-0 country borders as gzipped GeoJSON bytes, served to the client map with
/** Path of the gzipped admin-0 country borders, sent to the client map with
* Content-Encoding: gzip so the server never holds the parsed FeatureCollection. */
export function getCountryGeoGz(): Buffer | null {
return loadAdmin0Gz();
export function getCountryGeoFile(): string | null {
return getAdmin0File();
}
/** Parsed admin-0 FeatureCollection, parsed on demand (not cached). Not on the client hot
* path — the map is served the gz bytes via getCountryGeoGz — but kept for internal/test
* path — the map is sent the gz file via getCountryGeoFile — but kept for internal/test
* callers that need the objects. */
export function getCountryGeo(): any {
const gz = loadAdmin0Gz();
if (!gz) return { type: 'FeatureCollection', features: [] };
return JSON.parse(zlib.gunzipSync(gz).toString('utf8'));
const file = getAdmin0File();
if (!file) return { type: 'FeatureCollection', features: [] };
return JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf8'));
}
// A country's admin-1 file, or null for anything that is not a two-letter code, so
@@ -86,7 +86,7 @@ function readAdmin1Json(countryCode: string): Buffer | null {
}
/** One country's admin-1 regions as gzipped GeoJSON bytes, served to the client map with
* Content-Encoding: gzip like getCountryGeoGz, so the server never parses them. */
* Content-Encoding: gzip like getCountryGeoFile, so the server never parses them. */
export async function getRegionGeoGz(countryCode: string): Promise<Buffer | null> {
const file = admin1File(countryCode);
if (!file) return null;
+9 -5
View File
@@ -26,6 +26,7 @@ import { regionGeoParamsSchema } from '@trek/shared';
import type { AtlasLocateResponse, RegionGeoParams } from '@trek/shared';
import type { Response } from 'express';
import path from 'node:path';
/**
* /api/addons/atlas — visited countries/regions, region GeoJSON, bucket list.
@@ -123,10 +124,13 @@ export class AtlasController {
countryGeo(@Res() res: Response): void {
// Serve the pre-gzipped admin-0 bundle straight from disk. The browser decompresses
// transparently, so the wire shape is identical to before, but the server never parses
// or holds the ~145MB FeatureCollection (#1576). Content-Encoding is set explicitly, so
// the compression middleware leaves the body untouched.
const gz = this.atlas.countryGeoGz();
if (!gz) {
// or holds the ~145MB FeatureCollection (#1576), nor the 4MB file itself. Content-Encoding
// is set explicitly, so the compression middleware leaves the body untouched, and
// send() keeps the Content-Type and Cache-Control set here. Without Accept-Ranges and
// Last-Modified the headers are the ones res.send(buffer) gave. Root + basename, not the
// absolute path: see plugin-frame.controller.ts.
const file = this.atlas.countryGeoFile();
if (!file) {
res.setHeader('Cache-Control', 'no-store');
res.json({ type: 'FeatureCollection', features: [] });
return;
@@ -134,7 +138,7 @@ export class AtlasController {
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Content-Encoding', 'gzip');
res.setHeader('Cache-Control', 'public, max-age=86400');
res.send(gz);
res.sendFile(path.basename(file), { root: path.dirname(file), acceptRanges: false, lastModified: false });
}
@Get('country/:code')
+3 -3
View File
@@ -26,7 +26,7 @@ import type { ReservationEndpointsRepository, TravelerOwnedEndpointRow } from '.
import {
getCountryFromAddress,
getCountryFromCoords,
getCountryGeoGz,
getCountryGeoFile,
getRegionFromCoords,
geocodingInFlight,
resolveCountryCodeSync,
@@ -876,8 +876,8 @@ export class AtlasService {
};
}
countryGeoGz(): Buffer | null {
return getCountryGeoGz();
countryGeoFile(): string | null {
return getCountryGeoFile();
}
// ── Bucket list CRUD ──────────────────────────────────────────────────────
+57 -80
View File
@@ -11,8 +11,8 @@ import zlib from 'zlib';
// for the country an index suggestion names. Importing it from atlas tied maps into
// an import cycle with atlas (lint:boundaries).
//
// atlas-geo resolves admin-1 regions with the per-country asset path and the ring test
// exported here. The admin0 gz bytes and the country poly/box
// atlas-geo resolves admin-1 regions with the per-country asset path and the packed ring
// test exported here. The admin0 file path and the country poly/box
// indexes below are file-scoped for the reason atlas-geo gives: their lifetime stays
// process-global however many callers there are, so the indexes are built once per
// process, which the #1576 OOM fix depends on.
@@ -38,15 +38,17 @@ export function admin1AssetPath(countryCode: string): string {
return path.join(__dirname, '..', '..', 'assets', 'atlas', 'admin1', `${countryCode}.geojson.gz`);
}
let admin0Gz: Buffer | null | undefined;
export function loadAdmin0Gz(): Buffer | null {
if (admin0Gz !== undefined) return admin0Gz;
// The admin0 bytes are not kept: countries/geo sends the file from disk, and the country
// index below reads it once while it is built.
let admin0File: string | null | undefined;
export function getAdmin0File(): string | null {
if (admin0File !== undefined) return admin0File;
const file = assetPath('admin0');
if (!fs.existsSync(file)) {
console.warn(`[Atlas] admin0.geojson.gz missing — run \`node scripts/build-atlas-geo.mjs\``);
return (admin0Gz = null);
return (admin0File = null);
}
return (admin0Gz = fs.readFileSync(file));
return (admin0File = file);
}
// Feed arbitrary gunzip chunks; invokes onFeature(text) once per top-level Feature object
@@ -159,53 +161,14 @@ const MICRO_BOX_MAX_AREA = 2.0;
// ── Point-in-polygon over the bundled admin0 borders (#1331) ─────────────────
// Ray-casting (even-odd) test of (lng,lat) against a single GeoJSON ring.
// Ray-cast on a flat [lng,lat,lng,lat,…] ring. Same algorithm as the classic number[][]
// version, but the coordinates live in a Float64Array so the parsed admin0 geometry (with
// its millions of tiny [lng,lat] arrays, ~145MB) need not be retained — only these rings.
function pointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean {
let inside = false;
const n = ring.length / 2;
for (let i = 0, j = n - 1; i < n; j = i++) {
const xi = ring[2 * i],
yi = ring[2 * i + 1];
const xj = ring[2 * j],
yj = ring[2 * j + 1];
if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) {
inside = !inside;
}
}
return inside;
}
// True when (lng,lat) falls inside a compact Polygon/MultiPolygon, honouring holes.
// `rings` is every ring flattened; `polyRingCounts[k]` is how many rings polygon k owns
// (its first ring is the outer boundary, the rest are holes).
function pointInGeometry(lng: number, lat: number, geom: CompactGeom): boolean {
let ri = 0;
for (const rc of geom.polyRingCounts) {
if (pointInFlatRing(lng, lat, geom.rings[ri])) {
let inHole = false;
for (let h = 1; h < rc; h++) {
if (pointInFlatRing(lng, lat, geom.rings[ri + h])) {
inHole = true;
break;
}
}
if (!inHole) return true;
}
ri += rc;
}
return false;
}
// The ray-cast above on ring pairs [from, to) of packed coordinates. Each one is divided back
// by the scale before use: for coordinates with no more decimals than the scale holds that
// gives back the exact double the bundle had, so the answer is the one the Float64 ring gives.
// Ray-casting (even-odd) test of (lng,lat) against ring pairs [from, to) of packed
// coordinates. Each one is divided back by the scale before use: for coordinates with no
// more decimals than the scale holds that gives back the exact double the bundle had, so the
// answer is the one a Float64 ring of the same points gives.
function pointInPackedRing(
lng: number,
lat: number,
coords: Int32Array,
coords: Int16Array | Int32Array,
scale: number,
from: number,
to: number,
@@ -225,7 +188,8 @@ function pointInPackedRing(
return inside;
}
// pointInGeometry for a PackedGeom.
// True when (lng,lat) falls inside a packed Polygon/MultiPolygon, honouring holes: the
// first ring of each polygon is its outer boundary, the rest are holes.
export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom, scale: number): boolean {
const { coords, ringStarts, polyRingCounts } = geom;
let ri = 0;
@@ -245,20 +209,23 @@ export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom
return false;
}
// Compact polygon geometry: rings flattened into Float64Arrays, grouped into polygons by
// polyRingCounts. Replaces the retained parsed GeoJSON geometry.
type CompactGeom = { rings: Float64Array[]; polyRingCounts: number[] };
// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one Int32Array, each
// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts
// groups the rings into polygons as in CompactGeom. Three typed arrays per geometry instead
// of one per ring, at half the bytes per point.
export type PackedGeom = { coords: Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array };
// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one typed array, each
// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts[k]
// is how many rings polygon k owns. Three typed arrays per geometry instead of one per ring,
// at a quarter (Int16, admin-0) or half (Int32, admin-1) of the bytes per point.
export type PackedGeom = { coords: Int16Array | Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array };
type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
// ISO_A2 → compact admin0 geometry + bounding boxes, derived from the bundled admin0
// The admin-0 bundle is quantized to ADM0_DECIMALS = 2 (scripts/build-atlas-geo.mjs) and no
// coordinate passes ±180, so every one is a whole number of hundredths within ±18000 and fits
// an Int16 exactly (ATLAS-BUNDLE-010 guards it). A third decimal needs ×1000 here and an
// Int32Array, since ±180000 does not fit an Int16.
const ADMIN0_SCALE = 100;
// ISO_A2 → packed admin0 geometry + bounding boxes, derived from the bundled admin0
// borders on first use. The parsed FeatureCollection is dropped after this runs; only the
// Float64Array rings and boxes are retained (≈1MB vs the ≈145MB parsed geometry, #1576).
// Int16 coordinates and the boxes are retained (≈9MB for the 2M points, vs ≈34MB as
// Float64Array rings and ≈145MB as the parsed geometry, #1576).
//
// The boxes used to be a hand-maintained table, which drifted: 43 countries (NG, BY,
// GL, KP, TD, SS, …) had no box at all, so their coordinates fell into a *neighbour's*
@@ -269,20 +236,20 @@ type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
// One box is stored PER GEOMETRY PART, not per country. A single box around a country
// that straddles the antimeridian (RU, US, FJ, KI) would span nearly the whole globe;
// per-part boxes keep Alaska and Chukotka separate and handle the ±180 wrap for free.
let countryPolyIndex: Map<string, CompactGeom> | null = null;
let countryPolyIndex: Map<string, PackedGeom> | null = null;
let countryBoxIndex: Map<string, Box[]> | null = null;
function buildCountryIndexes(): void {
const polys = new Map<string, CompactGeom>();
const polys = new Map<string, PackedGeom>();
const boxes = new Map<string, Box[]>();
const gz = loadAdmin0Gz();
if (gz) {
const file = getAdmin0File();
if (file) {
// Parse ONE feature at a time off the gunzipped string rather than JSON.parse-ing the
// whole FeatureCollection: the full parse transiently allocates ~285MB (the 145MB
// object graph plus intermediates) and V8 keeps those pages, which alone exhausts a
// 512MB host before admin1 even loads (#1576).
const json = zlib.gunzipSync(gz).toString('utf8');
const json = zlib.gunzipSync(fs.readFileSync(file)).toString('utf8');
const consume = createFeatureSplitter((text) => {
const f = JSON.parse(text);
const raw = f.properties?.ISO_A2;
@@ -292,18 +259,27 @@ function buildCountryIndexes(): void {
const parts = (
f.geometry.type === 'Polygon' ? [f.geometry.coordinates] : f.geometry.coordinates
) as number[][][][];
const rings: Float64Array[] = [];
const polyRingCounts: number[] = [];
const codeBoxes = boxes.get(code) ?? [];
let points = 0;
let ringCount = 0;
for (const part of parts) {
polyRingCounts.push(part.length);
ringCount += part.length;
for (const ring of part) points += ring.length;
}
const coords = new Int16Array(points * 2);
const ringStarts = new Uint32Array(ringCount + 1);
const polyRingCounts = new Uint32Array(parts.length);
const codeBoxes = boxes.get(code) ?? [];
let p = 0;
let ri = 0;
for (const [k, part] of parts.entries()) {
polyRingCounts[k] = part.length;
for (const ring of part) {
const flat = new Float64Array(ring.length * 2);
for (let i = 0; i < ring.length; i++) {
flat[2 * i] = ring[i][0];
flat[2 * i + 1] = ring[i][1];
ringStarts[ri++] = p;
for (const [lng, lat] of ring) {
coords[2 * p] = Math.round(lng * ADMIN0_SCALE);
coords[2 * p + 1] = Math.round(lat * ADMIN0_SCALE);
p++;
}
rings.push(flat);
}
// Bounding box from the part's outer ring (part[0]).
let minLng = Infinity,
@@ -318,9 +294,10 @@ function buildCountryIndexes(): void {
}
codeBoxes.push([minLng, minLat, maxLng, maxLat]);
}
ringStarts[ri] = p;
// Matches the previous index exactly: geometry is overwritten (last feature for a
// code wins) while boxes accumulate across a code's features.
polys.set(code, { rings, polyRingCounts });
polys.set(code, { coords, ringStarts, polyRingCounts });
boxes.set(code, codeBoxes);
});
consume(json);
@@ -335,7 +312,7 @@ function buildCountryIndexes(): void {
countryBoxIndex = boxes;
}
function getCountryPolyIndex(): Map<string, CompactGeom> {
function getCountryPolyIndex(): Map<string, PackedGeom> {
if (!countryPolyIndex) buildCountryIndexes();
return countryPolyIndex!;
}
@@ -393,7 +370,7 @@ export function getCountryFromCoords(lat: number, lng: number): string | null {
if (looseBoxFallback === null) looseBoxFallback = code;
continue;
}
if (pointInGeometry(lng, lat, poly)) return code;
if (pointInPackedGeometry(lng, lat, poly, ADMIN0_SCALE)) return code;
}
// No tight micro-box and no polygon contained the point — prefer a deferred loose box
// (a real PS/XK point), else fall back to the smallest box (coastal slop / data gap).
+15 -2
View File
@@ -16,6 +16,7 @@ import { VisitedRegions } from '../../src/db/entities/VisitedRegions.entity';
import { AtlasModule } from '../../src/nest/atlas/atlas.module';
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
import { getAdmin0File } from '../../src/utils/country-borders';
import { createUser, createTrip } from '../helpers/factories';
import { markCountryVisited } from '../helpers/factories/atlas';
import { makePlace } from '../helpers/factories/places';
@@ -28,6 +29,7 @@ import { sessionCookie } from './harness';
import { Test } from '@nestjs/testing';
import cookieParser from 'cookie-parser';
import fs from 'fs';
import type { Server } from 'http';
import request from 'supertest';
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
@@ -85,11 +87,22 @@ describe('Atlas e2e (real auth guard + real service + temp SQLite)', () => {
it('200 countries/geo serves the bundled gzipped admin-0 that the client decompresses to a FeatureCollection', async () => {
const res = await request(server).get('/api/addons/atlas/countries/geo').set('Cookie', sessionCookie(userId));
expect(res.status).toBe(200);
expect(res.headers['content-encoding']).toBe('gzip');
const admin0 = getAdmin0File();
expect(admin0).not.toBeNull();
// Sent from disk with the headers res.send(buffer) gave when the bytes were kept in
// memory: send() replaces none of them and adds no Accept-Ranges or Last-Modified.
expect(res.headers).toMatchObject({
'content-type': 'application/json; charset=utf-8',
'content-encoding': 'gzip',
'cache-control': 'public, max-age=86400',
'content-length': String(fs.statSync(String(admin0)).size),
});
expect(res.headers.etag).toMatch(/^W\/"/);
expect(res.headers).not.toHaveProperty('accept-ranges');
expect(res.headers).not.toHaveProperty('last-modified');
// superagent transparently decompresses, mirroring the browser.
expect(res.body.type).toBe('FeatureCollection');
expect(res.body.features.length).toBeGreaterThan(0);
expect(res.headers['cache-control']).toContain('max-age=86400');
});
it('200 stats for an authenticated user — the trip-less branch keeps its bespoke shape', async () => {
@@ -131,6 +131,8 @@ describe('BOOTSTRAP (F6) — unified NestJS app serves the whole surface', () =>
.set('Cookie', authCookie(user.id));
expect(res.status).toBe(200);
expect(res.headers['content-encoding']).toBe('gzip');
// Gzipped once: the compression middleware leaves the file's own encoding alone.
expect(res.body.type).toBe('FeatureCollection');
});
it('BOOTSTRAP-ORM-001: the ORM unit of work resolves from the container', () => {
@@ -0,0 +1,183 @@
import { getAdmin0File, getCountryFromCoords } from '../../../src/utils/country-borders';
import fs from 'fs';
import { describe, expect, it } from 'vitest';
import zlib from 'zlib';
// The packed admin-0 country index behind getCountryFromCoords: Int16 ×100 coordinates in
// place of Float64Array rings.
type Geometry = { type: string; coordinates: unknown };
type Feature = { properties: { ISO_A2?: string | null }; geometry: Geometry | null };
const admin0 = getAdmin0File();
if (!admin0) throw new Error('admin0.geojson.gz missing');
const features = (JSON.parse(zlib.gunzipSync(fs.readFileSync(admin0)).toString('utf8')) as { features: Feature[] })
.features;
// ── The lookup as it was before the packing, kept as the reference ─────────
// Float64 rings, the same boxes, micro-territories and ranking: what getCountryFromCoords
// did before this index, copied from country-borders.ts.
type OldGeom = { rings: Float64Array[]; polyRingCounts: number[] };
type Box = [number, number, number, number];
const MICRO_TERRITORY_BOXES: Record<string, Box> = {
HK: [113.83, 22.15, 114.43, 22.56],
MO: [113.53, 22.1, 113.6, 22.21],
GI: [-5.36, 36.11, -5.33, 36.16],
PR: [-67.3, 17.88, -65.22, 18.53],
PS: [34.2, 29.5, 35.6, 32.6],
XK: [20.0, 41.9, 21.8, 43.3],
};
const MICRO_BOX_MAX_AREA = 2.0;
function oldIndexes(): { polys: Map<string, OldGeom>; boxes: Map<string, Box[]> } {
const polys = new Map<string, OldGeom>();
const boxes = new Map<string, Box[]>();
for (const f of features) {
const raw = f.properties?.ISO_A2;
if (!raw || raw === '-99' || !f.geometry) continue;
const code = String(raw).toUpperCase();
const g = f.geometry;
const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][];
const rings: Float64Array[] = [];
const polyRingCounts: number[] = [];
const codeBoxes = boxes.get(code) ?? [];
for (const part of parts) {
polyRingCounts.push(part.length);
for (const ring of part) rings.push(Float64Array.from(ring.flat()));
const lngs = part[0].map(([lng]) => lng);
const lats = part[0].map(([, lat]) => lat);
codeBoxes.push([Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)]);
}
polys.set(code, { rings, polyRingCounts });
boxes.set(code, codeBoxes);
}
for (const [code, box] of Object.entries(MICRO_TERRITORY_BOXES)) {
if (!boxes.has(code)) boxes.set(code, [box]);
}
return { polys, boxes };
}
function oldPointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean {
let inside = false;
const n = ring.length / 2;
for (let i = 0, j = n - 1; i < n; j = i++) {
const xi = ring[2 * i],
yi = ring[2 * i + 1];
const xj = ring[2 * j],
yj = ring[2 * j + 1];
if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) {
inside = !inside;
}
}
return inside;
}
function oldPointInGeometry(lng: number, lat: number, geom: OldGeom): boolean {
let ri = 0;
for (const rc of geom.polyRingCounts) {
if (oldPointInFlatRing(lng, lat, geom.rings[ri])) {
let inHole = false;
for (let h = 1; h < rc; h++) {
if (oldPointInFlatRing(lng, lat, geom.rings[ri + h])) {
inHole = true;
break;
}
}
if (!inHole) return true;
}
ri += rc;
}
return false;
}
function oldLookup(old: ReturnType<typeof oldIndexes>, lat: number, lng: number): string | null {
const candidates: { code: string; area: number }[] = [];
for (const [code, boxes] of old.boxes) {
for (const [minLng, minLat, maxLng, maxLat] of boxes) {
if (lat >= minLat && lat <= maxLat && lng >= minLng && lng <= maxLng) {
candidates.push({ code, area: (maxLng - minLng) * (maxLat - minLat) });
break;
}
}
}
if (candidates.length === 0) return null;
candidates.sort((a, b) => a.area - b.area);
let looseBoxFallback: string | null = null;
for (const { code, area } of candidates) {
const poly = old.polys.get(code);
if (!poly) {
if (area <= MICRO_BOX_MAX_AREA) return code;
if (looseBoxFallback === null) looseBoxFallback = code;
continue;
}
if (oldPointInGeometry(lng, lat, poly)) return code;
}
return looseBoxFallback ?? candidates[0].code;
}
// Deterministic points per country: about `perCountry` vertices spread over all its rings,
// each exactly, 1e-7° to either side of it and of the next edge's middle (where a ray-cast
// is most fragile), plus random points in the country's box.
function testPoints(f: Feature, perCountry: number, random: () => number): [number, number][] {
if (!f.geometry) return [];
const g = f.geometry;
const rings = ((g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]).flat();
const total = rings.reduce((n, ring) => n + ring.length, 0);
const step = Math.max(1, Math.floor(total / perCountry));
const points: [number, number][] = [];
let seen = 0;
let minLng = Infinity,
minLat = Infinity,
maxLng = -Infinity,
maxLat = -Infinity;
for (const ring of rings) {
for (let i = 0; i < ring.length - 1; i++) {
const [lng, lat] = ring[i];
minLng = Math.min(minLng, lng);
maxLng = Math.max(maxLng, lng);
minLat = Math.min(minLat, lat);
maxLat = Math.max(maxLat, lat);
if (seen++ % step !== 0) continue;
const midLng = (lng + ring[i + 1][0]) / 2;
const midLat = (lat + ring[i + 1][1]) / 2;
points.push(
[lat, lng],
[lat + 1e-7, lng + 1e-7],
[lat - 1e-7, lng - 1e-7],
[midLat + 1e-7, midLng],
[midLat - 1e-7, midLng],
);
}
}
for (let k = 0; k < 20; k++) {
points.push([minLat + random() * (maxLat - minLat), minLng + random() * (maxLng - minLng)]);
}
return points;
}
describe('getCountryFromCoords on the packed country index', () => {
it('ATLAS-SVC-055: answers every test point the way the Float64 lookup did, in every country', () => {
const old = oldIndexes();
let seed = 0x2f6b;
const random = () => (seed = (seed * 16807) % 2147483647) / 2147483647;
let checked = 0;
let inCountry = 0;
const different: string[] = [];
for (const f of features) {
// Each point near a border is tested against the big polygons whose boxes cover it
// (RU, CA, US, CN, ...), so 100 vertices per country already take ~15s.
for (const [lat, lng] of testPoints(f, 100, random)) {
const before = oldLookup(old, lat, lng);
const after = getCountryFromCoords(lat, lng);
checked++;
if (before) inCountry++;
if (after !== before) different.push(`${f.properties.ISO_A2} ${lat},${lng}: ${before} → ${after}`);
}
}
expect(checked).toBeGreaterThan(100_000);
expect(inCountry / checked, 'most points are in a country').toBeGreaterThan(0.9);
expect(different.slice(0, 10)).toEqual([]);
}, 120_000);
});
@@ -19,11 +19,12 @@ function makeController(svc: Partial<AtlasService> & Partial<AtlasWritesService>
}
function makeRes() {
const res = { setHeader: vi.fn(), send: vi.fn(), json: vi.fn(), type: vi.fn() };
const res = { setHeader: vi.fn(), send: vi.fn(), sendFile: vi.fn(), json: vi.fn(), type: vi.fn() };
res.type.mockReturnValue(res);
return res as unknown as Response & {
setHeader: ReturnType<typeof vi.fn>;
send: ReturnType<typeof vi.fn>;
sendFile: ReturnType<typeof vi.fn>;
json: ReturnType<typeof vi.fn>;
type: ReturnType<typeof vi.fn>;
};
@@ -89,20 +90,24 @@ describe('AtlasController (parity with the legacy /api/addons/atlas route)', ()
});
});
it('GET /countries/geo serves the gzipped admin-0 bundle with Content-Encoding', () => {
const gz = Buffer.from('gzipped-bytes');
const countryGeoGz = vi.fn().mockReturnValue(gz);
it('GET /countries/geo sends the gzipped admin-0 file from disk with Content-Encoding', () => {
const countryGeoFile = vi.fn().mockReturnValue('/srv/trek/assets/atlas/admin0.geojson.gz');
const res = makeRes();
makeController({ countryGeoGz }).countryGeo(res);
makeController({ countryGeoFile }).countryGeo(res);
expect(res.setHeader).toHaveBeenCalledWith('Content-Type', 'application/json; charset=utf-8');
expect(res.setHeader).toHaveBeenCalledWith('Content-Encoding', 'gzip');
expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400');
expect(res.send).toHaveBeenCalledWith(gz);
expect(res.sendFile).toHaveBeenCalledWith('admin0.geojson.gz', {
root: '/srv/trek/assets/atlas',
acceptRanges: false,
lastModified: false,
});
});
it('GET /countries/geo falls back to an empty FeatureCollection when the bundle is missing', () => {
const countryGeoGz = vi.fn().mockReturnValue(null);
const countryGeoFile = vi.fn().mockReturnValue(null);
const res = makeRes();
makeController({ countryGeoGz }).countryGeo(res);
makeController({ countryGeoFile }).countryGeo(res);
expect(res.json).toHaveBeenCalledWith({ type: 'FeatureCollection', features: [] });
expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'no-store');
});
@@ -17,6 +17,7 @@ describe('Atlas admin0 country bundle (#1609)', () => {
const bundlePath = path.join(__dirname, '..', '..', '..', 'assets', 'atlas', 'admin0.geojson.gz');
const features = JSON.parse(zlib.gunzipSync(fs.readFileSync(bundlePath)).toString()).features as {
properties: { ISO_A2: string | null; ADM0_A3: string; NAME?: string };
geometry: { type: string; coordinates: number[][][] | number[][][][] } | null;
}[];
it('ATLAS-BUNDLE-003 — Kosovo ships with a resolvable ISO_A2', () => {
@@ -24,6 +25,28 @@ describe('Atlas admin0 country bundle (#1609)', () => {
expect(kosovo.length, 'exactly one Kosovo feature').toBe(1);
expect(kosovo[0].properties.ISO_A2).toBe('XK');
});
// The server's country index keeps every coordinate in an Int16Array ×100 (ADMIN0_SCALE in
// country-borders.ts) and divides it back on use. A value with a third decimal comes back
// changed, and one past ±327.67 wraps around without an error, so a rebuild with more
// decimals (ADM0_DECIMALS) must fail here, not move borders.
it('ATLAS-BUNDLE-010 — every admin0 coordinate round-trips through Int16 ×100 unchanged', () => {
const int16 = new Int16Array(1);
let values = 0;
const lossy: string[] = [];
for (const f of features) {
const g = f.geometry;
if (!g) continue;
const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][];
for (const v of parts.flat(3)) {
values++;
int16[0] = Math.round(v * 100);
if (int16[0] / 100 !== v) lossy.push(`${f.properties.ADM0_A3}: ${v} comes back as ${int16[0] / 100}`);
}
}
expect(values / 2, 'coordinate pairs checked').toBeGreaterThan(2_000_000);
expect(lossy.slice(0, 5)).toEqual([]);
});
});
describe('Atlas admin1 region bundle (#1217)', () => {