mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-22 13:10:29 +00:00
perf(atlas): keep the admin0 country index as Int16 arrays and send countries/geo from disk
This commit is contained in:
@@ -372,7 +372,7 @@ async function pool(items, worker) {
|
||||
// the now-redundant consecutive duplicate points shrinks the bundles ~5-8x with no
|
||||
// visible effect at the atlas' zoom range (3-10). ADM0 fills are viewed zoomed out, so
|
||||
// they tolerate a coarser grid than ADM1 region borders.
|
||||
const ADM0_DECIMALS = 2; // ~1.1 km
|
||||
const ADM0_DECIMALS = 2; // ~1.1 km; the server's country index packs ×100 into Int16 (ADMIN0_SCALE in country-borders.ts), raise both
|
||||
const ADM1_DECIMALS = 3; // ~110 m; the server's region cache packs ×1000 (REGION_SCALE in atlas-geo.ts), raise both
|
||||
|
||||
function quantizeRing(ring, decimals) {
|
||||
|
||||
@@ -5,9 +5,9 @@ import path from 'path';
|
||||
import zlib from 'zlib';
|
||||
import {
|
||||
admin1AssetPath,
|
||||
getAdmin0File,
|
||||
getCountryFromCoords,
|
||||
isPointInCountryBox,
|
||||
loadAdmin0Gz,
|
||||
pointInPackedGeometry,
|
||||
type PackedGeom,
|
||||
} from '../../utils/country-borders';
|
||||
@@ -30,30 +30,30 @@ export { getCountryFromCoords, isPointInCountryBox };
|
||||
|
||||
// ── Bundled boundary GeoJSON (admin-0 countries + admin-1 regions) ─────────
|
||||
//
|
||||
// Where the bundles come from, the path they are read from, the admin-0 gz bytes and
|
||||
// the country index built on them are in utils/country-borders.ts. The admin-1 half
|
||||
// Where the bundles come from, the path they are read from, the admin-0 file and
|
||||
// the country index built on it are in utils/country-borders.ts. The admin-1 half
|
||||
// is below.
|
||||
//
|
||||
// Neither parsed bundle is cached. admin0 (~145MB) and admin1 (~260MB) parsed at once are
|
||||
// what exhausted a 512MB host while using Atlas (#1576). Instead we retain only compact
|
||||
// derivatives: the raw admin0 .gz bytes for direct serving and a Float64Array poly/box
|
||||
// index for server-side point-in-polygon (see buildCountryIndexes in
|
||||
// utils/country-borders.ts). admin1 ships as one gzipped file per country, which is
|
||||
// served from disk as it is and read one country at a time for region lookups.
|
||||
// derivatives: the admin0 .gz is sent from disk and an Int16Array poly/box index serves
|
||||
// server-side point-in-polygon (see buildCountryIndexes in utils/country-borders.ts).
|
||||
// admin1 ships as one gzipped file per country, which is served from disk as it is and
|
||||
// read one country at a time for region lookups.
|
||||
|
||||
/** admin-0 country borders as gzipped GeoJSON bytes, served to the client map with
|
||||
/** Path of the gzipped admin-0 country borders, sent to the client map with
|
||||
* Content-Encoding: gzip so the server never holds the parsed FeatureCollection. */
|
||||
export function getCountryGeoGz(): Buffer | null {
|
||||
return loadAdmin0Gz();
|
||||
export function getCountryGeoFile(): string | null {
|
||||
return getAdmin0File();
|
||||
}
|
||||
|
||||
/** Parsed admin-0 FeatureCollection, parsed on demand (not cached). Not on the client hot
|
||||
* path — the map is served the gz bytes via getCountryGeoGz — but kept for internal/test
|
||||
* path — the map is sent the gz file via getCountryGeoFile — but kept for internal/test
|
||||
* callers that need the objects. */
|
||||
export function getCountryGeo(): any {
|
||||
const gz = loadAdmin0Gz();
|
||||
if (!gz) return { type: 'FeatureCollection', features: [] };
|
||||
return JSON.parse(zlib.gunzipSync(gz).toString('utf8'));
|
||||
const file = getAdmin0File();
|
||||
if (!file) return { type: 'FeatureCollection', features: [] };
|
||||
return JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf8'));
|
||||
}
|
||||
|
||||
// A country's admin-1 file, or null for anything that is not a two-letter code, so
|
||||
@@ -86,7 +86,7 @@ function readAdmin1Json(countryCode: string): Buffer | null {
|
||||
}
|
||||
|
||||
/** One country's admin-1 regions as gzipped GeoJSON bytes, served to the client map with
|
||||
* Content-Encoding: gzip like getCountryGeoGz, so the server never parses them. */
|
||||
* Content-Encoding: gzip like getCountryGeoFile, so the server never parses them. */
|
||||
export async function getRegionGeoGz(countryCode: string): Promise<Buffer | null> {
|
||||
const file = admin1File(countryCode);
|
||||
if (!file) return null;
|
||||
|
||||
@@ -26,6 +26,7 @@ import { regionGeoParamsSchema } from '@trek/shared';
|
||||
import type { AtlasLocateResponse, RegionGeoParams } from '@trek/shared';
|
||||
|
||||
import type { Response } from 'express';
|
||||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* /api/addons/atlas — visited countries/regions, region GeoJSON, bucket list.
|
||||
@@ -123,10 +124,13 @@ export class AtlasController {
|
||||
countryGeo(@Res() res: Response): void {
|
||||
// Serve the pre-gzipped admin-0 bundle straight from disk. The browser decompresses
|
||||
// transparently, so the wire shape is identical to before, but the server never parses
|
||||
// or holds the ~145MB FeatureCollection (#1576). Content-Encoding is set explicitly, so
|
||||
// the compression middleware leaves the body untouched.
|
||||
const gz = this.atlas.countryGeoGz();
|
||||
if (!gz) {
|
||||
// or holds the ~145MB FeatureCollection (#1576), nor the 4MB file itself. Content-Encoding
|
||||
// is set explicitly, so the compression middleware leaves the body untouched, and
|
||||
// send() keeps the Content-Type and Cache-Control set here. Without Accept-Ranges and
|
||||
// Last-Modified the headers are the ones res.send(buffer) gave. Root + basename, not the
|
||||
// absolute path: see plugin-frame.controller.ts.
|
||||
const file = this.atlas.countryGeoFile();
|
||||
if (!file) {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.json({ type: 'FeatureCollection', features: [] });
|
||||
return;
|
||||
@@ -134,7 +138,7 @@ export class AtlasController {
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Content-Encoding', 'gzip');
|
||||
res.setHeader('Cache-Control', 'public, max-age=86400');
|
||||
res.send(gz);
|
||||
res.sendFile(path.basename(file), { root: path.dirname(file), acceptRanges: false, lastModified: false });
|
||||
}
|
||||
|
||||
@Get('country/:code')
|
||||
|
||||
@@ -26,7 +26,7 @@ import type { ReservationEndpointsRepository, TravelerOwnedEndpointRow } from '.
|
||||
import {
|
||||
getCountryFromAddress,
|
||||
getCountryFromCoords,
|
||||
getCountryGeoGz,
|
||||
getCountryGeoFile,
|
||||
getRegionFromCoords,
|
||||
geocodingInFlight,
|
||||
resolveCountryCodeSync,
|
||||
@@ -876,8 +876,8 @@ export class AtlasService {
|
||||
};
|
||||
}
|
||||
|
||||
countryGeoGz(): Buffer | null {
|
||||
return getCountryGeoGz();
|
||||
countryGeoFile(): string | null {
|
||||
return getCountryGeoFile();
|
||||
}
|
||||
|
||||
// ── Bucket list CRUD ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -11,8 +11,8 @@ import zlib from 'zlib';
|
||||
// for the country an index suggestion names. Importing it from atlas tied maps into
|
||||
// an import cycle with atlas (lint:boundaries).
|
||||
//
|
||||
// atlas-geo resolves admin-1 regions with the per-country asset path and the ring test
|
||||
// exported here. The admin0 gz bytes and the country poly/box
|
||||
// atlas-geo resolves admin-1 regions with the per-country asset path and the packed ring
|
||||
// test exported here. The admin0 file path and the country poly/box
|
||||
// indexes below are file-scoped for the reason atlas-geo gives: their lifetime stays
|
||||
// process-global however many callers there are, so the indexes are built once per
|
||||
// process, which the #1576 OOM fix depends on.
|
||||
@@ -38,15 +38,17 @@ export function admin1AssetPath(countryCode: string): string {
|
||||
return path.join(__dirname, '..', '..', 'assets', 'atlas', 'admin1', `${countryCode}.geojson.gz`);
|
||||
}
|
||||
|
||||
let admin0Gz: Buffer | null | undefined;
|
||||
export function loadAdmin0Gz(): Buffer | null {
|
||||
if (admin0Gz !== undefined) return admin0Gz;
|
||||
// The admin0 bytes are not kept: countries/geo sends the file from disk, and the country
|
||||
// index below reads it once while it is built.
|
||||
let admin0File: string | null | undefined;
|
||||
export function getAdmin0File(): string | null {
|
||||
if (admin0File !== undefined) return admin0File;
|
||||
const file = assetPath('admin0');
|
||||
if (!fs.existsSync(file)) {
|
||||
console.warn(`[Atlas] admin0.geojson.gz missing — run \`node scripts/build-atlas-geo.mjs\``);
|
||||
return (admin0Gz = null);
|
||||
return (admin0File = null);
|
||||
}
|
||||
return (admin0Gz = fs.readFileSync(file));
|
||||
return (admin0File = file);
|
||||
}
|
||||
|
||||
// Feed arbitrary gunzip chunks; invokes onFeature(text) once per top-level Feature object
|
||||
@@ -159,53 +161,14 @@ const MICRO_BOX_MAX_AREA = 2.0;
|
||||
|
||||
// ── Point-in-polygon over the bundled admin0 borders (#1331) ─────────────────
|
||||
|
||||
// Ray-casting (even-odd) test of (lng,lat) against a single GeoJSON ring.
|
||||
// Ray-cast on a flat [lng,lat,lng,lat,…] ring. Same algorithm as the classic number[][]
|
||||
// version, but the coordinates live in a Float64Array so the parsed admin0 geometry (with
|
||||
// its millions of tiny [lng,lat] arrays, ~145MB) need not be retained — only these rings.
|
||||
function pointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean {
|
||||
let inside = false;
|
||||
const n = ring.length / 2;
|
||||
for (let i = 0, j = n - 1; i < n; j = i++) {
|
||||
const xi = ring[2 * i],
|
||||
yi = ring[2 * i + 1];
|
||||
const xj = ring[2 * j],
|
||||
yj = ring[2 * j + 1];
|
||||
if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) {
|
||||
inside = !inside;
|
||||
}
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
// True when (lng,lat) falls inside a compact Polygon/MultiPolygon, honouring holes.
|
||||
// `rings` is every ring flattened; `polyRingCounts[k]` is how many rings polygon k owns
|
||||
// (its first ring is the outer boundary, the rest are holes).
|
||||
function pointInGeometry(lng: number, lat: number, geom: CompactGeom): boolean {
|
||||
let ri = 0;
|
||||
for (const rc of geom.polyRingCounts) {
|
||||
if (pointInFlatRing(lng, lat, geom.rings[ri])) {
|
||||
let inHole = false;
|
||||
for (let h = 1; h < rc; h++) {
|
||||
if (pointInFlatRing(lng, lat, geom.rings[ri + h])) {
|
||||
inHole = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!inHole) return true;
|
||||
}
|
||||
ri += rc;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// The ray-cast above on ring pairs [from, to) of packed coordinates. Each one is divided back
|
||||
// by the scale before use: for coordinates with no more decimals than the scale holds that
|
||||
// gives back the exact double the bundle had, so the answer is the one the Float64 ring gives.
|
||||
// Ray-casting (even-odd) test of (lng,lat) against ring pairs [from, to) of packed
|
||||
// coordinates. Each one is divided back by the scale before use: for coordinates with no
|
||||
// more decimals than the scale holds that gives back the exact double the bundle had, so the
|
||||
// answer is the one a Float64 ring of the same points gives.
|
||||
function pointInPackedRing(
|
||||
lng: number,
|
||||
lat: number,
|
||||
coords: Int32Array,
|
||||
coords: Int16Array | Int32Array,
|
||||
scale: number,
|
||||
from: number,
|
||||
to: number,
|
||||
@@ -225,7 +188,8 @@ function pointInPackedRing(
|
||||
return inside;
|
||||
}
|
||||
|
||||
// pointInGeometry for a PackedGeom.
|
||||
// True when (lng,lat) falls inside a packed Polygon/MultiPolygon, honouring holes: the
|
||||
// first ring of each polygon is its outer boundary, the rest are holes.
|
||||
export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom, scale: number): boolean {
|
||||
const { coords, ringStarts, polyRingCounts } = geom;
|
||||
let ri = 0;
|
||||
@@ -245,20 +209,23 @@ export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compact polygon geometry: rings flattened into Float64Arrays, grouped into polygons by
|
||||
// polyRingCounts. Replaces the retained parsed GeoJSON geometry.
|
||||
type CompactGeom = { rings: Float64Array[]; polyRingCounts: number[] };
|
||||
|
||||
// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one Int32Array, each
|
||||
// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts
|
||||
// groups the rings into polygons as in CompactGeom. Three typed arrays per geometry instead
|
||||
// of one per ring, at half the bytes per point.
|
||||
export type PackedGeom = { coords: Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array };
|
||||
// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one typed array, each
|
||||
// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts[k]
|
||||
// is how many rings polygon k owns. Three typed arrays per geometry instead of one per ring,
|
||||
// at a quarter (Int16, admin-0) or half (Int32, admin-1) of the bytes per point.
|
||||
export type PackedGeom = { coords: Int16Array | Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array };
|
||||
type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
|
||||
|
||||
// ISO_A2 → compact admin0 geometry + bounding boxes, derived from the bundled admin0
|
||||
// The admin-0 bundle is quantized to ADM0_DECIMALS = 2 (scripts/build-atlas-geo.mjs) and no
|
||||
// coordinate passes ±180, so every one is a whole number of hundredths within ±18000 and fits
|
||||
// an Int16 exactly (ATLAS-BUNDLE-010 guards it). A third decimal needs ×1000 here and an
|
||||
// Int32Array, since ±180000 does not fit an Int16.
|
||||
const ADMIN0_SCALE = 100;
|
||||
|
||||
// ISO_A2 → packed admin0 geometry + bounding boxes, derived from the bundled admin0
|
||||
// borders on first use. The parsed FeatureCollection is dropped after this runs; only the
|
||||
// Float64Array rings and boxes are retained (≈1MB vs the ≈145MB parsed geometry, #1576).
|
||||
// Int16 coordinates and the boxes are retained (≈9MB for the 2M points, vs ≈34MB as
|
||||
// Float64Array rings and ≈145MB as the parsed geometry, #1576).
|
||||
//
|
||||
// The boxes used to be a hand-maintained table, which drifted: 43 countries (NG, BY,
|
||||
// GL, KP, TD, SS, …) had no box at all, so their coordinates fell into a *neighbour's*
|
||||
@@ -269,20 +236,20 @@ type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
|
||||
// One box is stored PER GEOMETRY PART, not per country. A single box around a country
|
||||
// that straddles the antimeridian (RU, US, FJ, KI) would span nearly the whole globe;
|
||||
// per-part boxes keep Alaska and Chukotka separate and handle the ±180 wrap for free.
|
||||
let countryPolyIndex: Map<string, CompactGeom> | null = null;
|
||||
let countryPolyIndex: Map<string, PackedGeom> | null = null;
|
||||
let countryBoxIndex: Map<string, Box[]> | null = null;
|
||||
|
||||
function buildCountryIndexes(): void {
|
||||
const polys = new Map<string, CompactGeom>();
|
||||
const polys = new Map<string, PackedGeom>();
|
||||
const boxes = new Map<string, Box[]>();
|
||||
|
||||
const gz = loadAdmin0Gz();
|
||||
if (gz) {
|
||||
const file = getAdmin0File();
|
||||
if (file) {
|
||||
// Parse ONE feature at a time off the gunzipped string rather than JSON.parse-ing the
|
||||
// whole FeatureCollection: the full parse transiently allocates ~285MB (the 145MB
|
||||
// object graph plus intermediates) and V8 keeps those pages, which alone exhausts a
|
||||
// 512MB host before admin1 even loads (#1576).
|
||||
const json = zlib.gunzipSync(gz).toString('utf8');
|
||||
const json = zlib.gunzipSync(fs.readFileSync(file)).toString('utf8');
|
||||
const consume = createFeatureSplitter((text) => {
|
||||
const f = JSON.parse(text);
|
||||
const raw = f.properties?.ISO_A2;
|
||||
@@ -292,18 +259,27 @@ function buildCountryIndexes(): void {
|
||||
const parts = (
|
||||
f.geometry.type === 'Polygon' ? [f.geometry.coordinates] : f.geometry.coordinates
|
||||
) as number[][][][];
|
||||
const rings: Float64Array[] = [];
|
||||
const polyRingCounts: number[] = [];
|
||||
const codeBoxes = boxes.get(code) ?? [];
|
||||
let points = 0;
|
||||
let ringCount = 0;
|
||||
for (const part of parts) {
|
||||
polyRingCounts.push(part.length);
|
||||
ringCount += part.length;
|
||||
for (const ring of part) points += ring.length;
|
||||
}
|
||||
const coords = new Int16Array(points * 2);
|
||||
const ringStarts = new Uint32Array(ringCount + 1);
|
||||
const polyRingCounts = new Uint32Array(parts.length);
|
||||
const codeBoxes = boxes.get(code) ?? [];
|
||||
let p = 0;
|
||||
let ri = 0;
|
||||
for (const [k, part] of parts.entries()) {
|
||||
polyRingCounts[k] = part.length;
|
||||
for (const ring of part) {
|
||||
const flat = new Float64Array(ring.length * 2);
|
||||
for (let i = 0; i < ring.length; i++) {
|
||||
flat[2 * i] = ring[i][0];
|
||||
flat[2 * i + 1] = ring[i][1];
|
||||
ringStarts[ri++] = p;
|
||||
for (const [lng, lat] of ring) {
|
||||
coords[2 * p] = Math.round(lng * ADMIN0_SCALE);
|
||||
coords[2 * p + 1] = Math.round(lat * ADMIN0_SCALE);
|
||||
p++;
|
||||
}
|
||||
rings.push(flat);
|
||||
}
|
||||
// Bounding box from the part's outer ring (part[0]).
|
||||
let minLng = Infinity,
|
||||
@@ -318,9 +294,10 @@ function buildCountryIndexes(): void {
|
||||
}
|
||||
codeBoxes.push([minLng, minLat, maxLng, maxLat]);
|
||||
}
|
||||
ringStarts[ri] = p;
|
||||
// Matches the previous index exactly: geometry is overwritten (last feature for a
|
||||
// code wins) while boxes accumulate across a code's features.
|
||||
polys.set(code, { rings, polyRingCounts });
|
||||
polys.set(code, { coords, ringStarts, polyRingCounts });
|
||||
boxes.set(code, codeBoxes);
|
||||
});
|
||||
consume(json);
|
||||
@@ -335,7 +312,7 @@ function buildCountryIndexes(): void {
|
||||
countryBoxIndex = boxes;
|
||||
}
|
||||
|
||||
function getCountryPolyIndex(): Map<string, CompactGeom> {
|
||||
function getCountryPolyIndex(): Map<string, PackedGeom> {
|
||||
if (!countryPolyIndex) buildCountryIndexes();
|
||||
return countryPolyIndex!;
|
||||
}
|
||||
@@ -393,7 +370,7 @@ export function getCountryFromCoords(lat: number, lng: number): string | null {
|
||||
if (looseBoxFallback === null) looseBoxFallback = code;
|
||||
continue;
|
||||
}
|
||||
if (pointInGeometry(lng, lat, poly)) return code;
|
||||
if (pointInPackedGeometry(lng, lat, poly, ADMIN0_SCALE)) return code;
|
||||
}
|
||||
// No tight micro-box and no polygon contained the point — prefer a deferred loose box
|
||||
// (a real PS/XK point), else fall back to the smallest box (coastal slop / data gap).
|
||||
|
||||
@@ -16,6 +16,7 @@ import { VisitedRegions } from '../../src/db/entities/VisitedRegions.entity';
|
||||
import { AtlasModule } from '../../src/nest/atlas/atlas.module';
|
||||
import { TrekExceptionFilter } from '../../src/nest/common/trek-exception.filter';
|
||||
import { ZodValidationPipe } from '../../src/nest/common/zod-validation.pipe';
|
||||
import { getAdmin0File } from '../../src/utils/country-borders';
|
||||
import { createUser, createTrip } from '../helpers/factories';
|
||||
import { markCountryVisited } from '../helpers/factories/atlas';
|
||||
import { makePlace } from '../helpers/factories/places';
|
||||
@@ -28,6 +29,7 @@ import { sessionCookie } from './harness';
|
||||
import { Test } from '@nestjs/testing';
|
||||
|
||||
import cookieParser from 'cookie-parser';
|
||||
import fs from 'fs';
|
||||
import type { Server } from 'http';
|
||||
import request from 'supertest';
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
@@ -85,11 +87,22 @@ describe('Atlas e2e (real auth guard + real service + temp SQLite)', () => {
|
||||
it('200 countries/geo serves the bundled gzipped admin-0 that the client decompresses to a FeatureCollection', async () => {
|
||||
const res = await request(server).get('/api/addons/atlas/countries/geo').set('Cookie', sessionCookie(userId));
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-encoding']).toBe('gzip');
|
||||
const admin0 = getAdmin0File();
|
||||
expect(admin0).not.toBeNull();
|
||||
// Sent from disk with the headers res.send(buffer) gave when the bytes were kept in
|
||||
// memory: send() replaces none of them and adds no Accept-Ranges or Last-Modified.
|
||||
expect(res.headers).toMatchObject({
|
||||
'content-type': 'application/json; charset=utf-8',
|
||||
'content-encoding': 'gzip',
|
||||
'cache-control': 'public, max-age=86400',
|
||||
'content-length': String(fs.statSync(String(admin0)).size),
|
||||
});
|
||||
expect(res.headers.etag).toMatch(/^W\/"/);
|
||||
expect(res.headers).not.toHaveProperty('accept-ranges');
|
||||
expect(res.headers).not.toHaveProperty('last-modified');
|
||||
// superagent transparently decompresses, mirroring the browser.
|
||||
expect(res.body.type).toBe('FeatureCollection');
|
||||
expect(res.body.features.length).toBeGreaterThan(0);
|
||||
expect(res.headers['cache-control']).toContain('max-age=86400');
|
||||
});
|
||||
|
||||
it('200 stats for an authenticated user — the trip-less branch keeps its bespoke shape', async () => {
|
||||
|
||||
@@ -131,6 +131,8 @@ describe('BOOTSTRAP (F6) — unified NestJS app serves the whole surface', () =>
|
||||
.set('Cookie', authCookie(user.id));
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers['content-encoding']).toBe('gzip');
|
||||
// Gzipped once: the compression middleware leaves the file's own encoding alone.
|
||||
expect(res.body.type).toBe('FeatureCollection');
|
||||
});
|
||||
|
||||
it('BOOTSTRAP-ORM-001: the ORM unit of work resolves from the container', () => {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import { getAdmin0File, getCountryFromCoords } from '../../../src/utils/country-borders';
|
||||
|
||||
import fs from 'fs';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import zlib from 'zlib';
|
||||
|
||||
// The packed admin-0 country index behind getCountryFromCoords: Int16 ×100 coordinates in
|
||||
// place of Float64Array rings.
|
||||
|
||||
type Geometry = { type: string; coordinates: unknown };
|
||||
type Feature = { properties: { ISO_A2?: string | null }; geometry: Geometry | null };
|
||||
const admin0 = getAdmin0File();
|
||||
if (!admin0) throw new Error('admin0.geojson.gz missing');
|
||||
const features = (JSON.parse(zlib.gunzipSync(fs.readFileSync(admin0)).toString('utf8')) as { features: Feature[] })
|
||||
.features;
|
||||
|
||||
// ── The lookup as it was before the packing, kept as the reference ─────────
|
||||
// Float64 rings, the same boxes, micro-territories and ranking: what getCountryFromCoords
|
||||
// did before this index, copied from country-borders.ts.
|
||||
|
||||
type OldGeom = { rings: Float64Array[]; polyRingCounts: number[] };
|
||||
type Box = [number, number, number, number];
|
||||
|
||||
const MICRO_TERRITORY_BOXES: Record<string, Box> = {
|
||||
HK: [113.83, 22.15, 114.43, 22.56],
|
||||
MO: [113.53, 22.1, 113.6, 22.21],
|
||||
GI: [-5.36, 36.11, -5.33, 36.16],
|
||||
PR: [-67.3, 17.88, -65.22, 18.53],
|
||||
PS: [34.2, 29.5, 35.6, 32.6],
|
||||
XK: [20.0, 41.9, 21.8, 43.3],
|
||||
};
|
||||
const MICRO_BOX_MAX_AREA = 2.0;
|
||||
|
||||
function oldIndexes(): { polys: Map<string, OldGeom>; boxes: Map<string, Box[]> } {
|
||||
const polys = new Map<string, OldGeom>();
|
||||
const boxes = new Map<string, Box[]>();
|
||||
for (const f of features) {
|
||||
const raw = f.properties?.ISO_A2;
|
||||
if (!raw || raw === '-99' || !f.geometry) continue;
|
||||
const code = String(raw).toUpperCase();
|
||||
const g = f.geometry;
|
||||
const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][];
|
||||
const rings: Float64Array[] = [];
|
||||
const polyRingCounts: number[] = [];
|
||||
const codeBoxes = boxes.get(code) ?? [];
|
||||
for (const part of parts) {
|
||||
polyRingCounts.push(part.length);
|
||||
for (const ring of part) rings.push(Float64Array.from(ring.flat()));
|
||||
const lngs = part[0].map(([lng]) => lng);
|
||||
const lats = part[0].map(([, lat]) => lat);
|
||||
codeBoxes.push([Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)]);
|
||||
}
|
||||
polys.set(code, { rings, polyRingCounts });
|
||||
boxes.set(code, codeBoxes);
|
||||
}
|
||||
for (const [code, box] of Object.entries(MICRO_TERRITORY_BOXES)) {
|
||||
if (!boxes.has(code)) boxes.set(code, [box]);
|
||||
}
|
||||
return { polys, boxes };
|
||||
}
|
||||
|
||||
function oldPointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean {
|
||||
let inside = false;
|
||||
const n = ring.length / 2;
|
||||
for (let i = 0, j = n - 1; i < n; j = i++) {
|
||||
const xi = ring[2 * i],
|
||||
yi = ring[2 * i + 1];
|
||||
const xj = ring[2 * j],
|
||||
yj = ring[2 * j + 1];
|
||||
if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) {
|
||||
inside = !inside;
|
||||
}
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
function oldPointInGeometry(lng: number, lat: number, geom: OldGeom): boolean {
|
||||
let ri = 0;
|
||||
for (const rc of geom.polyRingCounts) {
|
||||
if (oldPointInFlatRing(lng, lat, geom.rings[ri])) {
|
||||
let inHole = false;
|
||||
for (let h = 1; h < rc; h++) {
|
||||
if (oldPointInFlatRing(lng, lat, geom.rings[ri + h])) {
|
||||
inHole = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!inHole) return true;
|
||||
}
|
||||
ri += rc;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function oldLookup(old: ReturnType<typeof oldIndexes>, lat: number, lng: number): string | null {
|
||||
const candidates: { code: string; area: number }[] = [];
|
||||
for (const [code, boxes] of old.boxes) {
|
||||
for (const [minLng, minLat, maxLng, maxLat] of boxes) {
|
||||
if (lat >= minLat && lat <= maxLat && lng >= minLng && lng <= maxLng) {
|
||||
candidates.push({ code, area: (maxLng - minLng) * (maxLat - minLat) });
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (candidates.length === 0) return null;
|
||||
candidates.sort((a, b) => a.area - b.area);
|
||||
let looseBoxFallback: string | null = null;
|
||||
for (const { code, area } of candidates) {
|
||||
const poly = old.polys.get(code);
|
||||
if (!poly) {
|
||||
if (area <= MICRO_BOX_MAX_AREA) return code;
|
||||
if (looseBoxFallback === null) looseBoxFallback = code;
|
||||
continue;
|
||||
}
|
||||
if (oldPointInGeometry(lng, lat, poly)) return code;
|
||||
}
|
||||
return looseBoxFallback ?? candidates[0].code;
|
||||
}
|
||||
|
||||
// Deterministic points per country: about `perCountry` vertices spread over all its rings,
|
||||
// each exactly, 1e-7° to either side of it and of the next edge's middle (where a ray-cast
|
||||
// is most fragile), plus random points in the country's box.
|
||||
function testPoints(f: Feature, perCountry: number, random: () => number): [number, number][] {
|
||||
if (!f.geometry) return [];
|
||||
const g = f.geometry;
|
||||
const rings = ((g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]).flat();
|
||||
const total = rings.reduce((n, ring) => n + ring.length, 0);
|
||||
const step = Math.max(1, Math.floor(total / perCountry));
|
||||
const points: [number, number][] = [];
|
||||
let seen = 0;
|
||||
let minLng = Infinity,
|
||||
minLat = Infinity,
|
||||
maxLng = -Infinity,
|
||||
maxLat = -Infinity;
|
||||
for (const ring of rings) {
|
||||
for (let i = 0; i < ring.length - 1; i++) {
|
||||
const [lng, lat] = ring[i];
|
||||
minLng = Math.min(minLng, lng);
|
||||
maxLng = Math.max(maxLng, lng);
|
||||
minLat = Math.min(minLat, lat);
|
||||
maxLat = Math.max(maxLat, lat);
|
||||
if (seen++ % step !== 0) continue;
|
||||
const midLng = (lng + ring[i + 1][0]) / 2;
|
||||
const midLat = (lat + ring[i + 1][1]) / 2;
|
||||
points.push(
|
||||
[lat, lng],
|
||||
[lat + 1e-7, lng + 1e-7],
|
||||
[lat - 1e-7, lng - 1e-7],
|
||||
[midLat + 1e-7, midLng],
|
||||
[midLat - 1e-7, midLng],
|
||||
);
|
||||
}
|
||||
}
|
||||
for (let k = 0; k < 20; k++) {
|
||||
points.push([minLat + random() * (maxLat - minLat), minLng + random() * (maxLng - minLng)]);
|
||||
}
|
||||
return points;
|
||||
}
|
||||
|
||||
describe('getCountryFromCoords on the packed country index', () => {
|
||||
it('ATLAS-SVC-055: answers every test point the way the Float64 lookup did, in every country', () => {
|
||||
const old = oldIndexes();
|
||||
let seed = 0x2f6b;
|
||||
const random = () => (seed = (seed * 16807) % 2147483647) / 2147483647;
|
||||
let checked = 0;
|
||||
let inCountry = 0;
|
||||
const different: string[] = [];
|
||||
for (const f of features) {
|
||||
// Each point near a border is tested against the big polygons whose boxes cover it
|
||||
// (RU, CA, US, CN, ...), so 100 vertices per country already take ~15s.
|
||||
for (const [lat, lng] of testPoints(f, 100, random)) {
|
||||
const before = oldLookup(old, lat, lng);
|
||||
const after = getCountryFromCoords(lat, lng);
|
||||
checked++;
|
||||
if (before) inCountry++;
|
||||
if (after !== before) different.push(`${f.properties.ISO_A2} ${lat},${lng}: ${before} → ${after}`);
|
||||
}
|
||||
}
|
||||
expect(checked).toBeGreaterThan(100_000);
|
||||
expect(inCountry / checked, 'most points are in a country').toBeGreaterThan(0.9);
|
||||
expect(different.slice(0, 10)).toEqual([]);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -19,11 +19,12 @@ function makeController(svc: Partial<AtlasService> & Partial<AtlasWritesService>
|
||||
}
|
||||
|
||||
function makeRes() {
|
||||
const res = { setHeader: vi.fn(), send: vi.fn(), json: vi.fn(), type: vi.fn() };
|
||||
const res = { setHeader: vi.fn(), send: vi.fn(), sendFile: vi.fn(), json: vi.fn(), type: vi.fn() };
|
||||
res.type.mockReturnValue(res);
|
||||
return res as unknown as Response & {
|
||||
setHeader: ReturnType<typeof vi.fn>;
|
||||
send: ReturnType<typeof vi.fn>;
|
||||
sendFile: ReturnType<typeof vi.fn>;
|
||||
json: ReturnType<typeof vi.fn>;
|
||||
type: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
@@ -89,20 +90,24 @@ describe('AtlasController (parity with the legacy /api/addons/atlas route)', ()
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /countries/geo serves the gzipped admin-0 bundle with Content-Encoding', () => {
|
||||
const gz = Buffer.from('gzipped-bytes');
|
||||
const countryGeoGz = vi.fn().mockReturnValue(gz);
|
||||
it('GET /countries/geo sends the gzipped admin-0 file from disk with Content-Encoding', () => {
|
||||
const countryGeoFile = vi.fn().mockReturnValue('/srv/trek/assets/atlas/admin0.geojson.gz');
|
||||
const res = makeRes();
|
||||
makeController({ countryGeoGz }).countryGeo(res);
|
||||
makeController({ countryGeoFile }).countryGeo(res);
|
||||
expect(res.setHeader).toHaveBeenCalledWith('Content-Type', 'application/json; charset=utf-8');
|
||||
expect(res.setHeader).toHaveBeenCalledWith('Content-Encoding', 'gzip');
|
||||
expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'public, max-age=86400');
|
||||
expect(res.send).toHaveBeenCalledWith(gz);
|
||||
expect(res.sendFile).toHaveBeenCalledWith('admin0.geojson.gz', {
|
||||
root: '/srv/trek/assets/atlas',
|
||||
acceptRanges: false,
|
||||
lastModified: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('GET /countries/geo falls back to an empty FeatureCollection when the bundle is missing', () => {
|
||||
const countryGeoGz = vi.fn().mockReturnValue(null);
|
||||
const countryGeoFile = vi.fn().mockReturnValue(null);
|
||||
const res = makeRes();
|
||||
makeController({ countryGeoGz }).countryGeo(res);
|
||||
makeController({ countryGeoFile }).countryGeo(res);
|
||||
expect(res.json).toHaveBeenCalledWith({ type: 'FeatureCollection', features: [] });
|
||||
expect(res.setHeader).toHaveBeenCalledWith('Cache-Control', 'no-store');
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@ describe('Atlas admin0 country bundle (#1609)', () => {
|
||||
const bundlePath = path.join(__dirname, '..', '..', '..', 'assets', 'atlas', 'admin0.geojson.gz');
|
||||
const features = JSON.parse(zlib.gunzipSync(fs.readFileSync(bundlePath)).toString()).features as {
|
||||
properties: { ISO_A2: string | null; ADM0_A3: string; NAME?: string };
|
||||
geometry: { type: string; coordinates: number[][][] | number[][][][] } | null;
|
||||
}[];
|
||||
|
||||
it('ATLAS-BUNDLE-003 — Kosovo ships with a resolvable ISO_A2', () => {
|
||||
@@ -24,6 +25,28 @@ describe('Atlas admin0 country bundle (#1609)', () => {
|
||||
expect(kosovo.length, 'exactly one Kosovo feature').toBe(1);
|
||||
expect(kosovo[0].properties.ISO_A2).toBe('XK');
|
||||
});
|
||||
|
||||
// The server's country index keeps every coordinate in an Int16Array ×100 (ADMIN0_SCALE in
|
||||
// country-borders.ts) and divides it back on use. A value with a third decimal comes back
|
||||
// changed, and one past ±327.67 wraps around without an error, so a rebuild with more
|
||||
// decimals (ADM0_DECIMALS) must fail here, not move borders.
|
||||
it('ATLAS-BUNDLE-010 — every admin0 coordinate round-trips through Int16 ×100 unchanged', () => {
|
||||
const int16 = new Int16Array(1);
|
||||
let values = 0;
|
||||
const lossy: string[] = [];
|
||||
for (const f of features) {
|
||||
const g = f.geometry;
|
||||
if (!g) continue;
|
||||
const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][];
|
||||
for (const v of parts.flat(3)) {
|
||||
values++;
|
||||
int16[0] = Math.round(v * 100);
|
||||
if (int16[0] / 100 !== v) lossy.push(`${f.properties.ADM0_A3}: ${v} comes back as ${int16[0] / 100}`);
|
||||
}
|
||||
}
|
||||
expect(values / 2, 'coordinate pairs checked').toBeGreaterThan(2_000_000);
|
||||
expect(lossy.slice(0, 5)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Atlas admin1 region bundle (#1217)', () => {
|
||||
|
||||
Reference in New Issue
Block a user