diff --git a/server/scripts/build-atlas-geo.mjs b/server/scripts/build-atlas-geo.mjs index 9560c27a7..68ac6dd40 100644 --- a/server/scripts/build-atlas-geo.mjs +++ b/server/scripts/build-atlas-geo.mjs @@ -373,7 +373,7 @@ async function pool(items, worker) { // visible effect at the atlas' zoom range (3-10). ADM0 fills are viewed zoomed out, so // they tolerate a coarser grid than ADM1 region borders. const ADM0_DECIMALS = 2; // ~1.1 km -const ADM1_DECIMALS = 3; // ~110 m +const ADM1_DECIMALS = 3; // ~110 m; the server's region cache packs ×1000 (REGION_SCALE in atlas-geo.ts), raise both function quantizeRing(ring, decimals) { const m = 10 ** decimals; diff --git a/server/src/db/repositories/PlaceRegions.repository.ts b/server/src/db/repositories/PlaceRegions.repository.ts index b6545911a..2cfa6e34e 100644 --- a/server/src/db/repositories/PlaceRegions.repository.ts +++ b/server/src/db/repositories/PlaceRegions.repository.ts @@ -101,7 +101,9 @@ export class PlaceRegionsRepository extends TrekRepository { /** * AT49 (`repairStaleRegionCache`'s read, #2527) — `SELECT pr.place_id AS id, pr.country_code, * pr.region_code, p.lat, p.lng, p.address FROM place_regions pr JOIN places p - * ON p.id = pr.place_id ORDER BY pr.place_id`. + * ON p.id = pr.place_id ORDER BY pr.country_code, pr.place_id`. By country first, so the + * pass resolves one country's places after another and its region cache, which keeps only + * so many countries, parses each country once. */ async listWithPlaceLocation(): Promise< { @@ -117,6 +119,7 @@ export class PlaceRegionsRepository extends TrekRepository { .selectFrom('place_regions as pr') .innerJoin('places as p', 'p.id', 'pr.place_id') .select(['pr.place_id as id', 'pr.country_code', 'pr.region_code', 'p.lat', 'p.lng', 'p.address']) + .orderBy('pr.country_code') .orderBy('pr.place_id') .execute(); } diff --git a/server/src/nest/atlas/atlas-geo.ts b/server/src/nest/atlas/atlas-geo.ts index ee84f9f6d..59566865a 100644 --- a/server/src/nest/atlas/atlas-geo.ts +++ b/server/src/nest/atlas/atlas-geo.ts @@ -8,9 +8,8 @@ import { getCountryFromCoords, isPointInCountryBox, loadAdmin0Gz, - pointInGeometry, - type Box, - type CompactGeom, + pointInPackedGeometry, + type PackedGeom, } from '../../utils/country-borders'; import { cacheKeyFor, getCached, nominatimFetch, setCached } from '../geo/nominatim.client'; @@ -389,25 +388,38 @@ export async function reverseGeocodeCountry(lat: number, lng: number): Promise maxLat) maxLat = lat; } - boxes.push([minLng, minLat, maxLng, maxLat]); + boxes.set([minLng, minLat, maxLng, maxLat].map((v) => Math.round(v * REGION_SCALE)), 4 * k); } - return { geom: { rings, polyRingCounts }, boxes }; + ringStarts[ri] = p; + return { geom: { coords, ringStarts, polyRingCounts }, boxes }; } /** @@ -518,32 +531,57 @@ const regionCache = new Map(); // stored region_code/region_name are then guaranteed to equal a bundle feature. // // The admin1 regions ship as one file per country (never parsed whole, #1576), so a -// country's region features are read, parsed and flattened to CompactGeom on first use -// and cached — only visited countries ever pay the parse. -type RegionFeature = { code: string; name: string; nameEn: string; geom: CompactGeom; boxes: Box[] }; -const regionFeatureCache = new Map(); +// country's region features are read, parsed and packed on first use and cached — only +// visited countries ever pay the parse. +type RegionFeature = { code: string; name: string; nameEn: string; geom: PackedGeom; boxes: Int32Array }; -async function getRegionFeatures(countryCode: string): Promise { +// Countries least recently used first, kept while their typed arrays fit REGION_CACHE_BYTES +// (every country packed keeps ~32 MB, CA alone ~5 MB). A country that falls out is parsed again +// on its next lookup, which allocates ~90 MB for a while for CA, so the budget holds the +// countries one install's places are in, and the repair pass reads its rows by country. +const REGION_CACHE_BYTES = 16 * 1024 * 1024; +const regionFeatureCache = new Map(); +let regionCacheBytes = 0; + +// Synchronous on purpose: a lookup that misses reads, parses and caches its country before +// any other lookup runs, so lookups that miss the same country at once parse it only once. +function getRegionFeatures(countryCode: string): RegionFeature[] { const cc = countryCode.toUpperCase(); - const cached = regionFeatureCache.get(cc); - if (cached) return cached; - const json = readAdmin1Json(cc); - if (!json) { - regionFeatureCache.set(cc, []); - return []; + const hit = regionFeatureCache.get(cc); + if (hit) { + regionFeatureCache.delete(cc); + regionFeatureCache.set(cc, hit); + return hit.features; } + const features = parseRegionFeatures(cc); + let bytes = 0; + for (const { geom, boxes } of features) { + bytes += geom.coords.byteLength + geom.ringStarts.byteLength + geom.polyRingCounts.byteLength + boxes.byteLength; + } + regionFeatureCache.set(cc, { features, bytes }); + regionCacheBytes += bytes; + for (const [code, entry] of regionFeatureCache) { + if (regionCacheBytes <= REGION_CACHE_BYTES || code === cc) break; + regionFeatureCache.delete(code); + regionCacheBytes -= entry.bytes; + } + return features; +} + +function parseRegionFeatures(cc: string): RegionFeature[] { + const json = readAdmin1Json(cc); + if (!json) return []; let features: { properties?: Record; geometry?: { type: string; coordinates: unknown } }[]; try { features = JSON.parse(json.toString('utf8')).features ?? []; } catch { - regionFeatureCache.set(cc, []); return []; } const out: RegionFeature[] = []; for (const f of features) { const code = f.properties?.iso_3166_2; if (!code || !f.geometry) continue; - const { geom, boxes } = compactGeomFromGeometry(f.geometry); + const { geom, boxes } = packRegionGeometry(f.geometry); out.push({ code, name: f.properties?.name || code, @@ -552,7 +590,6 @@ async function getRegionFeatures(countryCode: string): Promise boxes, }); } - regionFeatureCache.set(cc, out); return out; } @@ -562,11 +599,15 @@ async function getRegionFeatures(countryCode: string): Promise // no admin1 coverage in the bundle or the point falls outside every polygon (simplification // gaps at coastlines, etc.) — callers should fall back to reverse geocoding. export async function getRegionFromCoords(countryCode: string, lat: number, lng: number): Promise { - const features = await getRegionFeatures(countryCode); + const features = getRegionFeatures(countryCode); if (features.length === 0) return null; const candidates: { f: RegionFeature; area: number }[] = []; for (const f of features) { - for (const [minLng, minLat, maxLng, maxLat] of f.boxes) { + for (let i = 0; i < f.boxes.length; i += 4) { + const minLng = f.boxes[i] / REGION_SCALE; + const minLat = f.boxes[i + 1] / REGION_SCALE; + const maxLng = f.boxes[i + 2] / REGION_SCALE; + const maxLat = f.boxes[i + 3] / REGION_SCALE; if (lat >= minLat && lat <= maxLat && lng >= minLng && lng <= maxLng) { candidates.push({ f, area: (maxLng - minLng) * (maxLat - minLat) }); break; @@ -575,7 +616,7 @@ export async function getRegionFromCoords(countryCode: string, lat: number, lng: } candidates.sort((a, b) => a.area - b.area); for (const { f } of candidates) { - if (pointInGeometry(lng, lat, f.geom)) { + if (pointInPackedGeometry(lng, lat, f.geom, REGION_SCALE)) { return { country_code: countryCode.toUpperCase(), region_code: f.code, region_name: f.nameEn || f.name }; } } diff --git a/server/src/utils/country-borders.ts b/server/src/utils/country-borders.ts index 8fb5b738f..ebe91a0df 100644 --- a/server/src/utils/country-borders.ts +++ b/server/src/utils/country-borders.ts @@ -181,7 +181,7 @@ function pointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean // True when (lng,lat) falls inside a compact Polygon/MultiPolygon, honouring holes. // `rings` is every ring flattened; `polyRingCounts[k]` is how many rings polygon k owns // (its first ring is the outer boundary, the rest are holes). -export function pointInGeometry(lng: number, lat: number, geom: CompactGeom): boolean { +function pointInGeometry(lng: number, lat: number, geom: CompactGeom): boolean { let ri = 0; for (const rc of geom.polyRingCounts) { if (pointInFlatRing(lng, lat, geom.rings[ri])) { @@ -199,10 +199,62 @@ export function pointInGeometry(lng: number, lat: number, geom: CompactGeom): bo return false; } +// The ray-cast above on ring pairs [from, to) of packed coordinates. Each one is divided back +// by the scale before use: for coordinates with no more decimals than the scale holds that +// gives back the exact double the bundle had, so the answer is the one the Float64 ring gives. +function pointInPackedRing( + lng: number, + lat: number, + coords: Int32Array, + scale: number, + from: number, + to: number, +): boolean { + let inside = false; + let xj = coords[2 * (to - 1)] / scale; + let yj = coords[2 * (to - 1) + 1] / scale; + for (let i = from; i < to; i++) { + const xi = coords[2 * i] / scale; + const yi = coords[2 * i + 1] / scale; + if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) { + inside = !inside; + } + xj = xi; + yj = yi; + } + return inside; +} + +// pointInGeometry for a PackedGeom. +export function pointInPackedGeometry(lng: number, lat: number, geom: PackedGeom, scale: number): boolean { + const { coords, ringStarts, polyRingCounts } = geom; + let ri = 0; + for (const rc of polyRingCounts) { + if (pointInPackedRing(lng, lat, coords, scale, ringStarts[ri], ringStarts[ri + 1])) { + let inHole = false; + for (let h = 1; h < rc; h++) { + if (pointInPackedRing(lng, lat, coords, scale, ringStarts[ri + h], ringStarts[ri + h + 1])) { + inHole = true; + break; + } + } + if (!inHole) return true; + } + ri += rc; + } + return false; +} + // Compact polygon geometry: rings flattened into Float64Arrays, grouped into polygons by // polyRingCounts. Replaces the retained parsed GeoJSON geometry. -export type CompactGeom = { rings: Float64Array[]; polyRingCounts: number[] }; -export type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat] +type CompactGeom = { rings: Float64Array[]; polyRingCounts: number[] }; + +// Packed polygon geometry: every ring's [lng,lat] pairs back to back in one Int32Array, each +// value times a scale. Ring r is pairs ringStarts[r] up to ringStarts[r + 1]; polyRingCounts +// groups the rings into polygons as in CompactGeom. Three typed arrays per geometry instead +// of one per ring, at half the bytes per point. +export type PackedGeom = { coords: Int32Array; ringStarts: Uint32Array; polyRingCounts: Uint32Array }; +type Box = [number, number, number, number]; // [minLng, minLat, maxLng, maxLat] // ISO_A2 → compact admin0 geometry + bounding boxes, derived from the bundled admin0 // borders on first use. The parsed FeatureCollection is dropped after this runs; only the diff --git a/server/tests/unit/nest/atlas-region-cache.test.ts b/server/tests/unit/nest/atlas-region-cache.test.ts new file mode 100644 index 000000000..1806da7cf --- /dev/null +++ b/server/tests/unit/nest/atlas-region-cache.test.ts @@ -0,0 +1,218 @@ +import { admin1AssetPath } from '../../../src/utils/country-borders'; + +import fs from 'fs'; +import path from 'path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import zlib from 'zlib'; + +// The packed admin-1 region cache behind getRegionFromCoords: Int32 ×1000 coordinates, a +// byte budget, least recently used countries out first. Each test imports atlas-geo afresh, +// so it starts from an empty cache. + +const freshGeo = async () => { + vi.resetModules(); + return import('../../../src/nest/atlas/atlas-geo'); +}; + +const regionDir = path.dirname(admin1AssetPath('XX')); +const countryCodes = fs + .readdirSync(regionDir) + .filter((name) => name.endsWith('.geojson.gz')) + .map((name) => name.slice(0, 2)) + .sort(); + +type Geometry = { type: string; coordinates: unknown }; +type Feature = { properties: { iso_3166_2?: string; name?: string; name_en?: string }; geometry: Geometry | null }; +const readFeatures = (cc: string) => + (JSON.parse(zlib.gunzipSync(fs.readFileSync(admin1AssetPath(cc))).toString('utf8')) as { features: Feature[] }) + .features; + +function spyOnReads() { + const read = vi.spyOn(fs, 'readFileSync'); + return (cc: string) => + read.mock.calls.filter(([file]) => String(file).endsWith(`${path.sep}${cc}.geojson.gz`)).length; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// ── The lookup as it was before the packing, kept as the reference ───────── +// Float64 rings, one box tuple per part, the same ray-cast: what getRegionFromCoords did +// before this cache, copied from atlas-geo.ts and country-borders.ts. + +type OldRegion = { label: string; rings: Float64Array[]; polyRingCounts: number[]; boxes: number[][] }; + +function oldPack(f: Feature): OldRegion | null { + const code = f.properties.iso_3166_2; + if (!code || !f.geometry) return null; + const g = f.geometry; + const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]; + const rings: Float64Array[] = []; + const polyRingCounts: number[] = []; + const boxes: number[][] = []; + for (const part of parts) { + polyRingCounts.push(part.length); + for (const ring of part) rings.push(Float64Array.from(ring.flat())); + const lngs = part[0].map(([lng]) => lng); + const lats = part[0].map(([, lat]) => lat); + boxes.push([Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)]); + } + const name = f.properties.name_en || f.properties.name || code; + return { label: `${code} ${name}`, rings, polyRingCounts, boxes }; +} + +function oldPointInFlatRing(lng: number, lat: number, ring: Float64Array): boolean { + let inside = false; + const n = ring.length / 2; + for (let i = 0, j = n - 1; i < n; j = i++) { + const xi = ring[2 * i], + yi = ring[2 * i + 1]; + const xj = ring[2 * j], + yj = ring[2 * j + 1]; + if (yi > lat !== yj > lat && lng < ((xj - xi) * (lat - yi)) / (yj - yi) + xi) { + inside = !inside; + } + } + return inside; +} + +function oldPointInGeometry(lng: number, lat: number, r: OldRegion): boolean { + let ri = 0; + for (const rc of r.polyRingCounts) { + if (oldPointInFlatRing(lng, lat, r.rings[ri])) { + let inHole = false; + for (let h = 1; h < rc; h++) { + if (oldPointInFlatRing(lng, lat, r.rings[ri + h])) { + inHole = true; + break; + } + } + if (!inHole) return true; + } + ri += rc; + } + return false; +} + +function oldLookup(regions: OldRegion[], lat: number, lng: number): string | null { + const candidates: { r: OldRegion; area: number }[] = []; + for (const r of regions) { + for (const [minLng, minLat, maxLng, maxLat] of r.boxes) { + if (lat >= minLat && lat <= maxLat && lng >= minLng && lng <= maxLng) { + candidates.push({ r, area: (maxLng - minLng) * (maxLat - minLat) }); + break; + } + } + } + candidates.sort((a, b) => a.area - b.area); + return candidates.find(({ r }) => oldPointInGeometry(lng, lat, r))?.r.label ?? null; +} + +// Deterministic points per country: about `perCountry` vertices spread over all its rings, +// each exactly, 1e-7° to either side of it and of the next edge's middle (where a ray-cast +// is most fragile), plus random points in each region's box. +function testPoints(features: Feature[], perCountry: number): [number, number][] { + let seed = 0x2f6b; + const random = () => (seed = (seed * 16807) % 2147483647) / 2147483647; + const regions = features.flatMap((f) => { + if (!f.geometry) return []; + const g = f.geometry; + return [((g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]).flat()]; + }); + const total = regions.flat().reduce((n, ring) => n + ring.length, 0); + const step = Math.max(1, Math.floor(total / perCountry)); + const points: [number, number][] = []; + let seen = 0; + for (const rings of regions) { + let minLng = Infinity, + minLat = Infinity, + maxLng = -Infinity, + maxLat = -Infinity; + for (const ring of rings) { + for (let i = 0; i < ring.length - 1; i++) { + const [lng, lat] = ring[i]; + minLng = Math.min(minLng, lng); + maxLng = Math.max(maxLng, lng); + minLat = Math.min(minLat, lat); + maxLat = Math.max(maxLat, lat); + if (seen++ % step !== 0) continue; + const midLng = (lng + ring[i + 1][0]) / 2; + const midLat = (lat + ring[i + 1][1]) / 2; + points.push( + [lat, lng], + [lat + 1e-7, lng + 1e-7], + [lat - 1e-7, lng - 1e-7], + [midLat + 1e-7, midLng], + [midLat - 1e-7, midLng], + ); + } + } + for (let k = 0; k < 5; k++) { + points.push([minLat + random() * (maxLat - minLat), minLng + random() * (maxLng - minLng)]); + } + } + return points; +} + +// getRegionFromCoords does its work before it returns its promise, so lookups started one +// after another reach the cache in that order even when they are awaited together. +describe('getRegionFromCoords on the packed region cache', () => { + it('ATLAS-SVC-051: answers every test point the way the Float64 lookup did, in every country', async () => { + const geo = await freshGeo(); + const checks: { point: string; before: string | null; after: Promise }[] = []; + for (const cc of countryCodes) { + const features = readFeatures(cc); + const regions = features.map(oldPack).filter((r): r is OldRegion => r !== null); + for (const [lat, lng] of testPoints(features, 400)) { + checks.push({ + point: `${cc} ${lat},${lng}`, + before: oldLookup(regions, lat, lng), + after: geo.getRegionFromCoords(cc, lat, lng).then((r) => (r ? `${r.region_code} ${r.region_name}` : null)), + }); + } + } + const answers = await Promise.all(checks.map((c) => c.after)); + const different = checks.flatMap((c, i) => + answers[i] === c.before ? [] : [`${c.point}: ${c.before} → ${answers[i]}`], + ); + expect(checks.length).toBeGreaterThan(300_000); + expect(checks.filter((c) => c.before).length / checks.length, 'most points are in a region').toBeGreaterThan(0.5); + expect(different.slice(0, 10)).toEqual([]); + }, 120_000); + + it('ATLAS-SVC-052: two lookups that miss the same country at once read and parse it once', async () => { + const geo = await freshGeo(); + const readsOf = spyOnReads(); + const [ottawa, vancouver] = await Promise.all([ + geo.getRegionFromCoords('CA', 45.42, -75.69), + geo.getRegionFromCoords('ca', 49.28, -123.12), + ]); + expect(ottawa?.region_code).toBe('CA-ON'); + expect(vancouver?.region_code).toBe('CA-BC'); + expect(readsOf('CA')).toBe(1); + }); + + it('ATLAS-SVC-053: keeps a country in use while every other one passes through', async () => { + const geo = await freshGeo(); + const readsOf = spyOnReads(); + const answers = await Promise.all( + countryCodes.flatMap((cc) => [geo.getRegionFromCoords(cc, 0, 0), geo.getRegionFromCoords('CA', 45.42, -75.69)]), + ); + expect(new Set(answers.filter((_, i) => i % 2 === 1).map((r) => r?.region_code))).toEqual(new Set(['CA-ON'])); + expect(readsOf('CA')).toBe(1); + // Not everything stayed: the countries read first had to go to make room. + await geo.getRegionFromCoords(countryCodes[0], 0, 0); + expect(readsOf(countryCodes[0])).toBe(2); + }, 60_000); + + it('ATLAS-SVC-054: drops the least recently used country past the budget and reads it again', async () => { + const geo = await freshGeo(); + const readsOf = spyOnReads(); + await geo.getRegionFromCoords('CA', 45.42, -75.69); + await Promise.all(countryCodes.filter((cc) => cc !== 'CA').map((cc) => geo.getRegionFromCoords(cc, 0, 0))); + expect(readsOf('CA')).toBe(1); + expect((await geo.getRegionFromCoords('CA', 45.42, -75.69))?.region_code).toBe('CA-ON'); + expect(readsOf('CA')).toBe(2); + }, 60_000); +}); diff --git a/server/tests/unit/nest/atlasBundle.test.ts b/server/tests/unit/nest/atlasBundle.test.ts index ab28e47e6..0746e7a31 100644 --- a/server/tests/unit/nest/atlasBundle.test.ts +++ b/server/tests/unit/nest/atlasBundle.test.ts @@ -64,6 +64,26 @@ describe('Atlas admin1 region bundle (#1217)', () => { expect(features.map(key).sort()).toEqual(bundle.map(key).sort()); }); + // The server's region cache keeps every coordinate as an Int32 ×1000 (REGION_SCALE in + // atlas-geo.ts) and divides it back on use. That is exact only while the bundle has no more + // than three decimals (ADM1_DECIMALS); a rebuild with more must raise the scale with it. + it('ATLAS-BUNDLE-009 — every admin1 coordinate round-trips through Int32 ×1000 unchanged', () => { + type Geometry = { type: string; coordinates: number[][][] | number[][][][] }; + let values = 0; + const lossy: string[] = []; + for (const f of features as unknown as { properties: { iso_3166_2: string }; geometry: Geometry }[]) { + const g = f.geometry; + const parts = (g.type === 'Polygon' ? [g.coordinates] : g.coordinates) as number[][][][]; + for (const v of parts.flat(3)) { + values++; + const packed = Math.round(v * 1000); + if (packed / 1000 !== v || Math.abs(packed) > 2 ** 31 - 1) lossy.push(`${f.properties.iso_3166_2}: ${v}`); + } + } + expect(values / 2, 'coordinate pairs checked').toBeGreaterThan(3_000_000); + expect(lossy.slice(0, 5)).toEqual([]); + }); + it('ATLAS-BUNDLE-001 — previously-broken countries now have distinct region codes', () => { for (const a2 of ['ES', 'CN', 'CL', 'OM']) { const f = regions(a2);