diff --git a/NOTICE.md b/NOTICE.md index 8a0a6ed79..3b7eb3b02 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -20,7 +20,23 @@ geoBoundaries is licensed under **CC BY 4.0** The bundled files are simplified (coordinate-quantized) and re-tagged with the property names TREK consumes. Country borders (`admin0`) derive from the geoBoundaries CGAZ composite; sub-national regions (`admin1`) derive from the per-country open -(gbOpen) release. +(gbOpen) release, except for Ireland (below). + +## Tailte Éireann — counties of Ireland + +Ireland's regions in the same files (`admin1/IE.geojson.gz`, and its features in +`admin1.geojson.gz`) are the 26 counties from Tailte Éireann's *Counties - National +Statutory Boundaries - 2019 - Generalised 20m*, because geoBoundaries has no county +layer for Ireland. + +Contains Tailte Éireann data, © Tailte Éireann, licensed under **CC BY 4.0** +(https://creativecommons.org/licenses/by/4.0/). Source: +https://data-osi.opendata.arcgis.com/ + +The outlines are simplified, coordinate-quantized and re-tagged with ISO 3166-2:IE +codes by `server/scripts/build-atlas-geo.mjs`. Tailte Éireann publishes the +generalised boundaries for reference only; they may not match the statutory +boundaries exactly. ## OpenStreetMap — geocoding, place details & descriptions diff --git a/README.md b/README.md index 663ba1389..22b711357 100644 --- a/README.md +++ b/README.md @@ -439,8 +439,9 @@ Every variable, its default and what it does: see The Atlas map's country and sub-national (province/county) boundaries come from [**geoBoundaries**](https://www.geoboundaries.org/) (Runfola et al., 2020), licensed -[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). See [NOTICE.md](NOTICE.md) -for full third-party attributions. +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/); Ireland's counties come from +[**Tailte Éireann**](https://data-osi.opendata.arcgis.com/), also CC BY 4.0. See +[NOTICE.md](NOTICE.md) for full third-party attributions.
diff --git a/server/assets/atlas/admin1.geojson.gz b/server/assets/atlas/admin1.geojson.gz index 6c833aeae..00552eb65 100644 Binary files a/server/assets/atlas/admin1.geojson.gz and b/server/assets/atlas/admin1.geojson.gz differ diff --git a/server/assets/atlas/admin1/IE.geojson.gz b/server/assets/atlas/admin1/IE.geojson.gz index 966bc8936..9ebbddf2e 100644 Binary files a/server/assets/atlas/admin1/IE.geojson.gz and b/server/assets/atlas/admin1/IE.geojson.gz differ diff --git a/server/scripts/build-atlas-geo.mjs b/server/scripts/build-atlas-geo.mjs index 01b99a755..e1f04144b 100644 --- a/server/scripts/build-atlas-geo.mjs +++ b/server/scripts/build-atlas-geo.mjs @@ -14,7 +14,13 @@ // FeatureCollections that the server serves at runtime (no network at boot): one for // every country's borders, and one per country for its regions. // +// A few countries take their regions from another source instead (REGION_SOURCES below: +// Ireland's counties from Tailte Éireann). +// // geoBoundaries: CC BY 4.0 — https://www.geoboundaries.org/ (attribution required). +// Tailte Éireann: CC BY 4.0 — https://data-osi.opendata.arcgis.com/ (attribution required). +import { simplifySharedBorders } from './lib/shared-border-simplify.mjs'; + import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -317,13 +323,86 @@ const A3_TO_A2 = { */ const ADM_LEVEL = { GBR: 'ADM2' }; +/** + * The 26 counties of Ireland, keyed by the COUNTY field of Tailte Éireann's layer, with + * their ISO 3166-2:IE codes (the source carries none). + */ +const IE_COUNTIES = { + CARLOW: ['IE-CW', 'Carlow'], + CAVAN: ['IE-CN', 'Cavan'], + CLARE: ['IE-CE', 'Clare'], + CORK: ['IE-CO', 'Cork'], + DONEGAL: ['IE-DL', 'Donegal'], + DUBLIN: ['IE-D', 'Dublin'], + GALWAY: ['IE-G', 'Galway'], + KERRY: ['IE-KY', 'Kerry'], + KILDARE: ['IE-KE', 'Kildare'], + KILKENNY: ['IE-KK', 'Kilkenny'], + LAOIS: ['IE-LS', 'Laois'], + LEITRIM: ['IE-LM', 'Leitrim'], + LIMERICK: ['IE-LK', 'Limerick'], + LONGFORD: ['IE-LD', 'Longford'], + LOUTH: ['IE-LH', 'Louth'], + MAYO: ['IE-MO', 'Mayo'], + MEATH: ['IE-MH', 'Meath'], + MONAGHAN: ['IE-MN', 'Monaghan'], + OFFALY: ['IE-OY', 'Offaly'], + ROSCOMMON: ['IE-RN', 'Roscommon'], + SLIGO: ['IE-SO', 'Sligo'], + TIPPERARY: ['IE-TA', 'Tipperary'], + WATERFORD: ['IE-WD', 'Waterford'], + WESTMEATH: ['IE-WH', 'Westmeath'], + WEXFORD: ['IE-WX', 'Wexford'], + WICKLOW: ['IE-WW', 'Wicklow'], +}; + +/** + * Countries whose regions come from somewhere other than geoBoundaries, because no + * geoBoundaries level is the one people think in. + * + * IRL: ADM1 is the four provinces and ADM2 the 166 local electoral areas; there is no + * county layer at all (#2922). Tailte Éireann (CC BY 4.0) publishes the 26 counties, + * with Dublin, Cork, Galway and Limerick each one county (city and county together), + * which is what ISO 3166-2:IE lists. + * + * Each entry names the download, the file it is cached under, how many features it must + * yield, the tolerance its full-detail outlines are simplified with (see + * lib/shared-border-simplify.mjs) and how one feature's properties become the + * geoBoundaries `shapeName`/`shapeISO` that normalizeAdm1 reads, so everything after the + * download treats it like any other country. A feature it cannot place fails the build. + */ +const REGION_SOURCES = { + IRL: { + url: + 'https://services-eu1.arcgis.com/FH5XCsx8rYXqnjF5/arcgis/rest/services/' + + 'Counties___OSi_National_Statutory_Boundaries___Generalised_20m/FeatureServer/0/query' + + '?where=1%3D1&outFields=COUNTY&outSR=4326&f=geojson', + cacheFile: 'tailte-eireann-counties-2019-generalised-20m.geojson', + count: 26, + tolerance: 0.002, + toShape: ({ COUNTY }) => { + const county = IE_COUNTIES[COUNTY]; + return county && { shapeISO: county[0], shapeName: county[1] }; + }, + }, +}; + const COUNTRIES = Object.keys(A3_TO_A2); // every ISO alpha-3 code (ADM1 fetch list) +// ATLAS_ONLY=IRL[,GBR…] rebuilds just those countries' regions: their admin1/ files +// are rewritten and their features swapped into admin1.geojson.gz in place, while admin0 +// and every other country stay as they are. Saves fetching the whole world for one fix. +const ONLY = (process.env.ATLAS_ONLY || '') + .split(',') + .map((s) => s.trim().toUpperCase()) + .filter(Boolean); +for (const a3 of ONLY) if (!A3_TO_A2[a3]) throw new Error(`ATLAS_ONLY: unknown alpha-3 code ${a3}`); + // Cache raw downloads so re-runs (e.g. to tune simplification) don't re-fetch ~360 files. const CACHE_DIR = path.join(__dirname, '..', '.atlas-geo-cache', GB_REF); -async function fetchGeo(url) { - const cacheFile = path.join(CACHE_DIR, url.split('/').slice(-1)[0]); +async function fetchGeo(url, cacheName = url.split('/').slice(-1)[0]) { + const cacheFile = path.join(CACHE_DIR, cacheName); if (fs.existsSync(cacheFile)) { const cached = fs.readFileSync(cacheFile, 'utf8'); return cached === '' ? null : JSON.parse(cached); @@ -422,6 +501,30 @@ function normalizeAdm0Feature(f) { }; } +// A country's raw region layer: from its REGION_SOURCES entry when it has one, else the +// geoBoundaries level ADM_LEVEL names (ADM1 by default). +async function fetchRegions(a3) { + const source = REGION_SOURCES[a3]; + if (!source) return fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1')); + + const geo = await fetchGeo(source.url, source.cacheFile); + if (!geo?.features) throw new Error(`${a3}: region source could not be downloaded`); + const features = geo.features.map((f) => { + const shape = source.toShape(f.properties || {}); + if (!shape) throw new Error(`${a3}: region source has a feature it cannot place: ${JSON.stringify(f.properties)}`); + return { type: 'Feature', properties: shape, geometry: quantizeGeometry(f.geometry, ADM1_DECIMALS) }; + }); + const codes = new Set(features.map((f) => f.properties.shapeISO)); + if (features.length !== source.count || codes.size !== source.count) { + throw new Error( + `${a3}: expected ${source.count} distinct regions, the source has ${features.length} (${codes.size} distinct)`, + ); + } + // By code, so the output does not depend on the order the service happens to answer in. + features.sort((a, b) => (a.properties.shapeISO < b.properties.shapeISO ? -1 : 1)); + return { features: simplifySharedBorders(features, source.tolerance) }; +} + function normalizeAdm1(geo, a3, countryName) { if (!geo?.features) return []; const a2 = A3_TO_A2[a3] || null; @@ -477,7 +580,46 @@ function normalizeAdm1(geo, a3, countryName) { .filter(Boolean); } +const write = (name, features) => { + const fc = { type: 'FeatureCollection', features }; + const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 }); + const file = path.join(OUT_DIR, `${name}.geojson.gz`); + fs.writeFileSync(file, gz); + console.log( + `[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`, + ); +}; + +const readShipped = (name) => + JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(OUT_DIR, `${name}.geojson.gz`))).toString('utf8')).features; + +// The ATLAS_ONLY path. A full build writes the countries in COUNTRIES order, each one's +// features in a single run, so a rebuilt country goes back where its run was, or where +// it would have been, and admin1.geojson.gz ends up as a full build would leave it. +async function rebuildOnly(countries) { + console.log(`[atlas-geo] rebuilding the regions of ${countries.join(', ')} only`); + fs.mkdirSync(CACHE_DIR, { recursive: true }); + const a3ToName = Object.fromEntries(readShipped('admin0').map((f) => [f.properties.ADM0_A3, f.properties.NAME])); + const order = new Map(COUNTRIES.map((a3, i) => [A3_TO_A2[a3], i])); + let bundle = readShipped('admin1'); + + for (const a3 of countries) { + const a2 = A3_TO_A2[a3]; + const features = normalizeAdm1(await fetchRegions(a3), a3, a3ToName[a3] || a3); + if (!features.length) throw new Error(`${a3}: no regions to write`); + const first = bundle.findIndex((f) => f.properties.iso_a2 === a2); + const at = first !== -1 ? first : bundle.findIndex((f) => order.get(f.properties.iso_a2) > order.get(a2)); + const rest = bundle.filter((f) => f.properties.iso_a2 !== a2); + const cut = at === -1 ? rest.length : at; + bundle = [...rest.slice(0, cut), ...features, ...rest.slice(cut)]; + write(`admin1/${a2}`, features); + } + write('admin1', bundle); +} + async function main() { + if (ONLY.length) return rebuildOnly(ONLY); + console.log(`[atlas-geo] geoBoundaries ref ${GB_REF}; ${COUNTRIES.length} countries`); fs.mkdirSync(OUT_DIR, { recursive: true }); fs.mkdirSync(CACHE_DIR, { recursive: true }); @@ -496,9 +638,10 @@ async function main() { } } - // ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`). + // ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`), + // or the country's own source (REGION_SOURCES). console.log('[atlas-geo] downloading ADM1 (regions)…'); - const adm1Raw = await pool(COUNTRIES, (a3) => fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1'))); + const adm1Raw = await pool(COUNTRIES, fetchRegions); const adm1Features = []; let withCodes = 0; COUNTRIES.forEach((a3, idx) => { @@ -507,16 +650,6 @@ async function main() { adm1Features.push(...feats); }); - const write = (name, features) => { - const fc = { type: 'FeatureCollection', features }; - const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 }); - const file = path.join(OUT_DIR, `${name}.geojson.gz`); - fs.writeFileSync(file, gz); - console.log( - `[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`, - ); - }; - write('admin0', adm0Features); // Read only by two shipped migrations (Migration20200101021500/…031500), whose bodies // never change; the server itself reads the per-country files below. diff --git a/server/scripts/lib/shared-border-simplify.mjs b/server/scripts/lib/shared-border-simplify.mjs new file mode 100644 index 000000000..607ca09f0 --- /dev/null +++ b/server/scripts/lib/shared-border-simplify.mjs @@ -0,0 +1,159 @@ +/* + * Simplify the polygons of one country's regions without opening gaps between them. + * + * geoBoundaries ships a "_simplified" file per country, so the Atlas builder normally + * never simplifies anything itself. A region source that only offers full-detail + * outlines (Tailte Éireann's counties, #2922) needs it done here, and simplifying each + * polygon on its own would not do: two neighbours would each drop different points of + * the border they share, leaving slivers that belong to neither county, and a point in + * a sliver resolves to no region at all. + * + * So the rings are cut into arcs the way TopoJSON does it. A vertex is a junction when + * it does not have the same two neighbours everywhere it appears: where a shared border + * starts or ends, or where three regions meet. Between two junctions both sides hold + * the same run of points, and each arc is simplified (Douglas-Peucker) in one canonical + * direction, so both sides get the same points back. A ring with no junction at all (an + * island, or an enclave shared whole with its surroundings) is rotated to a canonical + * start first, for the same reason. + * + * Expects rings that are already quantized (identical borders carry identical + * coordinates) and closed. A ring left with fewer than three distinct points is dropped, + * a polygon whose outer ring is dropped goes with its holes, and a geometry with nothing + * left becomes null. + */ + +const key = (pt) => `${pt[0]},${pt[1]}`; + +// Squared distance from p to the segment a-b. +function segmentDistance2(p, a, b) { + const dx = b[0] - a[0]; + const dy = b[1] - a[1]; + const len2 = dx * dx + dy * dy; + let t = len2 === 0 ? 0 : ((p[0] - a[0]) * dx + (p[1] - a[1]) * dy) / len2; + t = Math.max(0, Math.min(1, t)); + const x = a[0] + t * dx - p[0]; + const y = a[1] + t * dy - p[1]; + return x * x + y * y; +} + +/** Douglas-Peucker over an open polyline; both ends are always kept. */ +export function douglasPeucker(points, tolerance) { + if (points.length <= 2) return points.slice(); + const keep = new Uint8Array(points.length); + keep[0] = 1; + keep[points.length - 1] = 1; + const tol2 = tolerance * tolerance; + const stack = [[0, points.length - 1]]; + while (stack.length) { + const [first, last] = stack.pop(); + let maxDist = -1; + let index = -1; + for (let i = first + 1; i < last; i++) { + const d = segmentDistance2(points[i], points[first], points[last]); + if (d > maxDist) { + maxDist = d; + index = i; + } + } + if (index !== -1 && maxDist > tol2) { + keep[index] = 1; + stack.push([first, index], [index, last]); + } + } + return points.filter((_, i) => keep[i]); +} + +// An arc simplified in its canonical direction, handed back in the direction it came in. +function simplifyArc(arc, tolerance) { + const head = key(arc[0]); + const tail = key(arc[arc.length - 1]); + const reversed = head === tail ? key(arc[1]) > key(arc[arc.length - 2]) : head > tail; + const canonical = reversed ? arc.slice().reverse() : arc; + const out = douglasPeucker(canonical, tolerance); + return reversed ? out.reverse() : out; +} + +function polygonsOf(geometry) { + if (!geometry) return []; + return geometry.type === 'Polygon' + ? [geometry.coordinates] + : geometry.type === 'MultiPolygon' + ? geometry.coordinates + : []; +} + +// A closed ring without its repeated closing point. +const open = (ring) => (ring.length > 1 && key(ring[0]) === key(ring[ring.length - 1]) ? ring.slice(0, -1) : ring); + +/** Every vertex whose neighbours differ between the rings (or places) it appears in. */ +function findJunctions(rings) { + const neighbours = new Map(); + const junctions = new Set(); + for (const ring of rings) { + const n = ring.length; + for (let i = 0; i < n; i++) { + const k = key(ring[i]); + const a = key(ring[(i - 1 + n) % n]); + const b = key(ring[(i + 1) % n]); + const pair = a < b ? `${a}|${b}` : `${b}|${a}`; + const seen = neighbours.get(k); + if (seen === undefined) neighbours.set(k, pair); + else if (seen !== pair) junctions.add(k); + } + } + return junctions; +} + +function simplifyRing(ring, junctions, tolerance) { + const n = ring.length; + const cuts = []; + for (let i = 0; i < n; i++) if (junctions.has(key(ring[i]))) cuts.push(i); + + let points; + if (cuts.length === 0) { + // No junction: start at the smallest vertex so a twin ring starts at the same place. + let start = 0; + for (let i = 1; i < n; i++) if (key(ring[i]) < key(ring[start])) start = i; + const rotated = [...ring.slice(start), ...ring.slice(0, start), ring[start]]; + points = simplifyArc(rotated, tolerance).slice(0, -1); + } else { + points = []; + for (let c = 0; c < cuts.length; c++) { + const from = cuts[c]; + const to = cuts[(c + 1) % cuts.length]; + const arc = []; + for (let i = from; ; i = (i + 1) % n) { + arc.push(ring[i]); + if (arc.length > 1 && i === to) break; + } + points.push(...simplifyArc(arc, tolerance).slice(0, -1)); + } + } + if (new Set(points.map(key)).size < 3) return null; + return [...points, points[0]]; +} + +/** + * Simplify every feature's (Multi)Polygon with the given tolerance (in coordinate + * units), keeping the borders that features share identical on both sides. Returns new + * features; properties are passed through untouched. + */ +export function simplifySharedBorders(features, tolerance) { + const rings = features.flatMap((f) => polygonsOf(f.geometry).flatMap((poly) => poly.map(open))); + const junctions = findJunctions(rings); + return features.map((f) => { + const polygons = polygonsOf(f.geometry) + .map((poly) => { + const [outer, ...holes] = poly.map((ring) => simplifyRing(open(ring), junctions, tolerance)); + return outer ? [outer, ...holes.filter(Boolean)] : null; + }) + .filter(Boolean); + const geometry = + polygons.length === 0 + ? null + : polygons.length === 1 + ? { type: 'Polygon', coordinates: polygons[0] } + : { type: 'MultiPolygon', coordinates: polygons }; + return { ...f, geometry }; + }); +} diff --git a/server/src/db/migrations/Migration20200101046380_ireland_gained_its_counties.ts b/server/src/db/migrations/Migration20200101046380_ireland_gained_its_counties.ts new file mode 100644 index 000000000..50f52ab91 --- /dev/null +++ b/server/src/db/migrations/Migration20200101046380_ireland_gained_its_counties.ts @@ -0,0 +1,110 @@ +import { Migration } from '@mikro-orm/migrations'; + +interface RegionRow { + id: number; + user_id: number; + region_code: string; + region_name: string; +} + +/** The 26 counties the Atlas now ships for Ireland: ISO 3166-2:IE code, name, province. */ +const COUNTIES: ReadonlyArray = [ + ['IE-CW', 'Carlow', 'IE-L'], + ['IE-D', 'Dublin', 'IE-L'], + ['IE-KE', 'Kildare', 'IE-L'], + ['IE-KK', 'Kilkenny', 'IE-L'], + ['IE-LS', 'Laois', 'IE-L'], + ['IE-LD', 'Longford', 'IE-L'], + ['IE-LH', 'Louth', 'IE-L'], + ['IE-MH', 'Meath', 'IE-L'], + ['IE-OY', 'Offaly', 'IE-L'], + ['IE-WH', 'Westmeath', 'IE-L'], + ['IE-WX', 'Wexford', 'IE-L'], + ['IE-WW', 'Wicklow', 'IE-L'], + ['IE-CE', 'Clare', 'IE-M'], + ['IE-CO', 'Cork', 'IE-M'], + ['IE-KY', 'Kerry', 'IE-M'], + ['IE-LK', 'Limerick', 'IE-M'], + ['IE-TA', 'Tipperary', 'IE-M'], + ['IE-WD', 'Waterford', 'IE-M'], + ['IE-G', 'Galway', 'IE-C'], + ['IE-LM', 'Leitrim', 'IE-C'], + ['IE-MO', 'Mayo', 'IE-C'], + ['IE-RN', 'Roscommon', 'IE-C'], + ['IE-SO', 'Sligo', 'IE-C'], + ['IE-CN', 'Cavan', 'IE-U'], + ['IE-DL', 'Donegal', 'IE-U'], + ['IE-MN', 'Monaghan', 'IE-U'], +]; + +/** "County Dublin", "Co. Dublin" and "dublin" all name Dublin. */ +const countyKey = (name: string): string => + name + .trim() + .toLowerCase() + .replace(/^(county|co\.?)\s+/, ''); + +/** + * Ireland gained its counties (#2922). + * + * geoBoundaries has no county layer for Ireland, so the Atlas showed the four + * provinces (IE-C, IE-L, IE-M, IE-U) next to a UK of counties. The bundle now ships + * Tailte Éireann's 26 counties under their ISO 3166-2:IE codes. What exists already: + * + * `place_regions` is a re-derivable cache full of province rows that nothing would + * ever re-derive, so Ireland's rows are cleared and come back as counties on the next + * Atlas load, as Great Britain's did when it gained its counties. + * + * `visited_regions` holds what a user marked by hand. A marked province stays as it + * is: which of its counties the user saw is not ours to guess. A row whose name is a + * county under another code (a Natural Earth or geocoder code from older versions) + * takes the county's code, and is dropped where the user already holds that county. + * + * `hidden_regions` remembers a region the user switched off. A hidden province hid + * every place in it, so each of its counties is hidden in its place; the province row + * stays and keeps hiding a marked province. + * + * `bucket_list` is a wish, not a polygon, and is left alone. + */ +export class Migration20200101046380_ireland_gained_its_counties extends Migration { + override name = 'Migration20200101046380_ireland_gained_its_counties'; + + override async up(): Promise { + await this.execute(`DELETE FROM place_regions WHERE UPPER(COALESCE(country_code, '')) = 'IE'`); + + for (const [code, , province] of COUNTIES) { + await this.execute( + `INSERT INTO hidden_regions (user_id, region_code, country_code) + SELECT h.user_id, ?, h.country_code FROM hidden_regions h + WHERE h.region_code = ? AND UPPER(h.country_code) = 'IE' + AND NOT EXISTS (SELECT 1 FROM hidden_regions x WHERE x.user_id = h.user_id AND x.region_code = ?)`, + [code, province, code], + ); + } + + const rows = (await this.execute( + `SELECT id, user_id, region_code, region_name FROM visited_regions + WHERE UPPER(COALESCE(country_code, '')) = 'IE' ORDER BY id`, + )) as RegionRow[]; + const counties = new Map(COUNTIES.map(([code, name]) => [countyKey(name), { code, name }])); + const countyCodes = new Set(COUNTIES.map(([code]) => code)); + const held = new Set(rows.map((r) => `${r.user_id}|${r.region_code}`)); + + for (const row of rows) { + if (countyCodes.has(row.region_code)) continue; + const county = counties.get(countyKey(row.region_name || '')); + if (!county) continue; + const target = `${row.user_id}|${county.code}`; + if (held.has(target)) { + await this.execute(`DELETE FROM visited_regions WHERE id = ?`, [row.id]); + } else { + await this.execute(`UPDATE visited_regions SET region_code = ?, region_name = ? WHERE id = ?`, [ + county.code, + county.name, + row.id, + ]); + held.add(target); + } + } + } +} diff --git a/server/tests/unit/db/ireland-counties-migration.test.ts b/server/tests/unit/db/ireland-counties-migration.test.ts new file mode 100644 index 000000000..e96d65901 --- /dev/null +++ b/server/tests/unit/db/ireland-counties-migration.test.ts @@ -0,0 +1,147 @@ +/** + * `Migration20200101046380_ireland_gained_its_counties` (#2922): the Atlas ships + * Ireland's 26 counties instead of its four provinces. Cached place regions are + * cleared, hand-marked provinces stay, county names under stale codes take the county + * code, and a hidden province hides its counties. + */ +import { createMigrationOrm, migrateTo, pendingNames, rawExec, rawQuery } from '../../helpers/migration-step'; +import type { MikroORM } from '@mikro-orm/sqlite'; + +import { afterEach, describe, expect, it } from 'vitest'; + +const TARGET = 'Migration20200101046380_ireland_gained_its_counties'; +const LEINSTER = [ + 'IE-CW', + 'IE-D', + 'IE-KE', + 'IE-KK', + 'IE-LD', + 'IE-LH', + 'IE-LS', + 'IE-MH', + 'IE-OY', + 'IE-WH', + 'IE-WW', + 'IE-WX', +]; + +let orm: MikroORM; + +async function ormBeforeTarget(): Promise { + const o = await createMigrationOrm(); + const names = await pendingNames(o); + const index = names.indexOf(TARGET); + expect(index).toBeGreaterThan(0); + await migrateTo(o, names[index - 1]); + return o; +} + +async function seedUsers(): Promise { + await rawExec( + orm, + `INSERT INTO users (id, username, email, password_hash) VALUES (1, 'one', 'one@test', 'x'), (2, 'two', 'two@test', 'x')`, + ); +} + +const visited = (userId: number) => + rawQuery<{ region_code: string; region_name: string; country_code: string }>( + orm, + 'SELECT region_code, region_name, country_code FROM visited_regions WHERE user_id = ? ORDER BY region_code', + [userId], + ); + +const hidden = async (userId: number) => + ( + await rawQuery<{ region_code: string }>( + orm, + 'SELECT region_code FROM hidden_regions WHERE user_id = ? ORDER BY region_code', + [userId], + ) + ).map((r) => r.region_code); + +afterEach(async () => { + await orm?.close(true); +}); + +describe('Ireland gained its counties', () => { + it('IE-COUNTIES-001: clears the cached Irish place regions and leaves other countries cached', async () => { + orm = await ormBeforeTarget(); + await seedUsers(); + await rawExec(orm, `INSERT INTO trips (id, user_id, title) VALUES (1, 1, 'Isles')`); + await rawExec( + orm, + `INSERT INTO places (id, trip_id, name, lat, lng) VALUES (1, 1, 'Dublin', 53.34, -6.25), (2, 1, 'Belfast', 54.6, -5.93)`, + ); + await rawExec( + orm, + `INSERT INTO place_regions (place_id, country_code, region_code, region_name) VALUES + (1, 'IE', 'IE-L', 'Leinster'), (2, 'GB', 'GB-BELFAST', 'Belfast')`, + ); + + await migrateTo(orm, TARGET); + + expect(await rawQuery(orm, 'SELECT place_id, region_code FROM place_regions ORDER BY place_id')).toEqual([ + { place_id: 2, region_code: 'GB-BELFAST' }, + ]); + }); + + it('IE-COUNTIES-002: keeps a marked province and a marked county, and moves a county name onto its code', async () => { + orm = await ormBeforeTarget(); + await seedUsers(); + await rawExec( + orm, + `INSERT INTO visited_regions (user_id, region_code, region_name, country_code) VALUES + (1, 'IE-L', 'Leinster', 'IE'), + (1, 'IE-KY', 'Kerry', 'IE'), + (1, 'IE-COR', 'County Cork', 'IE'), + (1, 'IE-GALWAY', 'Co. Galway', 'ie'), + (1, 'FR-IDF', 'Ile-de-France', 'FR')`, + ); + + await migrateTo(orm, TARGET); + + expect(await visited(1)).toEqual([ + { region_code: 'FR-IDF', region_name: 'Ile-de-France', country_code: 'FR' }, + { region_code: 'IE-CO', region_name: 'Cork', country_code: 'IE' }, + { region_code: 'IE-G', region_name: 'Galway', country_code: 'ie' }, + { region_code: 'IE-KY', region_name: 'Kerry', country_code: 'IE' }, + { region_code: 'IE-L', region_name: 'Leinster', country_code: 'IE' }, + ]); + }); + + it('IE-COUNTIES-003: drops a stale row whose county the user already holds, never touching another user', async () => { + orm = await ormBeforeTarget(); + await seedUsers(); + await rawExec( + orm, + `INSERT INTO visited_regions (user_id, region_code, region_name, country_code) VALUES + (1, 'IE-D', 'Dublin', 'IE'), + (1, 'IE-DUB', 'County Dublin', 'IE'), + (1, 'IE-DN', 'dublin', 'IE'), + (2, 'IE-DUB', 'County Dublin', 'IE')`, + ); + + await migrateTo(orm, TARGET); + + expect(await visited(1)).toEqual([{ region_code: 'IE-D', region_name: 'Dublin', country_code: 'IE' }]); + expect(await visited(2)).toEqual([{ region_code: 'IE-D', region_name: 'Dublin', country_code: 'IE' }]); + }); + + it('IE-COUNTIES-004: a hidden province hides each of its counties, once, and stays hidden itself', async () => { + orm = await ormBeforeTarget(); + await seedUsers(); + await rawExec( + orm, + `INSERT INTO hidden_regions (user_id, region_code, country_code) VALUES + (1, 'IE-L', 'IE'), + (1, 'IE-D', 'IE'), + (2, 'IE-U', 'IE'), + (2, 'FR-IDF', 'FR')`, + ); + + await migrateTo(orm, TARGET); + + expect(await hidden(1)).toEqual([...LEINSTER, 'IE-L'].sort()); + expect(await hidden(2)).toEqual(['FR-IDF', 'IE-CN', 'IE-DL', 'IE-MN', 'IE-U']); + }); +}); diff --git a/server/tests/unit/db/legacy-baseline.test.ts b/server/tests/unit/db/legacy-baseline.test.ts index ef7d8fd91..7dc5c389a 100644 --- a/server/tests/unit/db/legacy-baseline.test.ts +++ b/server/tests/unit/db/legacy-baseline.test.ts @@ -105,6 +105,7 @@ const UNNUMBERED = [ 'Migration20200101046350_a_place_lies_on_the_real_world', 'Migration20200101046360_a_new_install_starts_with_more_addons_on', 'Migration20200101046370_a_deleted_chat_message_keeps_no_text', + 'Migration20200101046380_ireland_gained_its_counties', ]; // The last step the positional runner ever had; planLegacyBaseline refuses a // schema_version past it. diff --git a/server/tests/unit/nest/atlas-region-files.test.ts b/server/tests/unit/nest/atlas-region-files.test.ts index 4d11555c2..d8d5b40b1 100644 --- a/server/tests/unit/nest/atlas-region-files.test.ts +++ b/server/tests/unit/nest/atlas-region-files.test.ts @@ -1,4 +1,4 @@ -import { getRegionGeoGz, getRegionGeoJson } from '../../../src/nest/atlas/atlas-geo'; +import { getRegionFromCoords, getRegionGeoGz, getRegionGeoJson } from '../../../src/nest/atlas/atlas-geo'; import { describe, it, expect } from 'vitest'; import zlib from 'zlib'; @@ -57,3 +57,33 @@ describe('getRegionGeoGz', () => { expect(await Promise.all(codes.map((code) => getRegionGeoGz(code)))).toEqual(codes.map(() => null)); }); }); + +describe('Ireland resolves to its counties (#2922)', () => { + // Inland, coastal and near-border places, one county each. The counties come from + // Tailte Éireann and are simplified by the builder itself, so a point that lands in no + // county or in the wrong one would mean the simplification opened a gap or moved a border. + const places: [string, number, number, string][] = [ + ['Dublin, Trinity College', 53.3438, -6.2546, 'IE-D'], + ['Howth village', 53.3866, -6.07, 'IE-D'], + ['Cork, St. Patrick Street', 51.8985, -8.4731, 'IE-CO'], + ['Galway, Eyre Square', 53.2743, -9.0491, 'IE-G'], + ['Killarney', 52.0599, -9.5044, 'IE-KY'], + ['Cliffs of Moher visitor centre', 52.9719, -9.4265, 'IE-CE'], + ['Kilkenny Castle', 52.6503, -7.2492, 'IE-KK'], + ['Letterkenny', 54.9503, -7.7334, 'IE-DL'], + ['Westport', 53.8008, -9.5223, 'IE-MO'], + ['Cashel, Rock of Cashel', 52.5201, -7.8906, 'IE-TA'], + ]; + + it.each(places)('ATLAS-SVC-051: %s lies in its county', async (_name, lat, lng, code) => { + expect((await getRegionFromCoords('IE', lat, lng))?.region_code).toBe(code); + }); + + it('ATLAS-SVC-052: a county comes back under its English name', async () => { + expect(await getRegionFromCoords('ie', 53.3438, -6.2546)).toEqual({ + country_code: 'IE', + region_code: 'IE-D', + region_name: 'Dublin', + }); + }); +}); diff --git a/server/tests/unit/nest/atlasBundle.test.ts b/server/tests/unit/nest/atlasBundle.test.ts index 04d288263..5e0185d04 100644 --- a/server/tests/unit/nest/atlasBundle.test.ts +++ b/server/tests/unit/nest/atlasBundle.test.ts @@ -152,6 +152,26 @@ describe('Atlas admin1 region bundle (#1217)', () => { } }); + /* + * geoBoundaries has no county layer for Ireland: its ADM1 is the four provinces and + * its ADM2 the 166 local electoral areas, so Ireland showed four provinces next to + * a UK of counties (#2922). Ireland's regions come from Tailte Éireann's 26 counties + * instead, under their ISO 3166-2:IE codes. + */ + it('ATLAS-BUNDLE-011 — Ireland ships its 26 counties, not the four provinces (#2922)', () => { + const ie = regions('IE') as unknown as { properties: { iso_3166_2: string; name: string; admin: string } }[]; + const codes = ie.map((f) => f.properties.iso_3166_2).sort(); + // ISO 3166-2:IE, Dublin, Cork, Galway and Limerick each one county (city and county together). + const iso = + 'IE-CE IE-CN IE-CO IE-CW IE-D IE-DL IE-G IE-KE IE-KK IE-KY IE-LD IE-LH IE-LK ' + + 'IE-LM IE-LS IE-MH IE-MN IE-MO IE-OY IE-RN IE-SO IE-TA IE-WD IE-WH IE-WW IE-WX'; + expect(codes).toEqual(iso.split(' ')); + const byCode = new Map(ie.map((f) => [f.properties.iso_3166_2, f.properties])); + expect(byCode.get('IE-D')?.name).toBe('Dublin'); + expect(byCode.get('IE-TA')?.name).toBe('Tipperary'); + expect(new Set(ie.map((f) => f.properties.admin))).toEqual(new Set(['Ireland'])); + }); + it('ATLAS-BUNDLE-006 — China ships Guangdong Province, not Guangzhou Province', () => { const cn = ( features as unknown as { properties: { iso_a2: string | null; name?: string; iso_3166_2: string } }[] diff --git a/server/tests/unit/scripts/shared-border-simplify.test.ts b/server/tests/unit/scripts/shared-border-simplify.test.ts new file mode 100644 index 000000000..96b76231d --- /dev/null +++ b/server/tests/unit/scripts/shared-border-simplify.test.ts @@ -0,0 +1,161 @@ +/** + * scripts/lib/shared-border-simplify.mjs simplifies a country's region outlines for the + * Atlas builder when the source has no simplified version of its own (Tailte Éireann's + * counties, #2922). What matters is that two regions sharing a border keep the same + * points on it, so no point between them falls into a sliver that belongs to neither. + */ +import { douglasPeucker, simplifySharedBorders } from '../../../scripts/lib/shared-border-simplify.mjs'; + +import { describe, expect, it } from 'vitest'; + +type Ring = number[][]; +type Geometry = { type: 'Polygon'; coordinates: Ring[] } | { type: 'MultiPolygon'; coordinates: Ring[][] }; +type Feature = { type: 'Feature'; properties: Record; geometry: Geometry | null }; + +const polygon = (name: string, ...rings: Ring[]): Feature => ({ + type: 'Feature', + properties: { name }, + geometry: { type: 'Polygon', coordinates: rings }, +}); + +// A border from about (1,0) up to about (1,1) that swings out and back, quantized like the builder's input. +const border: Ring = Array.from({ length: 41 }, (_, i) => [ + Math.round((1 + 0.05 * Math.sin(i * 0.7)) * 1000) / 1000, + i / 40, +]); + +/** + * Two neighbours: west runs up the shared border, east runs down it from a different + * start. Simplified each on its own, these two keep different points on the border + * (SIMPLIFY-003 checks that the fixture really does that). + */ +function neighbours(): Feature[] { + const west = [[0, 0], ...border, [0, 1], [0, 0]]; + const east = [[2, 0], [2, 1], ...border.slice().reverse(), [2, 0]]; + return [polygon('west', west), polygon('east', east)]; +} + +const outer = (f: Feature) => (f.geometry as { coordinates: Ring[] }).coordinates[0]; +// The points of a ring on the shared border, its repeated closing point left out. +const onBorder = (ring: Ring) => + ring + .slice(0, -1) + .filter(([x]) => x > 0.5 && x < 1.5) + .map((p) => p.join(',')) + .sort(); + +describe('douglasPeucker', () => { + it('SIMPLIFY-001: keeps both ends and drops points within the tolerance', () => { + expect( + douglasPeucker( + [ + [0, 0], + [1, 0.001], + [2, 0], + [3, 1], + ], + 0.01, + ), + ).toEqual([ + [0, 0], + [2, 0], + [3, 1], + ]); + }); + + it('SIMPLIFY-002: leaves a line of two points or fewer as it is', () => { + expect( + douglasPeucker( + [ + [0, 0], + [1, 1], + ], + 5, + ), + ).toEqual([ + [0, 0], + [1, 1], + ]); + }); +}); + +describe('simplifySharedBorders', () => { + it('SIMPLIFY-003: two neighbours keep exactly the same points on the border they share', () => { + const [west, east] = neighbours(); + // Each polygon simplified on its own drifts apart on the border: the bug this avoids. + expect(onBorder(douglasPeucker(outer(west), 0.01))).not.toEqual(onBorder(douglasPeucker(outer(east), 0.01))); + + const [westOut, eastOut] = simplifySharedBorders([west, east], 0.01) as Feature[]; + expect(onBorder(outer(westOut))).toEqual(onBorder(outer(eastOut))); + expect(onBorder(outer(westOut)).length).toBeLessThan(border.length); + // Its two ends are where three outlines meet, and stay put. + expect(onBorder(outer(westOut))).toEqual(expect.arrayContaining([border[0].join(','), border[40].join(',')])); + }); + + it('SIMPLIFY-004: the result does not depend on which neighbour comes first', () => { + const [east, west] = simplifySharedBorders(neighbours().reverse(), 0.01) as Feature[]; + const [west2, east2] = simplifySharedBorders(neighbours(), 0.01) as Feature[]; + expect(outer(west)).toEqual(outer(west2)); + expect(outer(east)).toEqual(outer(east2)); + }); + + it('SIMPLIFY-005: an enclave shared whole with the hole around it simplifies the same on both sides', () => { + const circle = (reverse: boolean): Ring => { + const pts = Array.from({ length: 40 }, (_, i) => { + const a = (i / 40) * 2 * Math.PI; + return [Math.round((5 + Math.cos(a)) * 1000) / 1000, Math.round((5 + Math.sin(a)) * 1000) / 1000]; + }); + const ring = reverse ? pts.reverse() : pts; + return [...ring, ring[0]]; + }; + const square: Ring = [ + [0, 0], + [10, 0], + [10, 10], + [0, 10], + [0, 0], + ]; + const [around, enclave] = simplifySharedBorders( + [polygon('around', square, circle(true)), polygon('enclave', circle(false))], + 0.05, + ) as Feature[]; + const hole = (around.geometry as { coordinates: Ring[] }).coordinates[1]; + const key = (ring: Ring) => ring.map((p) => p.join(',')).sort(); + expect(key(hole)).toEqual(key(outer(enclave))); + expect(outer(enclave).length).toBeLessThan(41); + }); + + it('SIMPLIFY-006: a polygon too small to keep is dropped, and a feature with nothing left gets no geometry', () => { + const speck: Ring = [ + [0, 0], + [0.001, 0], + [0.001, 0.001], + [0, 0], + ]; + const island = polygon('island', speck); + const [out] = simplifySharedBorders([island], 0.01) as Feature[]; + expect(out.geometry).toBeNull(); + expect(out.properties).toEqual({ name: 'island' }); + }); + + it('SIMPLIFY-007: a MultiPolygon keeps the parts that survive and becomes a Polygon when one is left', () => { + const big: Ring = [ + [0, 0], + [4, 0], + [4, 4], + [0, 4], + [0, 0], + ]; + const speck: Ring = [ + [9, 9], + [9.001, 9], + [9.001, 9.001], + [9, 9], + ]; + const [out] = simplifySharedBorders( + [{ type: 'Feature', properties: {}, geometry: { type: 'MultiPolygon', coordinates: [[big], [speck]] } }], + 0.01, + ) as Feature[]; + expect(out.geometry).toEqual({ type: 'Polygon', coordinates: [big] }); + }); +});