diff --git a/server/assets/atlas/admin1.geojson.gz b/server/assets/atlas/admin1.geojson.gz index 0481e99d3..ab15bff1b 100644 Binary files a/server/assets/atlas/admin1.geojson.gz and b/server/assets/atlas/admin1.geojson.gz differ diff --git a/server/scripts/build-atlas-geo.mjs b/server/scripts/build-atlas-geo.mjs index 385f4bfdc..29cd5d33c 100644 --- a/server/scripts/build-atlas-geo.mjs +++ b/server/scripts/build-atlas-geo.mjs @@ -49,6 +49,22 @@ const RETRIES = 3 // what restores the dropped territories (Greenland, Falklands, French Guiana, …). const A3_TO_A2 = {"ABW":"AW", "AFG":"AF", "AGO":"AO", "AIA":"AI", "ALA":"AX", "ALB":"AL", "AND":"AD", "ARE":"AE", "ARG":"AR", "ARM":"AM", "ASM":"AS", "ATA":"AQ", "ATF":"TF", "ATG":"AG", "AUS":"AU", "AUT":"AT", "AZE":"AZ", "BDI":"BI", "BEL":"BE", "BEN":"BJ", "BES":"BQ", "BFA":"BF", "BGD":"BD", "BGR":"BG", "BHR":"BH", "BHS":"BS", "BIH":"BA", "BLM":"BL", "BLR":"BY", "BLZ":"BZ", "BMU":"BM", "BOL":"BO", "BRA":"BR", "BRB":"BB", "BRN":"BN", "BTN":"BT", "BVT":"BV", "BWA":"BW", "CAF":"CF", "CAN":"CA", "CCK":"CC", "CHE":"CH", "CHL":"CL", "CHN":"CN", "CIV":"CI", "CMR":"CM", "COD":"CD", "COG":"CG", "COK":"CK", "COL":"CO", "COM":"KM", "CPV":"CV", "CRI":"CR", "CUB":"CU", "CUW":"CW", "CXR":"CX", "CYM":"KY", "CYP":"CY", "CZE":"CZ", "DEU":"DE", "DJI":"DJ", "DMA":"DM", "DNK":"DK", "DOM":"DO", "DZA":"DZ", "ECU":"EC", "EGY":"EG", "ERI":"ER", "ESH":"EH", "ESP":"ES", "EST":"EE", "ETH":"ET", "FIN":"FI", "FJI":"FJ", "FLK":"FK", "FRA":"FR", "FRO":"FO", "FSM":"FM", "GAB":"GA", "GBR":"GB", "GEO":"GE", "GGY":"GG", "GHA":"GH", "GIB":"GI", "GIN":"GN", "GLP":"GP", "GMB":"GM", "GNB":"GW", "GNQ":"GQ", "GRC":"GR", "GRD":"GD", "GRL":"GL", "GTM":"GT", "GUF":"GF", "GUM":"GU", "GUY":"GY", "HKG":"HK", "HMD":"HM", "HND":"HN", "HRV":"HR", "HTI":"HT", "HUN":"HU", "IDN":"ID", "IMN":"IM", "IND":"IN", "IOT":"IO", "IRL":"IE", "IRN":"IR", "IRQ":"IQ", "ISL":"IS", "ISR":"IL", "ITA":"IT", "JAM":"JM", "JEY":"JE", "JOR":"JO", "JPN":"JP", "KAZ":"KZ", "KEN":"KE", "KGZ":"KG", "KHM":"KH", "KIR":"KI", "KNA":"KN", "KOR":"KR", "KWT":"KW", "LAO":"LA", "LBN":"LB", "LBR":"LR", "LBY":"LY", "LCA":"LC", "LIE":"LI", "LKA":"LK", "LSO":"LS", "LTU":"LT", "LUX":"LU", "LVA":"LV", "MAC":"MO", "MAF":"MF", "MAR":"MA", "MCO":"MC", "MDA":"MD", "MDG":"MG", "MDV":"MV", "MEX":"MX", "MHL":"MH", "MKD":"MK", "MLI":"ML", "MLT":"MT", "MMR":"MM", "MNE":"ME", "MNG":"MN", "MNP":"MP", "MOZ":"MZ", "MRT":"MR", "MSR":"MS", "MTQ":"MQ", "MUS":"MU", "MWI":"MW", "MYS":"MY", "MYT":"YT", "NAM":"NA", "NCL":"NC", "NER":"NE", "NFK":"NF", "NGA":"NG", "NIC":"NI", "NIU":"NU", "NLD":"NL", "NOR":"NO", "NPL":"NP", "NRU":"NR", "NZL":"NZ", "OMN":"OM", "PAK":"PK", "PAN":"PA", "PCN":"PN", "PER":"PE", "PHL":"PH", "PLW":"PW", "PNG":"PG", "POL":"PL", "PRI":"PR", "PRK":"KP", "PRT":"PT", "PRY":"PY", "PSE":"PS", "PYF":"PF", "QAT":"QA", "REU":"RE", "ROU":"RO", "RUS":"RU", "RWA":"RW", "SAU":"SA", "SDN":"SD", "SEN":"SN", "SGP":"SG", "SGS":"GS", "SHN":"SH", "SJM":"SJ", "SLB":"SB", "SLE":"SL", "SLV":"SV", "SMR":"SM", "SOM":"SO", "SPM":"PM", "SRB":"RS", "SSD":"SS", "STP":"ST", "SUR":"SR", "SVK":"SK", "SVN":"SI", "SWE":"SE", "SWZ":"SZ", "SXM":"SX", "SYC":"SC", "SYR":"SY", "TCA":"TC", "TCD":"TD", "TGO":"TG", "THA":"TH", "TJK":"TJ", "TKL":"TK", "TKM":"TM", "TLS":"TL", "TON":"TO", "TTO":"TT", "TUN":"TN", "TUR":"TR", "TUV":"TV", "TWN":"TW", "TZA":"TZ", "UGA":"UG", "UKR":"UA", "UMI":"UM", "URY":"UY", "USA":"US", "UZB":"UZ", "VAT":"VA", "VCT":"VC", "VEN":"VE", "VGB":"VG", "VIR":"VI", "VNM":"VN", "VUT":"VU", "WLF":"WF", "WSM":"WS", "XKX":"XK", "YEM":"YE", "ZAF":"ZA", "ZMB":"ZM", "ZWE":"ZW"} +/** + * Countries whose sub-national level is not the one everybody else's ADM1 is. + * + * geoBoundaries' GBR ADM1 is the four constituent countries — England, Scotland, + * Wales, Northern Ireland — so an atlas built from it has no polygon below + * "England", and a London borough marked as visited has nothing to light up + * (#1974). Its ADM2 is the 216 counties and unitary authorities, which is the + * granularity the rest of the world gets from ADM1 and the granularity the + * pre-3.1.0 Natural Earth layer had. + * + * Deliberately a short, hand-picked list. ADM2 everywhere would multiply the + * bundle several times over for no gain: for most countries ADM1 already is the + * level people think in. + */ +const ADM_LEVEL = { GBR: 'ADM2' } + const COUNTRIES = Object.keys(A3_TO_A2) // every ISO alpha-3 code (ADM1 fetch list) // Cache raw downloads so re-runs (e.g. to tune simplification) don't re-fetch ~360 files. @@ -216,7 +232,7 @@ async function main() { // ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`). console.log('[atlas-geo] downloading ADM1 (regions)…') - const adm1Raw = await pool(COUNTRIES, a3 => fetchGeo(MEDIA(a3, 'ADM1'))) + const adm1Raw = await pool(COUNTRIES, a3 => fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1'))) const adm1Features = [] let withCodes = 0 COUNTRIES.forEach((a3, idx) => { diff --git a/server/src/db/migrations.ts b/server/src/db/migrations.ts index ad696002d..ac2840b5d 100644 --- a/server/src/db/migrations.ts +++ b/server/src/db/migrations.ts @@ -4011,6 +4011,62 @@ function runMigrations(db: Database.Database): void { CREATE INDEX IF NOT EXISTS idx_journey_books_journey ON journey_books(journey_id); `); }, + /* + * Great Britain gained its counties and boroughs back (#1974). + * + * geoBoundaries' GBR ADM1 is the four constituent countries, so since 3.1.0 + * the atlas had no polygon below "England" and a London borough marked as + * visited had nothing to light up. The bundle now ships GB at ADM2, which is + * the granularity the rest of the world gets and the granularity the + * pre-3.1.0 Natural Earth layer had. + * + * Two things need saying to the data that already exists: + * + * `visited_regions` rows marked before 3.1.0 hold the old Natural Earth + * borough codes. They are reconciled by name within the same country, + * exactly as the 3.1.1 step above does for every other country — a row that + * still cannot be resolved is left alone rather than destroyed, and the + * client's name fallback may well still highlight it. + * + * `place_regions` is a re-derivable cache, and it is full of GB-ENG rows + * that were correct under the old bundle. Nothing re-derives a row that is + * already there, so without clearing them every English place would keep + * reporting "England" forever. + */ + () => { + db.exec(`DELETE FROM place_regions WHERE UPPER(COALESCE(country_code, '')) = 'GB'`); + + type Row = { id: number; region_code: string; region_name: string; country_code: string }; + const rows = db + .prepare(`SELECT id, region_code, region_name, country_code FROM visited_regions WHERE UPPER(COALESCE(country_code, '')) = 'GB'`) + .all() as Row[]; + if (rows.length === 0) return; + + let features: { properties?: { iso_a2?: string; iso_3166_2?: string; name?: string } }[]; + try { + const file = path.join(__dirname, '..', '..', 'assets', 'atlas', 'admin1.geojson.gz'); + features = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf8')).features || []; + } catch { + return; // bundle unreadable — leave the rows untouched rather than guessing + } + + const validCodes = new Set(); + const nameToCode = new Map(); + for (const f of features) { + const p = f.properties || {}; + if ((p.iso_a2 || '').toUpperCase() !== 'GB' || !p.iso_3166_2) continue; + validCodes.add(p.iso_3166_2); + if (p.name) nameToCode.set(p.name.toLowerCase(), p.iso_3166_2); + } + if (validCodes.size === 0) return; + + const update = db.prepare('UPDATE OR IGNORE visited_regions SET region_code = ? WHERE id = ?'); + for (const row of rows) { + if (validCodes.has(row.region_code)) continue; + const byName = nameToCode.get((row.region_name || '').toLowerCase()); + if (byName) update.run(byName, row.id); + } + }, ]; if (currentVersion < migrations.length) { diff --git a/server/src/nest/atlas/atlas-geo.ts b/server/src/nest/atlas/atlas-geo.ts index 6b7776dbe..3858b7ea7 100644 --- a/server/src/nest/atlas/atlas-geo.ts +++ b/server/src/nest/atlas/atlas-geo.ts @@ -963,9 +963,9 @@ export async function reverseGeocodeRegion(lat: number, lng: number, placeAddres // Only reached when the bundle's own polygons for this country don't cover the point at // all (coastal/simplification gaps) — a genuinely rare miss. Nominatim's coarse address - // level (state/province) is what the bundle actually carries; the former GB "rescue" to a - // finer county/borough level targeted the old Natural Earth polygons and produced a code - // the current geoBoundaries bundle can never match, so it was removed. + // level (state/province) is what the bundle carries for most countries; GB is the + // exception and ships at county/borough level (#1974), which is also what its old + // Natural Earth polygons had. const address = await fetchNominatimAddress(lat, lng, 8); if (!address) return null; // transient failure — leave uncached so a later call retries const info = buildRegionInfo(address, false); diff --git a/server/tests/unit/nest/atlas.service.test.ts b/server/tests/unit/nest/atlas.service.test.ts index e35c1c1be..f25711d4b 100644 --- a/server/tests/unit/nest/atlas.service.test.ts +++ b/server/tests/unit/nest/atlas.service.test.ts @@ -973,10 +973,11 @@ describe('getVisitedRegions', () => { }); it('ATLAS-UNIT-021: a GB place resolves against the bundled admin1 polygon without calling Nominatim', async () => { - // The shipped geoBoundaries bundle only carries GB's 4 constituent countries - // (England/Scotland/Wales/Northern Ireland) — no county/borough level. Resolving - // Old Trafford's coordinates directly against those polygons lands on GB-ENG, the - // same feature the client highlights, with no reverse-geocode round trip at all. + // GB ships at ADM2 since #1974 — the 216 counties and unitary authorities, + // rather than the four constituent countries geoBoundaries calls ADM1. + // Old Trafford's coordinates therefore land on Trafford itself, which is + // both the feature the client highlights and the level a person thinks in; + // still with no reverse-geocode round trip at all. const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); @@ -992,8 +993,10 @@ describe('getVisitedRegions', () => { expect(mockFetch).not.toHaveBeenCalled(); expect(result.regions['GB']).toBeDefined(); - const codes = result.regions['GB'].map((r: any) => r.code); - expect(codes).toContain('GB-ENG'); + const names = result.regions['GB'].map((r: any) => (r.name || '').toLowerCase()); + // Below the constituent-country level: the borough, not the nation. + expect(names).not.toContain('england'); + expect(names.some((n: string) => n.includes('trafford'))).toBe(true); }); it('ATLAS-UNIT-022: a place whose Nominatim region level is finer than the bundle (Spain province vs autonomous community) still resolves to a bundle-matching feature', async () => { diff --git a/server/tests/unit/services/atlasBundle.test.ts b/server/tests/unit/services/atlasBundle.test.ts index d4ded7774..1eff452d7 100644 --- a/server/tests/unit/services/atlasBundle.test.ts +++ b/server/tests/unit/services/atlasBundle.test.ts @@ -51,4 +51,31 @@ describe('Atlas admin1 region bundle (#1217)', () => { expect(new Set(f.map(r => r.properties.iso_3166_2)).size).toBe(f.length); } }); + + /* + * geoBoundaries' GBR ADM1 is the four constituent countries, so a bundle built + * from it has no polygon below "England" — a London borough marked as visited + * has nothing to light up, and every English place reports "England" (#1974). + * The builder ships GB at ADM2 for exactly that reason. + * + * This is the guard that would have caught it when it landed in 3.1.0: four + * features for a whole country is not a region layer, it is a country layer + * with extra steps. + */ + it('ATLAS-BUNDLE-004 — Great Britain ships below the constituent-country level (#1974)', () => { + const gb = regions('GB'); + expect(gb.length, 'GB must ship counties and boroughs, not the four nations').toBeGreaterThan(50); + expect(new Set(gb.map(r => r.properties.iso_3166_2)).size, 'GB region codes must be unique').toBe(gb.length); + }); + + it('ATLAS-BUNDLE-005 — the London boroughs are in it (#1974)', () => { + const names = new Set( + (features as unknown as { properties: { iso_a2: string | null; name?: string } }[]) + .filter(f => f.properties.iso_a2 === 'GB') + .map(f => (f.properties.name || '').toLowerCase()), + ); + for (const borough of ['camden', 'westminster', 'hackney', 'islington']) { + expect(names.has(borough), `${borough} should be a region of its own`).toBe(true); + } + }); });