mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 11:06:06 +00:00
merge dev into feat/postgres-support
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env node
|
||||
// Build server/assets/airlines-icao.json, the IATA airline code -> ICAO callsign prefix table the
|
||||
// live flight status uses to turn a booked flight number ("LH 716") into the callsign an aircraft
|
||||
// transmits ("DLH716").
|
||||
//
|
||||
// Source: VRS standing data (github.com/vradarserver/standing-data, CC0): airlines/schema-01 for
|
||||
// the codes and the file list of routes/schema-01 for which airline still flies.
|
||||
//
|
||||
// Only airlines with both codes are kept. IATA reuses its two-character codes, so one IATA code
|
||||
// often names several ICAO airlines (LH: Lufthansa and Lufthansa Cargo; U2: easyJet and United
|
||||
// Feeder Service). The airline list carries no "active" flag, so the rule is: the ICAO airline
|
||||
// with the most route data in the standing data wins (the summed size of its routes files), since
|
||||
// routes are only recorded for callsigns seen flying. A shared code where no candidate has any
|
||||
// route data is left out rather than guessed; a booking with it is then simply not tracked.
|
||||
import fs from 'node:fs';
|
||||
import https from 'node:https';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const OUT = path.join(__dirname, '..', 'assets', 'airlines-icao.json');
|
||||
const AIRLINES = 'https://raw.githubusercontent.com/vradarserver/standing-data/main/airlines/schema-01/airlines.csv';
|
||||
const TREE = 'https://api.github.com/repos/vradarserver/standing-data/git/trees/main?recursive=1';
|
||||
|
||||
function fetchText(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(url, { headers: { 'User-Agent': 'trek-build-airlines', Accept: '*/*' } }, (res) => {
|
||||
if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
|
||||
let data = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', () => resolve(data));
|
||||
})
|
||||
.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/** One CSV line into its fields: optional double quotes, "" as an escaped quote. */
|
||||
function parseLine(line) {
|
||||
const fields = [];
|
||||
let cur = '';
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"' && line[i + 1] === '"') {
|
||||
cur += '"';
|
||||
i++;
|
||||
} else if (ch === '"') inQuotes = false;
|
||||
else cur += ch;
|
||||
} else if (ch === '"') inQuotes = true;
|
||||
else if (ch === ',') {
|
||||
fields.push(cur);
|
||||
cur = '';
|
||||
} else cur += ch;
|
||||
}
|
||||
fields.push(cur);
|
||||
return fields;
|
||||
}
|
||||
|
||||
const csv = (await fetchText(AIRLINES)).replace(/^/, '');
|
||||
const tree = JSON.parse(await fetchText(TREE));
|
||||
if (tree.truncated) throw new Error('The standing-data tree came back truncated; the route sizes would be incomplete');
|
||||
|
||||
// Bytes of route data per ICAO code: routes/schema-01/<letter>/<CODE>-all.csv or <CODE>-<digit>.csv.
|
||||
const routeBytes = new Map();
|
||||
for (const entry of tree.tree) {
|
||||
const match = /^routes\/schema-01\/[A-Z0-9]\/([A-Z]{3})-(?:all|\d)\.csv$/.exec(entry.path);
|
||||
if (!match) continue;
|
||||
routeBytes.set(match[1], (routeBytes.get(match[1]) ?? 0) + (entry.size ?? 0));
|
||||
}
|
||||
|
||||
const rows = csv.split(/\r?\n/);
|
||||
const header = parseLine(rows[0]);
|
||||
const ICAO = header.indexOf('ICAO');
|
||||
const IATA = header.indexOf('IATA');
|
||||
const candidates = new Map();
|
||||
for (const line of rows.slice(1)) {
|
||||
if (!line.trim()) continue;
|
||||
const fields = parseLine(line);
|
||||
const icao = fields[ICAO]?.trim().toUpperCase();
|
||||
const iata = fields[IATA]?.trim().toUpperCase();
|
||||
if (!icao || !iata || !/^[A-Z]{3}$/.test(icao) || !/^(?=.*[A-Z])[A-Z0-9]{2}$/.test(iata)) continue;
|
||||
const list = candidates.get(iata) ?? [];
|
||||
if (!list.includes(icao)) list.push(icao);
|
||||
candidates.set(iata, list);
|
||||
}
|
||||
|
||||
const table = {};
|
||||
let shared = 0;
|
||||
let dropped = 0;
|
||||
for (const iata of [...candidates.keys()].sort()) {
|
||||
const list = candidates.get(iata);
|
||||
if (list.length === 1) {
|
||||
table[iata] = list[0];
|
||||
continue;
|
||||
}
|
||||
shared++;
|
||||
const ranked = list
|
||||
.map((icao) => ({ icao, bytes: routeBytes.get(icao) ?? 0 }))
|
||||
.sort((a, b) => b.bytes - a.bytes || a.icao.localeCompare(b.icao));
|
||||
if (ranked[0].bytes === 0) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
table[iata] = ranked[0].icao;
|
||||
}
|
||||
|
||||
fs.writeFileSync(OUT, JSON.stringify(table));
|
||||
console.log(
|
||||
`Wrote ${Object.keys(table).length} airline codes to ${OUT}; ${shared} shared IATA codes resolved by route data, ${dropped} left out without any`,
|
||||
);
|
||||
@@ -14,7 +14,13 @@
|
||||
// FeatureCollections that the server serves at runtime (no network at boot): one for
|
||||
// every country's borders, and one per country for its regions.
|
||||
//
|
||||
// A few countries take their regions from another source instead (REGION_SOURCES below:
|
||||
// Ireland's counties from Tailte Éireann).
|
||||
//
|
||||
// geoBoundaries: CC BY 4.0 — https://www.geoboundaries.org/ (attribution required).
|
||||
// Tailte Éireann: CC BY 4.0 — https://data-osi.opendata.arcgis.com/ (attribution required).
|
||||
import { simplifySharedBorders } from './lib/shared-border-simplify.mjs';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -317,13 +323,86 @@ const A3_TO_A2 = {
|
||||
*/
|
||||
const ADM_LEVEL = { GBR: 'ADM2' };
|
||||
|
||||
/**
|
||||
* The 26 counties of Ireland, keyed by the COUNTY field of Tailte Éireann's layer, with
|
||||
* their ISO 3166-2:IE codes (the source carries none).
|
||||
*/
|
||||
const IE_COUNTIES = {
|
||||
CARLOW: ['IE-CW', 'Carlow'],
|
||||
CAVAN: ['IE-CN', 'Cavan'],
|
||||
CLARE: ['IE-CE', 'Clare'],
|
||||
CORK: ['IE-CO', 'Cork'],
|
||||
DONEGAL: ['IE-DL', 'Donegal'],
|
||||
DUBLIN: ['IE-D', 'Dublin'],
|
||||
GALWAY: ['IE-G', 'Galway'],
|
||||
KERRY: ['IE-KY', 'Kerry'],
|
||||
KILDARE: ['IE-KE', 'Kildare'],
|
||||
KILKENNY: ['IE-KK', 'Kilkenny'],
|
||||
LAOIS: ['IE-LS', 'Laois'],
|
||||
LEITRIM: ['IE-LM', 'Leitrim'],
|
||||
LIMERICK: ['IE-LK', 'Limerick'],
|
||||
LONGFORD: ['IE-LD', 'Longford'],
|
||||
LOUTH: ['IE-LH', 'Louth'],
|
||||
MAYO: ['IE-MO', 'Mayo'],
|
||||
MEATH: ['IE-MH', 'Meath'],
|
||||
MONAGHAN: ['IE-MN', 'Monaghan'],
|
||||
OFFALY: ['IE-OY', 'Offaly'],
|
||||
ROSCOMMON: ['IE-RN', 'Roscommon'],
|
||||
SLIGO: ['IE-SO', 'Sligo'],
|
||||
TIPPERARY: ['IE-TA', 'Tipperary'],
|
||||
WATERFORD: ['IE-WD', 'Waterford'],
|
||||
WESTMEATH: ['IE-WH', 'Westmeath'],
|
||||
WEXFORD: ['IE-WX', 'Wexford'],
|
||||
WICKLOW: ['IE-WW', 'Wicklow'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Countries whose regions come from somewhere other than geoBoundaries, because no
|
||||
* geoBoundaries level is the one people think in.
|
||||
*
|
||||
* IRL: ADM1 is the four provinces and ADM2 the 166 local electoral areas; there is no
|
||||
* county layer at all (#2922). Tailte Éireann (CC BY 4.0) publishes the 26 counties,
|
||||
* with Dublin, Cork, Galway and Limerick each one county (city and county together),
|
||||
* which is what ISO 3166-2:IE lists.
|
||||
*
|
||||
* Each entry names the download, the file it is cached under, how many features it must
|
||||
* yield, the tolerance its full-detail outlines are simplified with (see
|
||||
* lib/shared-border-simplify.mjs) and how one feature's properties become the
|
||||
* geoBoundaries `shapeName`/`shapeISO` that normalizeAdm1 reads, so everything after the
|
||||
* download treats it like any other country. A feature it cannot place fails the build.
|
||||
*/
|
||||
const REGION_SOURCES = {
|
||||
IRL: {
|
||||
url:
|
||||
'https://services-eu1.arcgis.com/FH5XCsx8rYXqnjF5/arcgis/rest/services/' +
|
||||
'Counties___OSi_National_Statutory_Boundaries___Generalised_20m/FeatureServer/0/query' +
|
||||
'?where=1%3D1&outFields=COUNTY&outSR=4326&f=geojson',
|
||||
cacheFile: 'tailte-eireann-counties-2019-generalised-20m.geojson',
|
||||
count: 26,
|
||||
tolerance: 0.002,
|
||||
toShape: ({ COUNTY }) => {
|
||||
const county = IE_COUNTIES[COUNTY];
|
||||
return county && { shapeISO: county[0], shapeName: county[1] };
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const COUNTRIES = Object.keys(A3_TO_A2); // every ISO alpha-3 code (ADM1 fetch list)
|
||||
|
||||
// ATLAS_ONLY=IRL[,GBR…] rebuilds just those countries' regions: their admin1/<A2> files
|
||||
// are rewritten and their features swapped into admin1.geojson.gz in place, while admin0
|
||||
// and every other country stay as they are. Saves fetching the whole world for one fix.
|
||||
const ONLY = (process.env.ATLAS_ONLY || '')
|
||||
.split(',')
|
||||
.map((s) => s.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
for (const a3 of ONLY) if (!A3_TO_A2[a3]) throw new Error(`ATLAS_ONLY: unknown alpha-3 code ${a3}`);
|
||||
|
||||
// Cache raw downloads so re-runs (e.g. to tune simplification) don't re-fetch ~360 files.
|
||||
const CACHE_DIR = path.join(__dirname, '..', '.atlas-geo-cache', GB_REF);
|
||||
|
||||
async function fetchGeo(url) {
|
||||
const cacheFile = path.join(CACHE_DIR, url.split('/').slice(-1)[0]);
|
||||
async function fetchGeo(url, cacheName = url.split('/').slice(-1)[0]) {
|
||||
const cacheFile = path.join(CACHE_DIR, cacheName);
|
||||
if (fs.existsSync(cacheFile)) {
|
||||
const cached = fs.readFileSync(cacheFile, 'utf8');
|
||||
return cached === '' ? null : JSON.parse(cached);
|
||||
@@ -422,6 +501,30 @@ function normalizeAdm0Feature(f) {
|
||||
};
|
||||
}
|
||||
|
||||
// A country's raw region layer: from its REGION_SOURCES entry when it has one, else the
|
||||
// geoBoundaries level ADM_LEVEL names (ADM1 by default).
|
||||
async function fetchRegions(a3) {
|
||||
const source = REGION_SOURCES[a3];
|
||||
if (!source) return fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1'));
|
||||
|
||||
const geo = await fetchGeo(source.url, source.cacheFile);
|
||||
if (!geo?.features) throw new Error(`${a3}: region source could not be downloaded`);
|
||||
const features = geo.features.map((f) => {
|
||||
const shape = source.toShape(f.properties || {});
|
||||
if (!shape) throw new Error(`${a3}: region source has a feature it cannot place: ${JSON.stringify(f.properties)}`);
|
||||
return { type: 'Feature', properties: shape, geometry: quantizeGeometry(f.geometry, ADM1_DECIMALS) };
|
||||
});
|
||||
const codes = new Set(features.map((f) => f.properties.shapeISO));
|
||||
if (features.length !== source.count || codes.size !== source.count) {
|
||||
throw new Error(
|
||||
`${a3}: expected ${source.count} distinct regions, the source has ${features.length} (${codes.size} distinct)`,
|
||||
);
|
||||
}
|
||||
// By code, so the output does not depend on the order the service happens to answer in.
|
||||
features.sort((a, b) => (a.properties.shapeISO < b.properties.shapeISO ? -1 : 1));
|
||||
return { features: simplifySharedBorders(features, source.tolerance) };
|
||||
}
|
||||
|
||||
function normalizeAdm1(geo, a3, countryName) {
|
||||
if (!geo?.features) return [];
|
||||
const a2 = A3_TO_A2[a3] || null;
|
||||
@@ -477,7 +580,46 @@ function normalizeAdm1(geo, a3, countryName) {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
const write = (name, features) => {
|
||||
const fc = { type: 'FeatureCollection', features };
|
||||
const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 });
|
||||
const file = path.join(OUT_DIR, `${name}.geojson.gz`);
|
||||
fs.writeFileSync(file, gz);
|
||||
console.log(
|
||||
`[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`,
|
||||
);
|
||||
};
|
||||
|
||||
const readShipped = (name) =>
|
||||
JSON.parse(zlib.gunzipSync(fs.readFileSync(path.join(OUT_DIR, `${name}.geojson.gz`))).toString('utf8')).features;
|
||||
|
||||
// The ATLAS_ONLY path. A full build writes the countries in COUNTRIES order, each one's
|
||||
// features in a single run, so a rebuilt country goes back where its run was, or where
|
||||
// it would have been, and admin1.geojson.gz ends up as a full build would leave it.
|
||||
async function rebuildOnly(countries) {
|
||||
console.log(`[atlas-geo] rebuilding the regions of ${countries.join(', ')} only`);
|
||||
fs.mkdirSync(CACHE_DIR, { recursive: true });
|
||||
const a3ToName = Object.fromEntries(readShipped('admin0').map((f) => [f.properties.ADM0_A3, f.properties.NAME]));
|
||||
const order = new Map(COUNTRIES.map((a3, i) => [A3_TO_A2[a3], i]));
|
||||
let bundle = readShipped('admin1');
|
||||
|
||||
for (const a3 of countries) {
|
||||
const a2 = A3_TO_A2[a3];
|
||||
const features = normalizeAdm1(await fetchRegions(a3), a3, a3ToName[a3] || a3);
|
||||
if (!features.length) throw new Error(`${a3}: no regions to write`);
|
||||
const first = bundle.findIndex((f) => f.properties.iso_a2 === a2);
|
||||
const at = first !== -1 ? first : bundle.findIndex((f) => order.get(f.properties.iso_a2) > order.get(a2));
|
||||
const rest = bundle.filter((f) => f.properties.iso_a2 !== a2);
|
||||
const cut = at === -1 ? rest.length : at;
|
||||
bundle = [...rest.slice(0, cut), ...features, ...rest.slice(cut)];
|
||||
write(`admin1/${a2}`, features);
|
||||
}
|
||||
write('admin1', bundle);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (ONLY.length) return rebuildOnly(ONLY);
|
||||
|
||||
console.log(`[atlas-geo] geoBoundaries ref ${GB_REF}; ${COUNTRIES.length} countries`);
|
||||
fs.mkdirSync(OUT_DIR, { recursive: true });
|
||||
fs.mkdirSync(CACHE_DIR, { recursive: true });
|
||||
@@ -496,9 +638,10 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
// ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`).
|
||||
// ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`),
|
||||
// or the country's own source (REGION_SOURCES).
|
||||
console.log('[atlas-geo] downloading ADM1 (regions)…');
|
||||
const adm1Raw = await pool(COUNTRIES, (a3) => fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1')));
|
||||
const adm1Raw = await pool(COUNTRIES, fetchRegions);
|
||||
const adm1Features = [];
|
||||
let withCodes = 0;
|
||||
COUNTRIES.forEach((a3, idx) => {
|
||||
@@ -507,16 +650,6 @@ async function main() {
|
||||
adm1Features.push(...feats);
|
||||
});
|
||||
|
||||
const write = (name, features) => {
|
||||
const fc = { type: 'FeatureCollection', features };
|
||||
const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 });
|
||||
const file = path.join(OUT_DIR, `${name}.geojson.gz`);
|
||||
fs.writeFileSync(file, gz);
|
||||
console.log(
|
||||
`[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`,
|
||||
);
|
||||
};
|
||||
|
||||
write('admin0', adm0Features);
|
||||
// Read only by two shipped migrations (Migration20200101021500/…031500), whose bodies
|
||||
// never change; the server itself reads the per-country files below.
|
||||
|
||||
@@ -36,17 +36,29 @@ fi
|
||||
# which on a large or network-backed one (NFS, CephFS, Longhorn) kept the
|
||||
# container busy for minutes before Node printed its first line (#2918). So only
|
||||
# the dir and what sits directly in it are looked at: a volume that is not
|
||||
# node's yet (a fresh one, or one copied in as root) shows there, and is handed
|
||||
# over once. A volume that refuses chown altogether (NFS with root_squash) is
|
||||
# said once per start instead of being walked file by file; such a pod belongs
|
||||
# under runAsUser 1000, see the Helm page of the wiki. Best effort either way: a
|
||||
# node's yet (a fresh one, or one whose top level was copied in as root) shows
|
||||
# there, and is handed over once. Files copied in as root further down, into a
|
||||
# subdirectory node already owns, do not show and are left alone; the
|
||||
# Troubleshooting page of the wiki has the one-off chown for that.
|
||||
#
|
||||
# Only the owning user has to be node, because that is what Node runs as and
|
||||
# what gosu leaves it with. The group is left to whoever set it: a Kubernetes
|
||||
# fsGroup other than 1000 gives every file its own gid, and taking that back
|
||||
# here would make the kubelet's OnRootMismatch walk the volume again on the
|
||||
# next start, and this script after it, every time. Only a root group (a fresh
|
||||
# volume under plain docker) is handed to node as well.
|
||||
#
|
||||
# A volume that refuses chown altogether (NFS with root_squash) is said once
|
||||
# per start instead of being walked file by file; such a pod belongs under
|
||||
# runAsUser 1000, see the Helm page of the wiki. Best effort either way: a
|
||||
# read-only host filesystem or a locked-down bind mount must keep booting.
|
||||
for dir in /app/data /app/uploads; do
|
||||
[ -d "$dir" ] || continue
|
||||
if [ -n "$(find "$dir" -maxdepth 1 \( ! -user node -o ! -group node \) -print -quit 2>/dev/null)" ]; then
|
||||
if chown node:node "$dir" 2>/dev/null; then
|
||||
if [ -n "$(find "$dir" -maxdepth 1 \( ! -user node -o -group 0 \) -print -quit 2>/dev/null)" ]; then
|
||||
if chown node "$dir" 2>/dev/null; then
|
||||
echo "Correcting ownership of $dir for the node user..."
|
||||
chown -R -h node:node "$dir" 2>/dev/null || true
|
||||
chown -R -h node "$dir" 2>/dev/null || true
|
||||
find "$dir" -group 0 -exec chgrp -h node {} + 2>/dev/null || true
|
||||
else
|
||||
echo "WARNING: $dir cannot be handed to the node user (chown refused); run the container as uid 1000 instead."
|
||||
fi
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"@typescript-eslint/no-non-null-assertion": 263,
|
||||
"@typescript-eslint/no-require-imports": 18,
|
||||
"@typescript-eslint/no-unsafe-function-type": 1,
|
||||
"@typescript-eslint/no-unused-vars": 36,
|
||||
"@typescript-eslint/no-unused-vars": 35,
|
||||
"no-await-in-loop": 556,
|
||||
"no-console": 232,
|
||||
"no-empty": 6
|
||||
@@ -16,8 +16,8 @@
|
||||
"no-misleading-character-class": 1
|
||||
},
|
||||
"tests": {
|
||||
"@typescript-eslint/no-explicit-any": 1555,
|
||||
"@typescript-eslint/no-non-null-assertion": 1755,
|
||||
"@typescript-eslint/no-explicit-any": 1553,
|
||||
"@typescript-eslint/no-non-null-assertion": 1747,
|
||||
"@typescript-eslint/no-require-imports": 25,
|
||||
"@typescript-eslint/no-unsafe-function-type": 7,
|
||||
"@typescript-eslint/no-unused-vars": 96,
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* Simplify the polygons of one country's regions without opening gaps between them.
|
||||
*
|
||||
* geoBoundaries ships a "_simplified" file per country, so the Atlas builder normally
|
||||
* never simplifies anything itself. A region source that only offers full-detail
|
||||
* outlines (Tailte Éireann's counties, #2922) needs it done here, and simplifying each
|
||||
* polygon on its own would not do: two neighbours would each drop different points of
|
||||
* the border they share, leaving slivers that belong to neither county, and a point in
|
||||
* a sliver resolves to no region at all.
|
||||
*
|
||||
* So the rings are cut into arcs the way TopoJSON does it. A vertex is a junction when
|
||||
* it does not have the same two neighbours everywhere it appears: where a shared border
|
||||
* starts or ends, or where three regions meet. Between two junctions both sides hold
|
||||
* the same run of points, and each arc is simplified (Douglas-Peucker) in one canonical
|
||||
* direction, so both sides get the same points back. A ring with no junction at all (an
|
||||
* island, or an enclave shared whole with its surroundings) is rotated to a canonical
|
||||
* start first, for the same reason.
|
||||
*
|
||||
* Expects rings that are already quantized (identical borders carry identical
|
||||
* coordinates) and closed. A ring left with fewer than three distinct points is dropped,
|
||||
* a polygon whose outer ring is dropped goes with its holes, and a geometry with nothing
|
||||
* left becomes null.
|
||||
*/
|
||||
|
||||
const key = (pt) => `${pt[0]},${pt[1]}`;
|
||||
|
||||
// Squared distance from p to the segment a-b.
|
||||
function segmentDistance2(p, a, b) {
|
||||
const dx = b[0] - a[0];
|
||||
const dy = b[1] - a[1];
|
||||
const len2 = dx * dx + dy * dy;
|
||||
let t = len2 === 0 ? 0 : ((p[0] - a[0]) * dx + (p[1] - a[1]) * dy) / len2;
|
||||
t = Math.max(0, Math.min(1, t));
|
||||
const x = a[0] + t * dx - p[0];
|
||||
const y = a[1] + t * dy - p[1];
|
||||
return x * x + y * y;
|
||||
}
|
||||
|
||||
/** Douglas-Peucker over an open polyline; both ends are always kept. */
|
||||
export function douglasPeucker(points, tolerance) {
|
||||
if (points.length <= 2) return points.slice();
|
||||
const keep = new Uint8Array(points.length);
|
||||
keep[0] = 1;
|
||||
keep[points.length - 1] = 1;
|
||||
const tol2 = tolerance * tolerance;
|
||||
const stack = [[0, points.length - 1]];
|
||||
while (stack.length) {
|
||||
const [first, last] = stack.pop();
|
||||
let maxDist = -1;
|
||||
let index = -1;
|
||||
for (let i = first + 1; i < last; i++) {
|
||||
const d = segmentDistance2(points[i], points[first], points[last]);
|
||||
if (d > maxDist) {
|
||||
maxDist = d;
|
||||
index = i;
|
||||
}
|
||||
}
|
||||
if (index !== -1 && maxDist > tol2) {
|
||||
keep[index] = 1;
|
||||
stack.push([first, index], [index, last]);
|
||||
}
|
||||
}
|
||||
return points.filter((_, i) => keep[i]);
|
||||
}
|
||||
|
||||
// An arc simplified in its canonical direction, handed back in the direction it came in.
|
||||
function simplifyArc(arc, tolerance) {
|
||||
const head = key(arc[0]);
|
||||
const tail = key(arc[arc.length - 1]);
|
||||
const reversed = head === tail ? key(arc[1]) > key(arc[arc.length - 2]) : head > tail;
|
||||
const canonical = reversed ? arc.slice().reverse() : arc;
|
||||
const out = douglasPeucker(canonical, tolerance);
|
||||
return reversed ? out.reverse() : out;
|
||||
}
|
||||
|
||||
function polygonsOf(geometry) {
|
||||
if (!geometry) return [];
|
||||
return geometry.type === 'Polygon'
|
||||
? [geometry.coordinates]
|
||||
: geometry.type === 'MultiPolygon'
|
||||
? geometry.coordinates
|
||||
: [];
|
||||
}
|
||||
|
||||
// A closed ring without its repeated closing point.
|
||||
const open = (ring) => (ring.length > 1 && key(ring[0]) === key(ring[ring.length - 1]) ? ring.slice(0, -1) : ring);
|
||||
|
||||
/** Every vertex whose neighbours differ between the rings (or places) it appears in. */
|
||||
function findJunctions(rings) {
|
||||
const neighbours = new Map();
|
||||
const junctions = new Set();
|
||||
for (const ring of rings) {
|
||||
const n = ring.length;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const k = key(ring[i]);
|
||||
const a = key(ring[(i - 1 + n) % n]);
|
||||
const b = key(ring[(i + 1) % n]);
|
||||
const pair = a < b ? `${a}|${b}` : `${b}|${a}`;
|
||||
const seen = neighbours.get(k);
|
||||
if (seen === undefined) neighbours.set(k, pair);
|
||||
else if (seen !== pair) junctions.add(k);
|
||||
}
|
||||
}
|
||||
return junctions;
|
||||
}
|
||||
|
||||
function simplifyRing(ring, junctions, tolerance) {
|
||||
const n = ring.length;
|
||||
const cuts = [];
|
||||
for (let i = 0; i < n; i++) if (junctions.has(key(ring[i]))) cuts.push(i);
|
||||
|
||||
let points;
|
||||
if (cuts.length === 0) {
|
||||
// No junction: start at the smallest vertex so a twin ring starts at the same place.
|
||||
let start = 0;
|
||||
for (let i = 1; i < n; i++) if (key(ring[i]) < key(ring[start])) start = i;
|
||||
const rotated = [...ring.slice(start), ...ring.slice(0, start), ring[start]];
|
||||
points = simplifyArc(rotated, tolerance).slice(0, -1);
|
||||
} else {
|
||||
points = [];
|
||||
for (let c = 0; c < cuts.length; c++) {
|
||||
const from = cuts[c];
|
||||
const to = cuts[(c + 1) % cuts.length];
|
||||
const arc = [];
|
||||
for (let i = from; ; i = (i + 1) % n) {
|
||||
arc.push(ring[i]);
|
||||
if (arc.length > 1 && i === to) break;
|
||||
}
|
||||
points.push(...simplifyArc(arc, tolerance).slice(0, -1));
|
||||
}
|
||||
}
|
||||
if (new Set(points.map(key)).size < 3) return null;
|
||||
return [...points, points[0]];
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify every feature's (Multi)Polygon with the given tolerance (in coordinate
|
||||
* units), keeping the borders that features share identical on both sides. Returns new
|
||||
* features; properties are passed through untouched.
|
||||
*/
|
||||
export function simplifySharedBorders(features, tolerance) {
|
||||
const rings = features.flatMap((f) => polygonsOf(f.geometry).flatMap((poly) => poly.map(open)));
|
||||
const junctions = findJunctions(rings);
|
||||
return features.map((f) => {
|
||||
const polygons = polygonsOf(f.geometry)
|
||||
.map((poly) => {
|
||||
const [outer, ...holes] = poly.map((ring) => simplifyRing(open(ring), junctions, tolerance));
|
||||
return outer ? [outer, ...holes.filter(Boolean)] : null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
const geometry =
|
||||
polygons.length === 0
|
||||
? null
|
||||
: polygons.length === 1
|
||||
? { type: 'Polygon', coordinates: polygons[0] }
|
||||
: { type: 'MultiPolygon', coordinates: polygons };
|
||||
return { ...f, geometry };
|
||||
});
|
||||
}
|
||||
@@ -317,6 +317,9 @@ async function main() {
|
||||
// service_key is the installation's key towards the liketrek services.
|
||||
// Missed here it reads back as unusable, the installation registers anew and
|
||||
// loses its id, and a revoked key would come back as a fresh one.
|
||||
//
|
||||
// The mail import's IMAP password and webhook secret: missed, the box stops
|
||||
// being read and every signed mail is refused until an admin types them again.
|
||||
for (const key of [
|
||||
'oidc_client_secret',
|
||||
'smtp_pass',
|
||||
@@ -327,6 +330,8 @@ async function main() {
|
||||
'amap_api_key',
|
||||
'web_push_vapid_private_key',
|
||||
'service_key',
|
||||
'mail_ingest_imap_pass',
|
||||
'mail_ingest_webhook_secret',
|
||||
]) {
|
||||
const row = db.prepare('SELECT value FROM app_settings WHERE key = ?').get(key) as { value: string } | undefined;
|
||||
if (!row?.value) continue;
|
||||
@@ -567,6 +572,28 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
// --- mail_ingest: forwarding address tokens and the stored mails ---
|
||||
// A person sees their address again through its token_enc; a forwarded mail
|
||||
// waits for its reading, and for the inbox's ways out, as raw_eml_enc. Missed,
|
||||
// the addresses still take mail (they are found by hash) but show as unset,
|
||||
// and every stored mail reads as unreadable.
|
||||
for (const [table, column] of [
|
||||
['mail_ingest_addresses', 'token_enc'],
|
||||
['mail_ingest_messages', 'raw_eml_enc'],
|
||||
] as const) {
|
||||
if (!tableExists(table)) continue;
|
||||
const rows = db.prepare(`SELECT id, ${column} AS value FROM ${table} WHERE ${column} IS NOT NULL`).all() as {
|
||||
id: number;
|
||||
value: string;
|
||||
}[];
|
||||
for (const row of rows) {
|
||||
const newVal = migrateApiKeyValue(row.value, `${table}[${row.id}].${column}`);
|
||||
if (newVal !== null) {
|
||||
db.prepare(`UPDATE ${table} SET ${column} = ? WHERE id = ?`).run(newVal, row.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- addons: the AI Parsing addon's instance-wide API key ---
|
||||
// Stored encrypted as `apiKey` inside the addon's JSON config
|
||||
// (llm-parse/llm-config.ts). Left out, LLM parsing quietly runs without a
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"src/db/repositories/RoadtripDayTracks.repository.ts": 1,
|
||||
"src/db/repositories/RoadtripPreferences.repository.ts": 1,
|
||||
"src/db/repositories/RoadtripVias.repository.ts": 6,
|
||||
"src/db/repositories/Tags.repository.ts": 3,
|
||||
"src/db/repositories/Tags.repository.ts": 2,
|
||||
"src/db/repositories/TodoItems.repository.ts": 2,
|
||||
"src/db/repositories/TripDocumentLinks.repository.ts": 5,
|
||||
"src/db/repositories/TripInviteTokens.repository.ts": 4,
|
||||
|
||||
@@ -40,6 +40,7 @@
|
||||
"DocumentSyncItems": "doc-sync",
|
||||
"FerrataTiles": "ferratas",
|
||||
"FileLinks": "files",
|
||||
"FlightLiveStates": "flight-live",
|
||||
"GoogleApiUsage": "google-quota",
|
||||
"GooglePlacePhotoMeta": "place-photos",
|
||||
"HiddenCountries": "atlas",
|
||||
@@ -58,6 +59,11 @@
|
||||
"JourneyShareTokens": "journey",
|
||||
"JourneyTrips": "journey",
|
||||
"Journeys": "journey",
|
||||
"MailIngestAddresses": "mail-ingest",
|
||||
"MailIngestCandidates": "mail-ingest",
|
||||
"MailIngestCursor": "mail-ingest",
|
||||
"MailIngestMessages": "mail-ingest",
|
||||
"MailIngestPrefs": "mail-ingest",
|
||||
"McpTokens": "tokens",
|
||||
"NotificationChannelPreferences": "notifications",
|
||||
"Notifications": "notifications",
|
||||
|
||||
@@ -3,18 +3,18 @@
|
||||
"scripts/generate-entities.ts": 2259,
|
||||
"src/db/repositories/Places.repository.ts": 1435,
|
||||
"src/db/repositories/Reservations.repository.ts": 1551,
|
||||
"src/db/repositories/Trips.repository.ts": 1121,
|
||||
"src/db/repositories/Trips.repository.ts": 1120,
|
||||
"src/db/repositories/Users.repository.ts": 1445,
|
||||
"src/nest/atlas/atlas.service.ts": 1177,
|
||||
"src/nest/atlas/atlas.service.ts": 1174,
|
||||
"src/nest/auth/auth.service.ts": 1143,
|
||||
"src/nest/budget/budget.service.ts": 1612,
|
||||
"src/nest/collections/collections.service.ts": 1426,
|
||||
"src/nest/budget/budget.service.ts": 1607,
|
||||
"src/nest/collections/collections.service.ts": 1414,
|
||||
"src/nest/doc-sync/doc-sync.service.ts": 1060,
|
||||
"src/nest/doc-sync/providers/synology-drive.client.ts": 1160,
|
||||
"src/nest/journey/journey-domain.service.ts": 1659,
|
||||
"src/nest/places/places.service.ts": 1546,
|
||||
"src/nest/plugins/plugin-runtime.service.ts": 1460,
|
||||
"src/nest/plugins/runtime/plugin-sdk.ts": 1093,
|
||||
"src/nest/reservations/reservations.service.ts": 1237,
|
||||
"src/nest/vacay/vacay.service.ts": 1409
|
||||
"src/nest/reservations/reservations.service.ts": 1168,
|
||||
"src/nest/vacay/vacay.service.ts": 1393
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
"tests/unit/nest/auth.controller.test.ts": 1,
|
||||
"tests/unit/nest/auth.service.test.ts": 9,
|
||||
"tests/unit/nest/backup.controller.test.ts": 1,
|
||||
"tests/unit/nest/booking-import/booking-import.service.test.ts": 3,
|
||||
"tests/unit/nest/booking-import/booking-import.service.test.ts": 1,
|
||||
"tests/unit/nest/booking-import/import-jobs.service.test.ts": 1,
|
||||
"tests/unit/nest/booking-import/kitinerary-extractor.service.test.ts": 1,
|
||||
"tests/unit/nest/budget.service.calc.test.ts": 1,
|
||||
@@ -97,7 +97,7 @@
|
||||
"tests/unit/nest/memories/trek-photo-cache.job.test.ts": 1,
|
||||
"tests/unit/nest/notification-preferences.service.test.ts": 4,
|
||||
"tests/unit/nest/notification-transports.test.ts": 3,
|
||||
"tests/unit/nest/notifications/storage-health-notifier.service.test.ts": 2,
|
||||
"tests/unit/nest/notifications/storage-health-notifier.service.test.ts": 1,
|
||||
"tests/unit/nest/notifications/vapid-keys.service.test.ts": 2,
|
||||
"tests/unit/nest/notifications/web-push.service.test.ts": 1,
|
||||
"tests/unit/nest/oauth.controller.test.ts": 1,
|
||||
|
||||
@@ -28,7 +28,6 @@
|
||||
"src/nest/integrations/airtrail-link.service.ts#AirtrailLinkService.pushReservationToAirtrail": 2,
|
||||
"src/nest/integrations/airtrail-sync.service.ts#AirtrailSyncService.runAirtrailSync": 2,
|
||||
"src/nest/integrations/airtrail-sync.service.ts#AirtrailSyncService.syncOwner": 8,
|
||||
"src/nest/integrations/airtrail.service.ts#AirtrailService.saveSettings": 2,
|
||||
"src/nest/integrations/dawarich-sync.service.ts#DawarichSyncService.runSync": 2,
|
||||
"src/nest/integrations/dawarich-sync.service.ts#DawarichSyncService.syncTripWindow": 3,
|
||||
"src/nest/integrations/dawarich-sync.service.ts#DawarichSyncService.syncUserOnce": 3,
|
||||
|
||||
Reference in New Issue
Block a user