mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-18 21:16:47 +00:00
89ad4ec56a
The PBF reader streams the extract and refuses a varint longer than ten bytes; the block offsets are read as the plain int64 the format defines. Region reads queue without loose promises: a region queued while another is read is still picked up, removing a region calls its read off, a shutdown stops the read under way and the next boot queues it again and clears the download folder. The admin list shows how far a read has come. Geofabrik is asked only on its own servers, redirects included, and an outage of its list answers 502 instead of an unknown region. Nested ids such as us/texas are accepted. A region being read again or after a failed read still counts with the data it holds, a region without a boundary spans its objects, and an object two overlapping regions hold is answered once. Stored geometry, tags and boundaries go through the JSON column declarations. An obstacle is related to the route both ways, so a long weir passed in its middle and a route inside an area count. Tests cover the reader with extracts written in the test, the tag rules, the extraction and lock grouping, the geo helpers, the Geofabrik client, the region queue, the read side, both controllers, the job, the repositories and the HTTP routes; the domain gets its coverage entry.
189 lines
7.1 KiB
TypeScript
189 lines
7.1 KiB
TypeScript
/**
|
|
* The two-pass extraction of PAD-05A over extracts written in the test: the
|
|
* objects kept with their own geometry (a weir across a river stays a line),
|
|
* their tag hash, and the parts of one lock grouped while separate locks
|
|
* stay apart.
|
|
*/
|
|
import {
|
|
extractPaddleFeatures,
|
|
groupLockParts,
|
|
type ExtractedFeature,
|
|
} from '../../../../src/nest/paddle-obstacles/paddle-extract';
|
|
import { pbfFile, type FixtureNode } from '../../../helpers/osm-pbf-fixture';
|
|
|
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
|
|
let dir: string;
|
|
let n = 0;
|
|
|
|
beforeAll(async () => {
|
|
dir = await mkdtemp(join(tmpdir(), 'trek-paddle-extract-'));
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await rm(dir, { recursive: true, force: true });
|
|
});
|
|
|
|
async function extract(blocks: Parameters<typeof pbfFile>[0], timestamp?: number) {
|
|
const path = join(dir, `region-${n++}.osm.pbf`);
|
|
await writeFile(path, pbfFile(blocks, { timestamp }));
|
|
const progress: number[] = [];
|
|
const result = await extractPaddleFeatures(path, (fraction) => progress.push(fraction));
|
|
return { ...result, progress };
|
|
}
|
|
|
|
const byRef = (features: ExtractedFeature[], ref: string) => features.find((feature) => feature.osm_ref === ref);
|
|
|
|
/** The Regen at Lindenmühle, Regenstauf: the weir way 29256405 drawn across the river. */
|
|
const weirNodes: FixtureNode[] = [
|
|
{ id: 1, lat: 49.1231, lon: 12.1301 },
|
|
{ id: 2, lat: 49.1236, lon: 12.1309 },
|
|
{ id: 3, lat: 49.124, lon: 12.1315 },
|
|
];
|
|
|
|
describe('extractPaddleFeatures', () => {
|
|
it('PEXT-001: keeps a weir way as a line with its own geometry, box and kept tags', async () => {
|
|
const { features, timestamp, progress } = await extract(
|
|
[
|
|
{ dense: weirNodes },
|
|
{
|
|
ways: [
|
|
{
|
|
id: 29256405,
|
|
refs: [1, 2, 3],
|
|
tags: { waterway: 'weir', name: 'Wehr Lindenmühle', whitewater: 'hazard', source: 'bing' },
|
|
},
|
|
],
|
|
},
|
|
],
|
|
1_760_000_000,
|
|
);
|
|
|
|
expect(timestamp).toBe(1_760_000_000);
|
|
const weir = byRef(features, 'way/29256405');
|
|
expect(weir).toMatchObject({
|
|
kind: 'weir',
|
|
name: 'Wehr Lindenmühle',
|
|
tags: { waterway: 'weir', name: 'Wehr Lindenmühle', whitewater: 'hazard' },
|
|
group_key: null,
|
|
});
|
|
expect(weir?.geometry.type).toBe('LineString');
|
|
expect(weir?.geometry.coordinates).toHaveLength(3);
|
|
expect(weir?.min_lat).toBeCloseTo(49.1231, 6);
|
|
expect(weir?.max_lat).toBeCloseTo(49.124, 6);
|
|
expect(weir?.min_lng).toBeCloseTo(12.1301, 6);
|
|
expect(weir?.max_lng).toBeCloseTo(12.1315, 6);
|
|
expect(weir?.tag_hash).toMatch(/^[0-9a-f]{16}$/);
|
|
expect(progress.at(-1)).toBe(1);
|
|
expect([...progress].sort((a, b) => a - b)).toEqual(progress);
|
|
});
|
|
|
|
it('PEXT-002: keeps tagged nodes that are paddle objects as points, and leaves the rest', async () => {
|
|
const { features, progress } = await extract([
|
|
{
|
|
dense: [
|
|
{ id: 10, lat: 53.27, lon: 12.81, tags: { canoe: 'put_in', name: 'Einsetzstelle Mirow' } },
|
|
{ id: 11, lat: 53.28, lon: 12.82, tags: { amenity: 'bench' } },
|
|
{ id: 12, lat: 53.29, lon: 12.83 },
|
|
],
|
|
},
|
|
]);
|
|
expect(features.map((feature) => feature.osm_ref)).toEqual(['node/10']);
|
|
expect(features[0]).toMatchObject({ kind: 'put_in', geometry: { type: 'Point' } });
|
|
expect(features[0]?.min_lat).toBe(features[0]?.max_lat);
|
|
// No way to look up: the second pass is skipped and the progress jumps to the end.
|
|
expect(progress.at(-1)).toBe(1);
|
|
});
|
|
|
|
it('PEXT-003: a closed way is an area, a way with fewer than two known nodes is dropped', async () => {
|
|
const { features } = await extract([
|
|
{
|
|
dense: [
|
|
{ id: 1, lat: 0, lon: 0 },
|
|
{ id: 2, lat: 0, lon: 0.001 },
|
|
{ id: 3, lat: 0.001, lon: 0.001 },
|
|
],
|
|
},
|
|
{
|
|
ways: [
|
|
{ id: 100, refs: [1, 2, 3, 1], tags: { lock: 'yes', waterway: 'canal' } },
|
|
{ id: 101, refs: [1, 99], tags: { waterway: 'dam' } },
|
|
],
|
|
},
|
|
]);
|
|
expect(byRef(features, 'way/100')?.geometry.type).toBe('Polygon');
|
|
expect(byRef(features, 'way/101')).toBeUndefined();
|
|
});
|
|
|
|
it('PEXT-004: thins a very long way to every n-th node and keeps both of its ends', async () => {
|
|
const nodes = Array.from({ length: 4501 }, (_, i) => ({ id: i + 1, lat: 50 + i * 1e-5, lon: 8 }));
|
|
const { features } = await extract([
|
|
{ dense: nodes },
|
|
{ ways: [{ id: 7, refs: nodes.map((node) => node.id), tags: { waterway: 'river', canoe: 'yes' } }] },
|
|
]);
|
|
const line = byRef(features, 'way/7');
|
|
expect(line?.kind).toBe('access');
|
|
const coordinates = line?.geometry.type === 'LineString' ? line.geometry.coordinates : [];
|
|
expect(coordinates.length).toBeLessThanOrEqual(2001);
|
|
expect(coordinates[0]?.[1]).toBeCloseTo(50, 6);
|
|
expect(coordinates.at(-1)?.[1]).toBeCloseTo(50 + 4500 * 1e-5, 6);
|
|
});
|
|
|
|
it('PEXT-005: the hash changes with a kept tag and with the geometry, not with a dropped tag', async () => {
|
|
const weir = (tags: Record<string, string>, lat = 49.1) =>
|
|
extract([{ dense: [{ id: 5, lat, lon: 12, tags: { waterway: 'weir', ...tags } }] }]).then(
|
|
(result) => result.features[0]?.tag_hash,
|
|
);
|
|
const base = await weir({ name: 'A' });
|
|
expect(await weir({ name: 'A' })).toBe(base);
|
|
expect(await weir({ name: 'A', source: 'survey' })).toBe(base);
|
|
expect(await weir({ name: 'B' })).not.toBe(base);
|
|
expect(await weir({ name: 'A', canoe: 'no' })).not.toBe(base);
|
|
expect(await weir({ name: 'A' }, 49.2)).not.toBe(base);
|
|
});
|
|
});
|
|
|
|
describe('groupLockParts', () => {
|
|
const part = (ref: string, kind: ExtractedFeature['kind'], lat: number, lng: number): ExtractedFeature => ({
|
|
osm_ref: ref,
|
|
kind,
|
|
name: null,
|
|
geometry: { type: 'Point', coordinates: [lng, lat] },
|
|
tags: {},
|
|
tag_hash: ref,
|
|
group_key: null,
|
|
min_lat: lat,
|
|
max_lat: lat,
|
|
min_lng: lng,
|
|
max_lng: lng,
|
|
});
|
|
|
|
it('PEXT-006: the chamber and the gates of one lock share a key; a lock 2 km on stays apart', () => {
|
|
const chamber = part('way/30', 'lock', 53.3, 13.0);
|
|
const upper = part('node/31', 'lock_gate', 53.3, 13.0012);
|
|
const lower = part('node/32', 'lock_gate', 53.3, 12.9988);
|
|
const other = part('way/40', 'lock', 53.3, 13.03);
|
|
const weir = part('way/50', 'weir', 53.3, 13.0001);
|
|
const features = [chamber, upper, lower, other, weir];
|
|
|
|
groupLockParts(features);
|
|
|
|
expect(chamber.group_key).toBe('lock:node/31');
|
|
expect(upper.group_key).toBe('lock:node/31');
|
|
expect(lower.group_key).toBe('lock:node/31');
|
|
expect(other.group_key).toBeNull();
|
|
expect(weir.group_key).toBeNull();
|
|
});
|
|
|
|
it('PEXT-007: parts chained within reach of each other form one lock', () => {
|
|
const a = part('node/1', 'lock_gate', 0, 0);
|
|
const b = part('node/2', 'lock_gate', 0, 0.002);
|
|
const c = part('node/3', 'lock_gate', 0, 0.004);
|
|
groupLockParts([c, a, b]);
|
|
expect(new Set([a.group_key, b.group_key, c.group_key])).toEqual(new Set(['lock:node/1']));
|
|
});
|
|
});
|