k1nq/TREK
1
0
Fork 0
mirror of https://github.com/mauriceboe/TREK.git synced 2026-10-23 15:36:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: format server and shared with prettier

Browse Source 
A mechanical prettier --write over server/src, server/tests, server/scripts
and shared/src with the workspaces' .prettierrc; nothing but layout and
import order changes, and every file keeps the same set of imports. shared
is formatted throughout and its format baseline is empty. The server
baseline keeps the 35 files that Prettier would grow past their lint:size
limit (they need a split first, which is not a formatting change), and the
generated entity files stay as their generator writes them.
This commit is contained in:
Maurice
2026-10-10 21:23:37 +02:00
parent f4e518b2df
commit 724c88c201
1978 changed files with 78448 additions and 39347 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/scripts/build-atlas-geo.mjs
+402 -137
View File
@@ -13,21 +13,20 @@
// FeatureCollections that the server serves at runtime (no network at boot).
//
// geoBoundaries: CC BY 4.0 — https://www.geoboundaries.org/ (attribution required).
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import zlib from 'node:zlib';
import fs from 'node:fs'
import path from 'node:path'
import zlib from 'node:zlib'
import { fileURLToPath } from 'node:url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const OUT_DIR = path.join(__dirname, '..', 'assets', 'atlas')
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUT_DIR = path.join(__dirname, '..', 'assets', 'atlas');
// Pinned geoBoundaries revision (override with GB_REF=<sha|branch|tag>). The LFS media
// endpoint resolves a commit SHA, branch, or tag in the <ref> path segment.
const GB_REF = process.env.GB_REF || '5c25134028196d43ce97b5071934fd0cfc92f09f'
const GB_REF = process.env.GB_REF || '5c25134028196d43ce97b5071934fd0cfc92f09f';
const MEDIA = (a3, level) =>
`https://media.githubusercontent.com/media/wmgeolab/geoBoundaries/${GB_REF}` +
`/releaseData/gbOpen/${a3}/${level}/geoBoundaries-${a3}-${level}_simplified.geojson`
`/releaseData/gbOpen/${a3}/${level}/geoBoundaries-${a3}-${level}_simplified.geojson`;
// Country borders come from CGAZ (the Comprehensive Global Administrative Zones composite)
// rather than per-country gbOpen ADM0: CGAZ is gap-filled, so it includes territories
@@ -36,10 +35,10 @@ const MEDIA = (a3, level) =>
// irrelevant. (gbOpen ADM0 maxes Norway at 71°N and has no Svalbard at all.)
const CGAZ_ADM0 =
`https://media.githubusercontent.com/media/wmgeolab/geoBoundaries/${GB_REF}` +
`/releaseData/CGAZ/geoBoundariesCGAZ_ADM0.geojson`
`/releaseData/CGAZ/geoBoundariesCGAZ_ADM0.geojson`;
const CONCURRENCY = 8
const RETRIES = 3
const CONCURRENCY = 8;
const RETRIES = 3;
// Complete ISO-3166-1 alpha-3 → alpha-2 map (source: lukes/ISO-3166-Countries-with-
// Regional-Codes, plus user-assigned XKX/XK for Kosovo, which geoBoundaries ships but
@@ -47,7 +46,258 @@ const RETRIES = 3
// code; missing ones 404 and are skipped) and stamps `iso_a2`/`ISO_A2` (geoBoundaries
// keys by alpha-3 `shapeGroup`). A complete map — not the client's curated ~180 — is
// what restores the dropped territories (Greenland, Falklands, French Guiana, …).
const A3_TO_A2 = {"ABW":"AW", "AFG":"AF", "AGO":"AO", "AIA":"AI", "ALA":"AX", "ALB":"AL", "AND":"AD", "ARE":"AE", "ARG":"AR", "ARM":"AM", "ASM":"AS", "ATA":"AQ", "ATF":"TF", "ATG":"AG", "AUS":"AU", "AUT":"AT", "AZE":"AZ", "BDI":"BI", "BEL":"BE", "BEN":"BJ", "BES":"BQ", "BFA":"BF", "BGD":"BD", "BGR":"BG", "BHR":"BH", "BHS":"BS", "BIH":"BA", "BLM":"BL", "BLR":"BY", "BLZ":"BZ", "BMU":"BM", "BOL":"BO", "BRA":"BR", "BRB":"BB", "BRN":"BN", "BTN":"BT", "BVT":"BV", "BWA":"BW", "CAF":"CF", "CAN":"CA", "CCK":"CC", "CHE":"CH", "CHL":"CL", "CHN":"CN", "CIV":"CI", "CMR":"CM", "COD":"CD", "COG":"CG", "COK":"CK", "COL":"CO", "COM":"KM", "CPV":"CV", "CRI":"CR", "CUB":"CU", "CUW":"CW", "CXR":"CX", "CYM":"KY", "CYP":"CY", "CZE":"CZ", "DEU":"DE", "DJI":"DJ", "DMA":"DM", "DNK":"DK", "DOM":"DO", "DZA":"DZ", "ECU":"EC", "EGY":"EG", "ERI":"ER", "ESH":"EH", "ESP":"ES", "EST":"EE", "ETH":"ET", "FIN":"FI", "FJI":"FJ", "FLK":"FK", "FRA":"FR", "FRO":"FO", "FSM":"FM", "GAB":"GA", "GBR":"GB", "GEO":"GE", "GGY":"GG", "GHA":"GH", "GIB":"GI", "GIN":"GN", "GLP":"GP", "GMB":"GM", "GNB":"GW", "GNQ":"GQ", "GRC":"GR", "GRD":"GD", "GRL":"GL", "GTM":"GT", "GUF":"GF", "GUM":"GU", "GUY":"GY", "HKG":"HK", "HMD":"HM", "HND":"HN", "HRV":"HR", "HTI":"HT", "HUN":"HU", "IDN":"ID", "IMN":"IM", "IND":"IN", "IOT":"IO", "IRL":"IE", "IRN":"IR", "IRQ":"IQ", "ISL":"IS", "ISR":"IL", "ITA":"IT", "JAM":"JM", "JEY":"JE", "JOR":"JO", "JPN":"JP", "KAZ":"KZ", "KEN":"KE", "KGZ":"KG", "KHM":"KH", "KIR":"KI", "KNA":"KN", "KOR":"KR", "KWT":"KW", "LAO":"LA", "LBN":"LB", "LBR":"LR", "LBY":"LY", "LCA":"LC", "LIE":"LI", "LKA":"LK", "LSO":"LS", "LTU":"LT", "LUX":"LU", "LVA":"LV", "MAC":"MO", "MAF":"MF", "MAR":"MA", "MCO":"MC", "MDA":"MD", "MDG":"MG", "MDV":"MV", "MEX":"MX", "MHL":"MH", "MKD":"MK", "MLI":"ML", "MLT":"MT", "MMR":"MM", "MNE":"ME", "MNG":"MN", "MNP":"MP", "MOZ":"MZ", "MRT":"MR", "MSR":"MS", "MTQ":"MQ", "MUS":"MU", "MWI":"MW", "MYS":"MY", "MYT":"YT", "NAM":"NA", "NCL":"NC", "NER":"NE", "NFK":"NF", "NGA":"NG", "NIC":"NI", "NIU":"NU", "NLD":"NL", "NOR":"NO", "NPL":"NP", "NRU":"NR", "NZL":"NZ", "OMN":"OM", "PAK":"PK", "PAN":"PA", "PCN":"PN", "PER":"PE", "PHL":"PH", "PLW":"PW", "PNG":"PG", "POL":"PL", "PRI":"PR", "PRK":"KP", "PRT":"PT", "PRY":"PY", "PSE":"PS", "PYF":"PF", "QAT":"QA", "REU":"RE", "ROU":"RO", "RUS":"RU", "RWA":"RW", "SAU":"SA", "SDN":"SD", "SEN":"SN", "SGP":"SG", "SGS":"GS", "SHN":"SH", "SJM":"SJ", "SLB":"SB", "SLE":"SL", "SLV":"SV", "SMR":"SM", "SOM":"SO", "SPM":"PM", "SRB":"RS", "SSD":"SS", "STP":"ST", "SUR":"SR", "SVK":"SK", "SVN":"SI", "SWE":"SE", "SWZ":"SZ", "SXM":"SX", "SYC":"SC", "SYR":"SY", "TCA":"TC", "TCD":"TD", "TGO":"TG", "THA":"TH", "TJK":"TJ", "TKL":"TK", "TKM":"TM", "TLS":"TL", "TON":"TO", "TTO":"TT", "TUN":"TN", "TUR":"TR", "TUV":"TV", "TWN":"TW", "TZA":"TZ", "UGA":"UG", "UKR":"UA", "UMI":"UM", "URY":"UY", "USA":"US", "UZB":"UZ", "VAT":"VA", "VCT":"VC", "VEN":"VE", "VGB":"VG", "VIR":"VI", "VNM":"VN", "VUT":"VU", "WLF":"WF", "WSM":"WS", "XKX":"XK", "YEM":"YE", "ZAF":"ZA", "ZMB":"ZM", "ZWE":"ZW"}
const A3_TO_A2 = {
ABW: 'AW',
AFG: 'AF',
AGO: 'AO',
AIA: 'AI',
ALA: 'AX',
ALB: 'AL',
AND: 'AD',
ARE: 'AE',
ARG: 'AR',
ARM: 'AM',
ASM: 'AS',
ATA: 'AQ',
ATF: 'TF',
ATG: 'AG',
AUS: 'AU',
AUT: 'AT',
AZE: 'AZ',
BDI: 'BI',
BEL: 'BE',
BEN: 'BJ',
BES: 'BQ',
BFA: 'BF',
BGD: 'BD',
BGR: 'BG',
BHR: 'BH',
BHS: 'BS',
BIH: 'BA',
BLM: 'BL',
BLR: 'BY',
BLZ: 'BZ',
BMU: 'BM',
BOL: 'BO',
BRA: 'BR',
BRB: 'BB',
BRN: 'BN',
BTN: 'BT',
BVT: 'BV',
BWA: 'BW',
CAF: 'CF',
CAN: 'CA',
CCK: 'CC',
CHE: 'CH',
CHL: 'CL',
CHN: 'CN',
CIV: 'CI',
CMR: 'CM',
COD: 'CD',
COG: 'CG',
COK: 'CK',
COL: 'CO',
COM: 'KM',
CPV: 'CV',
CRI: 'CR',
CUB: 'CU',
CUW: 'CW',
CXR: 'CX',
CYM: 'KY',
CYP: 'CY',
CZE: 'CZ',
DEU: 'DE',
DJI: 'DJ',
DMA: 'DM',
DNK: 'DK',
DOM: 'DO',
DZA: 'DZ',
ECU: 'EC',
EGY: 'EG',
ERI: 'ER',
ESH: 'EH',
ESP: 'ES',
EST: 'EE',
ETH: 'ET',
FIN: 'FI',
FJI: 'FJ',
FLK: 'FK',
FRA: 'FR',
FRO: 'FO',
FSM: 'FM',
GAB: 'GA',
GBR: 'GB',
GEO: 'GE',
GGY: 'GG',
GHA: 'GH',
GIB: 'GI',
GIN: 'GN',
GLP: 'GP',
GMB: 'GM',
GNB: 'GW',
GNQ: 'GQ',
GRC: 'GR',
GRD: 'GD',
GRL: 'GL',
GTM: 'GT',
GUF: 'GF',
GUM: 'GU',
GUY: 'GY',
HKG: 'HK',
HMD: 'HM',
HND: 'HN',
HRV: 'HR',
HTI: 'HT',
HUN: 'HU',
IDN: 'ID',
IMN: 'IM',
IND: 'IN',
IOT: 'IO',
IRL: 'IE',
IRN: 'IR',
IRQ: 'IQ',
ISL: 'IS',
ISR: 'IL',
ITA: 'IT',
JAM: 'JM',
JEY: 'JE',
JOR: 'JO',
JPN: 'JP',
KAZ: 'KZ',
KEN: 'KE',
KGZ: 'KG',
KHM: 'KH',
KIR: 'KI',
KNA: 'KN',
KOR: 'KR',
KWT: 'KW',
LAO: 'LA',
LBN: 'LB',
LBR: 'LR',
LBY: 'LY',
LCA: 'LC',
LIE: 'LI',
LKA: 'LK',
LSO: 'LS',
LTU: 'LT',
LUX: 'LU',
LVA: 'LV',
MAC: 'MO',
MAF: 'MF',
MAR: 'MA',
MCO: 'MC',
MDA: 'MD',
MDG: 'MG',
MDV: 'MV',
MEX: 'MX',
MHL: 'MH',
MKD: 'MK',
MLI: 'ML',
MLT: 'MT',
MMR: 'MM',
MNE: 'ME',
MNG: 'MN',
MNP: 'MP',
MOZ: 'MZ',
MRT: 'MR',
MSR: 'MS',
MTQ: 'MQ',
MUS: 'MU',
MWI: 'MW',
MYS: 'MY',
MYT: 'YT',
NAM: 'NA',
NCL: 'NC',
NER: 'NE',
NFK: 'NF',
NGA: 'NG',
NIC: 'NI',
NIU: 'NU',
NLD: 'NL',
NOR: 'NO',
NPL: 'NP',
NRU: 'NR',
NZL: 'NZ',
OMN: 'OM',
PAK: 'PK',
PAN: 'PA',
PCN: 'PN',
PER: 'PE',
PHL: 'PH',
PLW: 'PW',
PNG: 'PG',
POL: 'PL',
PRI: 'PR',
PRK: 'KP',
PRT: 'PT',
PRY: 'PY',
PSE: 'PS',
PYF: 'PF',
QAT: 'QA',
REU: 'RE',
ROU: 'RO',
RUS: 'RU',
RWA: 'RW',
SAU: 'SA',
SDN: 'SD',
SEN: 'SN',
SGP: 'SG',
SGS: 'GS',
SHN: 'SH',
SJM: 'SJ',
SLB: 'SB',
SLE: 'SL',
SLV: 'SV',
SMR: 'SM',
SOM: 'SO',
SPM: 'PM',
SRB: 'RS',
SSD: 'SS',
STP: 'ST',
SUR: 'SR',
SVK: 'SK',
SVN: 'SI',
SWE: 'SE',
SWZ: 'SZ',
SXM: 'SX',
SYC: 'SC',
SYR: 'SY',
TCA: 'TC',
TCD: 'TD',
TGO: 'TG',
THA: 'TH',
TJK: 'TJ',
TKL: 'TK',
TKM: 'TM',
TLS: 'TL',
TON: 'TO',
TTO: 'TT',
TUN: 'TN',
TUR: 'TR',
TUV: 'TV',
TWN: 'TW',
TZA: 'TZ',
UGA: 'UG',
UKR: 'UA',
UMI: 'UM',
URY: 'UY',
USA: 'US',
UZB: 'UZ',
VAT: 'VA',
VCT: 'VC',
VEN: 'VE',
VGB: 'VG',
VIR: 'VI',
VNM: 'VN',
VUT: 'VU',
WLF: 'WF',
WSM: 'WS',
XKX: 'XK',
YEM: 'YE',
ZAF: 'ZA',
ZMB: 'ZM',
ZWE: 'ZW',
};
/**
* Countries whose sub-national level is not the one everybody else's ADM1 is.
@@ -63,52 +313,55 @@ const A3_TO_A2 = {"ABW":"AW", "AFG":"AF", "AGO":"AO", "AIA":"AI", "ALA":"AX", "A
* bundle several times over for no gain: for most countries ADM1 already is the
* level people think in.
*/
const ADM_LEVEL = { GBR: 'ADM2' }
const ADM_LEVEL = { GBR: 'ADM2' };
const COUNTRIES = Object.keys(A3_TO_A2) // every ISO alpha-3 code (ADM1 fetch list)
const COUNTRIES = Object.keys(A3_TO_A2); // every ISO alpha-3 code (ADM1 fetch list)
// Cache raw downloads so re-runs (e.g. to tune simplification) don't re-fetch ~360 files.
const CACHE_DIR = path.join(__dirname, '..', '.atlas-geo-cache', GB_REF)
const CACHE_DIR = path.join(__dirname, '..', '.atlas-geo-cache', GB_REF);
async function fetchGeo(url) {
const cacheFile = path.join(CACHE_DIR, url.split('/').slice(-1)[0])
const cacheFile = path.join(CACHE_DIR, url.split('/').slice(-1)[0]);
if (fs.existsSync(cacheFile)) {
const cached = fs.readFileSync(cacheFile, 'utf8')
return cached === '' ? null : JSON.parse(cached)
const cached = fs.readFileSync(cacheFile, 'utf8');
return cached === '' ? null : JSON.parse(cached);
}
for (let attempt = 1; attempt <= RETRIES; attempt++) {
try {
const res = await fetch(url, { headers: { 'User-Agent': 'TREK atlas builder' } })
if (res.status === 404) { fs.writeFileSync(cacheFile, ''); return null } // no file — skip
if (!res.ok) throw new Error(`HTTP ${res.status}`)
const text = await res.text()
if (text.startsWith('version https://git-lfs')) throw new Error('got LFS pointer, not content')
const parsed = JSON.parse(text)
fs.writeFileSync(cacheFile, text)
return parsed
const res = await fetch(url, { headers: { 'User-Agent': 'TREK atlas builder' } });
if (res.status === 404) {
fs.writeFileSync(cacheFile, '');
return null;
} // no file — skip
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const text = await res.text();
if (text.startsWith('version https://git-lfs')) throw new Error('got LFS pointer, not content');
const parsed = JSON.parse(text);
fs.writeFileSync(cacheFile, text);
return parsed;
} catch (err) {
if (attempt === RETRIES) {
console.warn(` ! ${url.split('/').slice(-1)[0]}: ${err.message}`)
return null
console.warn(` ! ${url.split('/').slice(-1)[0]}: ${err.message}`);
return null;
}
await new Promise(r => setTimeout(r, 500 * attempt))
await new Promise((r) => setTimeout(r, 500 * attempt));
}
}
return null
return null;
}
// Run async tasks with a fixed concurrency cap.
async function pool(items, worker) {
const results = []
let i = 0
const results = [];
let i = 0;
const runners = Array.from({ length: CONCURRENCY }, async () => {
while (i < items.length) {
const idx = i++
results[idx] = await worker(items[idx], idx)
const idx = i++;
results[idx] = await worker(items[idx], idx);
}
})
await Promise.all(runners)
return results
});
await Promise.all(runners);
return results;
}
// Geometry size control. geoBoundaries' "_simplified" files still carry ~12-decimal
@@ -117,149 +370,161 @@ async function pool(items, worker) {
// the now-redundant consecutive duplicate points shrinks the bundles ~5-8x with no
// visible effect at the atlas' zoom range (3-10). ADM0 fills are viewed zoomed out, so
// they tolerate a coarser grid than ADM1 region borders.
const ADM0_DECIMALS = 2 // ~1.1 km
const ADM1_DECIMALS = 3 // ~110 m
const ADM0_DECIMALS = 2; // ~1.1 km
const ADM1_DECIMALS = 3; // ~110 m
function quantizeRing(ring, decimals) {
const m = 10 ** decimals
const out = []
let prevX, prevY
const m = 10 ** decimals;
const out = [];
let prevX, prevY;
for (const pt of ring) {
const x = Math.round(pt[0] * m) / m
const y = Math.round(pt[1] * m) / m
if (x === prevX && y === prevY) continue
out.push([x, y])
prevX = x; prevY = y
const x = Math.round(pt[0] * m) / m;
const y = Math.round(pt[1] * m) / m;
if (x === prevX && y === prevY) continue;
out.push([x, y]);
prevX = x;
prevY = y;
}
return out
return out;
}
// Quantize a (Multi)Polygon, dropping rings that collapse below a valid ring (<4 pts).
function quantizeGeometry(geom, decimals) {
if (!geom) return null
if (!geom) return null;
if (geom.type === 'Polygon') {
const rings = geom.coordinates.map(r => quantizeRing(r, decimals)).filter(r => r.length >= 4)
return rings.length ? { type: 'Polygon', coordinates: rings } : null
const rings = geom.coordinates.map((r) => quantizeRing(r, decimals)).filter((r) => r.length >= 4);
return rings.length ? { type: 'Polygon', coordinates: rings } : null;
}
if (geom.type === 'MultiPolygon') {
const polys = geom.coordinates
.map(poly => poly.map(r => quantizeRing(r, decimals)).filter(r => r.length >= 4))
.filter(poly => poly.length)
return polys.length ? { type: 'MultiPolygon', coordinates: polys } : null
.map((poly) => poly.map((r) => quantizeRing(r, decimals)).filter((r) => r.length >= 4))
.filter((poly) => poly.length);
return polys.length ? { type: 'MultiPolygon', coordinates: polys } : null;
}
return geom
return geom;
}
// Normalize one CGAZ ADM0 feature (keyed by alpha-3 `shapeGroup`) to the property names
// the client country layer reads (ISO_A2/ADM0_A3/NAME/ADMIN). Returns null for the CRS
// pseudo-entry or anything without a group/geometry.
function normalizeAdm0Feature(f) {
const a3 = f.properties?.shapeGroup
if (!a3) return null
const name = f.properties?.shapeName || a3
const geometry = quantizeGeometry(f.geometry, ADM0_DECIMALS)
if (!geometry) return null
const a3 = f.properties?.shapeGroup;
if (!a3) return null;
const name = f.properties?.shapeName || a3;
const geometry = quantizeGeometry(f.geometry, ADM0_DECIMALS);
if (!geometry) return null;
return {
type: 'Feature',
properties: { ISO_A2: A3_TO_A2[a3] || null, ADM0_A3: a3, NAME: name, ADMIN: name },
geometry,
}
};
}
function normalizeAdm1(geo, a3, countryName) {
if (!geo?.features) return []
const a2 = A3_TO_A2[a3] || null
if (!geo?.features) return [];
const a2 = A3_TO_A2[a3] || null;
// Ensure every region in a country ends up with a distinct iso_3166_2 — the Atlas
// marks/unmarks regions by this code, so duplicates make one mark light up the whole
// country.
const used = new Set()
const used = new Set();
const uniq = (base) => {
let code = base, n = 2
while (used.has(code)) code = `${base}-${n++}`
used.add(code)
return code
}
return geo.features.map(f => {
let name = f.properties?.shapeName || ''
// geoBoundaries erroneously names Guangdong Province as "Guangzhou Province" (Guangzhou is the capital city).
if (a3 === 'CHN' && name === 'Guangzhou Province') {
name = 'Guangdong Province'
}
const geometry = quantizeGeometry(f.geometry, ADM1_DECIMALS)
if (!geometry) return null
// shapeISO is a real ISO 3166-2 code for most features, but geoBoundaries sometimes
// fills it with the bare country code instead of a subdivision code — e.g. every
// Spanish region gets "ESP", every Chinese "CHN" (also CL/OM). Keep it only when it
// is a real `XX-…` subdivision code and not already taken; otherwise synthesize a
// stable, unique-per-country id from the region name so each region is independently
// markable.
const raw = f.properties?.shapeISO || ''
let code
if (/^[A-Za-z]{2}-[A-Za-z0-9]+$/.test(raw) && !used.has(raw)) {
code = raw
used.add(code)
} else if (a2) {
code = uniq(`${a2}-${name.replace(/[^A-Za-z0-9]/g, '').toUpperCase() || 'RGN'}`)
} else {
code = raw
}
return {
type: 'Feature',
// Property names the Atlas region layer + server getRegionGeo already read.
properties: {
iso_a2: a2,
iso_3166_2: code,
name,
name_en: name,
admin: countryName,
},
geometry,
}
}).filter(Boolean)
let code = base,
n = 2;
while (used.has(code)) code = `${base}-${n++}`;
used.add(code);
return code;
};
return geo.features
.map((f) => {
let name = f.properties?.shapeName || '';
// geoBoundaries erroneously names Guangdong Province as "Guangzhou Province" (Guangzhou is the capital city).
if (a3 === 'CHN' && name === 'Guangzhou Province') {
name = 'Guangdong Province';
}
const geometry = quantizeGeometry(f.geometry, ADM1_DECIMALS);
if (!geometry) return null;
// shapeISO is a real ISO 3166-2 code for most features, but geoBoundaries sometimes
// fills it with the bare country code instead of a subdivision code — e.g. every
// Spanish region gets "ESP", every Chinese "CHN" (also CL/OM). Keep it only when it
// is a real `XX-…` subdivision code and not already taken; otherwise synthesize a
// stable, unique-per-country id from the region name so each region is independently
// markable.
const raw = f.properties?.shapeISO || '';
let code;
if (/^[A-Za-z]{2}-[A-Za-z0-9]+$/.test(raw) && !used.has(raw)) {
code = raw;
used.add(code);
} else if (a2) {
code = uniq(`${a2}-${name.replace(/[^A-Za-z0-9]/g, '').toUpperCase() || 'RGN'}`);
} else {
code = raw;
}
return {
type: 'Feature',
// Property names the Atlas region layer + server getRegionGeo already read.
properties: {
iso_a2: a2,
iso_3166_2: code,
name,
name_en: name,
admin: countryName,
},
geometry,
};
})
.filter(Boolean);
}
async function main() {
console.log(`[atlas-geo] geoBoundaries ref ${GB_REF}; ${COUNTRIES.length} countries`)
fs.mkdirSync(OUT_DIR, { recursive: true })
fs.mkdirSync(CACHE_DIR, { recursive: true })
console.log(`[atlas-geo] geoBoundaries ref ${GB_REF}; ${COUNTRIES.length} countries`);
fs.mkdirSync(OUT_DIR, { recursive: true });
fs.mkdirSync(CACHE_DIR, { recursive: true });
// ADM0 (countries) — one comprehensive CGAZ file (large; cached). Also yields the
// English country name (shapeGroup → shapeName) used for the ADM1 `admin` field.
console.log('[atlas-geo] downloading CGAZ ADM0 (countries)…')
const cgaz = await fetchGeo(CGAZ_ADM0)
const adm0Features = []
const a3ToName = {}
console.log('[atlas-geo] downloading CGAZ ADM0 (countries)…');
const cgaz = await fetchGeo(CGAZ_ADM0);
const adm0Features = [];
const a3ToName = {};
for (const f of cgaz?.features || []) {
const nf = normalizeAdm0Feature(f)
if (nf) { a3ToName[nf.properties.ADM0_A3] = nf.properties.NAME; adm0Features.push(nf) }
const nf = normalizeAdm0Feature(f);
if (nf) {
a3ToName[nf.properties.ADM0_A3] = nf.properties.NAME;
adm0Features.push(nf);
}
}
// ADM1 (sub-national regions) — per-country gbOpen (carries ISO 3166-2 `shapeISO`).
console.log('[atlas-geo] downloading ADM1 (regions)…')
const adm1Raw = await pool(COUNTRIES, a3 => fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1')))
const adm1Features = []
let withCodes = 0
console.log('[atlas-geo] downloading ADM1 (regions)…');
const adm1Raw = await pool(COUNTRIES, (a3) => fetchGeo(MEDIA(a3, ADM_LEVEL[a3] || 'ADM1')));
const adm1Features = [];
let withCodes = 0;
COUNTRIES.forEach((a3, idx) => {
const feats = normalizeAdm1(adm1Raw[idx], a3, a3ToName[a3] || a3)
for (const f of feats) if (f.properties.iso_3166_2) withCodes++
adm1Features.push(...feats)
})
const feats = normalizeAdm1(adm1Raw[idx], a3, a3ToName[a3] || a3);
for (const f of feats) if (f.properties.iso_3166_2) withCodes++;
adm1Features.push(...feats);
});
const write = (name, features) => {
const fc = { type: 'FeatureCollection', features }
const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 })
const file = path.join(OUT_DIR, `${name}.geojson.gz`)
fs.writeFileSync(file, gz)
console.log(`[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`)
}
const fc = { type: 'FeatureCollection', features };
const gz = zlib.gzipSync(Buffer.from(JSON.stringify(fc)), { level: 9 });
const file = path.join(OUT_DIR, `${name}.geojson.gz`);
fs.writeFileSync(file, gz);
console.log(
`[atlas-geo] wrote ${path.relative(path.join(__dirname, '..'), file)} — ${features.length} features, ${(gz.length / 1e6).toFixed(1)} MB gz`,
);
};
write('admin0', adm0Features)
write('admin1', adm1Features)
write('admin0', adm0Features);
write('admin1', adm1Features);
const missing1 = COUNTRIES.filter((a3, i) => !normalizeAdm1(adm1Raw[i], a3, '').length)
console.log(`[atlas-geo] ADM0 country features: ${adm0Features.length}`)
console.log(`[atlas-geo] ADM1 countries without regions (skipped/404): ${missing1.length}`)
console.log(`[atlas-geo] ADM1 features with ISO 3166-2 code: ${withCodes}/${adm1Features.length}`)
const missing1 = COUNTRIES.filter((a3, i) => !normalizeAdm1(adm1Raw[i], a3, '').length);
console.log(`[atlas-geo] ADM0 country features: ${adm0Features.length}`);
console.log(`[atlas-geo] ADM1 countries without regions (skipped/404): ${missing1.length}`);
console.log(`[atlas-geo] ADM1 features with ISO 3166-2 code: ${withCodes}/${adm1Features.length}`);
}
main().catch(err => { console.error(err); process.exit(1) })
main().catch((err) => {
console.error(err);
process.exit(1);
});
server/scripts/build-legacy-fixture-v244.mjs
+271 -27
View File
@@ -37,7 +37,17 @@ const HEADER = `-- Legacy-runner install at schema_version 244: the ${TAG} relea
function bootRelease(tmp) {
const archive = path.join(tmp, 'src.tar');
execFileSync('git', ['-C', REPO, 'archive', '-o', archive, TAG, 'server/src', 'server/package.json', 'server/tsconfig.json']);
execFileSync('git', [
'-C',
REPO,
'archive',
'-o',
archive,
TAG,
'server/src',
'server/package.json',
'server/tsconfig.json',
]);
// Relative names: GNU tar reads a drive letter as a remote host.
execFileSync('tar', ['-xf', 'src.tar'], { cwd: tmp });
fs.rmSync(archive);
@@ -70,14 +80,38 @@ function seed(db) {
db.prepare(`UPDATE users SET password_hash = 'fixture-login-disabled' WHERE id = 1`).run();
// Mixed case on purpose: the case-insensitive email index must accept it.
ins('users', { id: 2, username: 'alice', email: 'Alice.Traveller@Example.test', password_hash: 'fixture-login-disabled', display_name: 'Alice' });
ins('users', {
id: 2,
username: 'alice',
email: 'Alice.Traveller@Example.test',
password_hash: 'fixture-login-disabled',
display_name: 'Alice',
});
ins('users', { id: 3, username: 'bob', email: 'bob@example.test', password_hash: 'fixture-login-disabled' });
ins('settings', { user_id: 2, key: 'temperature_unit', value: 'celsius' });
ins('settings', { user_id: 3, key: 'language', value: 'de' });
// Trip 1 is upcoming and shared; trip 2 lies in the past with a reminder long due.
ins('trips', { id: 1, user_id: 2, title: 'Lisbon long weekend', description: 'Synthetic upgrade fixture trip', start_date: '2027-05-10', end_date: '2027-05-12', currency: 'EUR', reminder_days: 3 });
ins('trips', { id: 2, user_id: 3, title: 'Alps hiking', start_date: '2026-03-01', end_date: '2026-03-02', currency: 'CHF', reminder_days: 1, is_archived: 1 });
ins('trips', {
id: 1,
user_id: 2,
title: 'Lisbon long weekend',
description: 'Synthetic upgrade fixture trip',
start_date: '2027-05-10',
end_date: '2027-05-12',
currency: 'EUR',
reminder_days: 3,
});
ins('trips', {
id: 2,
user_id: 3,
title: 'Alps hiking',
start_date: '2026-03-01',
end_date: '2026-03-02',
currency: 'CHF',
reminder_days: 1,
is_archived: 1,
});
ins('trip_members', { trip_id: 1, user_id: 3, invited_by: 2 });
for (const [id, trip_id, day_number, date, title] of [
@@ -90,16 +124,59 @@ function seed(db) {
ins('days', { id, trip_id, day_number, date, title });
}
ins('places', { id: 1, trip_id: 1, name: 'Hotel Alfama', lat: 38.7115, lng: -9.13, address: 'Rua dos Remedios 1, Lisboa', category_id: 1 });
ins('places', { id: 2, trip_id: 1, name: 'Torre de Belem', lat: 38.6916, lng: -9.216, address: 'Av. Brasilia, Lisboa', category_id: 3, price: 12.5, currency: 'EUR', notes: 'Book online' });
ins('places', { id: 3, trip_id: 1, name: 'Time Out Market', lat: 38.7069, lng: -9.1459, category_id: 2, place_time: '19:30' });
ins('places', {
id: 1,
trip_id: 1,
name: 'Hotel Alfama',
lat: 38.7115,
lng: -9.13,
address: 'Rua dos Remedios 1, Lisboa',
category_id: 1,
});
ins('places', {
id: 2,
trip_id: 1,
name: 'Torre de Belem',
lat: 38.6916,
lng: -9.216,
address: 'Av. Brasilia, Lisboa',
category_id: 3,
price: 12.5,
currency: 'EUR',
notes: 'Book online',
});
ins('places', {
id: 3,
trip_id: 1,
name: 'Time Out Market',
lat: 38.7069,
lng: -9.1459,
category_id: 2,
place_time: '19:30',
});
ins('places', { id: 4, trip_id: 2, name: 'Gornergrat', lat: 45.9833, lng: 7.7847, category_id: 3 });
ins('places', { id: 5, trip_id: 1, name: 'Fado bar', lat: 38.7110, lng: -9.1290, category_id: 2 });
ins('places', { id: 5, trip_id: 1, name: 'Fado bar', lat: 38.711, lng: -9.129, category_id: 2 });
// The canaries tests/helpers/legacy-upgrade-suite.ts asserts on, as in legacy-v242:
// user-set times on assignments 1 to 3 of day 2, and the booked night after them.
ins('day_accommodations', { id: 1, trip_id: 1, place_id: 1, start_day_id: 1, end_day_id: 3, check_in: '15:00', check_out: '11:00', confirmation: 'HTL-4242' });
ins('day_assignments', { id: 1, day_id: 2, place_id: 2, order_index: 0, assignment_time: '09:00', assignment_end_time: '12:00' });
ins('day_accommodations', {
id: 1,
trip_id: 1,
place_id: 1,
start_day_id: 1,
end_day_id: 3,
check_in: '15:00',
check_out: '11:00',
confirmation: 'HTL-4242',
});
ins('day_assignments', {
id: 1,
day_id: 2,
place_id: 2,
order_index: 0,
assignment_time: '09:00',
assignment_end_time: '12:00',
});
ins('day_assignments', { id: 2, day_id: 2, place_id: 3, order_index: 1, assignment_time: '18:00' });
ins('day_assignments', { id: 3, day_id: 2, place_id: 5, order_index: 2, assignment_time: '20:00' });
ins('day_assignments', { id: 4, day_id: 2, place_id: 1, order_index: 3, accommodation_id: 1 });
@@ -107,36 +184,203 @@ function seed(db) {
ins('day_assignments', { id: 6, day_id: 4, place_id: 4, order_index: 0 });
ins('day_notes', { day_id: 2, trip_id: 1, text: 'Tram 15 to Belem', time: '09:30', sort_order: 0 });
ins('reservations', { id: 1, trip_id: 1, day_id: 1, title: 'TP 1351 Frankfurt to Lisbon', reservation_time: '2027-05-10T08:15', reservation_end_time: '2027-05-10T10:05', confirmation_number: 'ABC123', status: 'confirmed', type: 'flight' });
ins('reservation_endpoints', { reservation_id: 1, role: 'from', sequence: 0, name: 'Frankfurt', code: 'FRA', lat: 50.0379, lng: 8.5622, timezone: 'Europe/Berlin', local_time: '08:15', local_date: '2027-05-10' });
ins('reservation_endpoints', { reservation_id: 1, role: 'to', sequence: 1, name: 'Lisbon', code: 'LIS', lat: 38.7742, lng: -9.1342, timezone: 'Europe/Lisbon', local_time: '10:05', local_date: '2027-05-10' });
ins('reservations', { id: 2, trip_id: 1, day_id: 1, end_day_id: 3, place_id: 1, accommodation_id: 1, title: 'Hotel Alfama', confirmation_number: 'HTL-4242', status: 'confirmed', type: 'hotel' });
ins('reservations', {
id: 1,
trip_id: 1,
day_id: 1,
title: 'TP 1351 Frankfurt to Lisbon',
reservation_time: '2027-05-10T08:15',
reservation_end_time: '2027-05-10T10:05',
confirmation_number: 'ABC123',
status: 'confirmed',
type: 'flight',
});
ins('reservation_endpoints', {
reservation_id: 1,
role: 'from',
sequence: 0,
name: 'Frankfurt',
code: 'FRA',
lat: 50.0379,
lng: 8.5622,
timezone: 'Europe/Berlin',
local_time: '08:15',
local_date: '2027-05-10',
});
ins('reservation_endpoints', {
reservation_id: 1,
role: 'to',
sequence: 1,
name: 'Lisbon',
code: 'LIS',
lat: 38.7742,
lng: -9.1342,
timezone: 'Europe/Lisbon',
local_time: '10:05',
local_date: '2027-05-10',
});
ins('reservations', {
id: 2,
trip_id: 1,
day_id: 1,
end_day_id: 3,
place_id: 1,
accommodation_id: 1,
title: 'Hotel Alfama',
confirmation_number: 'HTL-4242',
status: 'confirmed',
type: 'hotel',
});
ins('budget_items', { id: 1, trip_id: 1, category: 'Accommodation', name: 'Hotel Alfama, two nights', total_price: 312.4, persons: 2, days: 2, paid_by_user_id: 2, expense_date: '2027-05-10', reservation_id: 2, currency: 'EUR' });
ins('budget_items', {
id: 1,
trip_id: 1,
category: 'Accommodation',
name: 'Hotel Alfama, two nights',
total_price: 312.4,
persons: 2,
days: 2,
paid_by_user_id: 2,
expense_date: '2027-05-10',
reservation_id: 2,
currency: 'EUR',
});
ins('budget_item_members', { budget_item_id: 1, user_id: 2, paid: 1, amount: 156.2 });
ins('budget_item_members', { budget_item_id: 1, user_id: 3, paid: 0, amount: 156.2 });
ins('budget_items', { id: 2, trip_id: 1, category: 'Food', name: 'Dinner at the market', total_price: 64, persons: 2, paid_by_user_id: 3, currency: 'EUR' });
ins('budget_items', {
id: 2,
trip_id: 1,
category: 'Food',
name: 'Dinner at the market',
total_price: 64,
persons: 2,
paid_by_user_id: 3,
currency: 'EUR',
});
ins('packing_bags', { id: 1, trip_id: 1, name: 'Carry-on', color: '#3b82f6', weight_limit_grams: 8000, sort_order: 0, user_id: 2 });
ins('packing_items', { trip_id: 1, name: 'Passport', checked: 1, category: 'Documents', sort_order: 0, bag_id: 1, quantity: 1 });
ins('packing_items', { trip_id: 1, name: 'Sunscreen', checked: 0, category: 'Toiletries', sort_order: 1, weight_grams: 150, quantity: 1 });
ins('packing_items', { trip_id: 1, name: 'Private charger', checked: 0, category: 'Tech', sort_order: 2, is_private: 1, owner_id: 3, quantity: 1 });
ins('packing_bags', {
id: 1,
trip_id: 1,
name: 'Carry-on',
color: '#3b82f6',
weight_limit_grams: 8000,
sort_order: 0,
user_id: 2,
});
ins('packing_items', {
trip_id: 1,
name: 'Passport',
checked: 1,
category: 'Documents',
sort_order: 0,
bag_id: 1,
quantity: 1,
});
ins('packing_items', {
trip_id: 1,
name: 'Sunscreen',
checked: 0,
category: 'Toiletries',
sort_order: 1,
weight_grams: 150,
quantity: 1,
});
ins('packing_items', {
trip_id: 1,
name: 'Private charger',
checked: 0,
category: 'Tech',
sort_order: 2,
is_private: 1,
owner_id: 3,
quantity: 1,
});
ins('todo_items', { trip_id: 1, name: 'Buy Lisboa Card', checked: 0, category: 'Tickets', sort_order: 0, due_date: '2027-05-01', assigned_user_id: 3, priority: 1 });
ins('collab_notes', { trip_id: 1, user_id: 2, category: 'Ideas', title: 'Fado evening', content: 'Ask the hotel for a tip' });
ins('todo_items', {
trip_id: 1,
name: 'Buy Lisboa Card',
checked: 0,
category: 'Tickets',
sort_order: 0,
due_date: '2027-05-01',
assigned_user_id: 3,
priority: 1,
});
ins('collab_notes', {
trip_id: 1,
user_id: 2,
category: 'Ideas',
title: 'Fado evening',
content: 'Ask the hotel for a tip',
});
ins('trip_files', { id: 1, trip_id: 1, reservation_id: 1, filename: 'fixture-boarding-pass.pdf', original_name: 'Boarding pass.pdf', file_size: 48213, mime_type: 'application/pdf', uploaded_by: 2 });
ins('trip_files', { id: 2, trip_id: 1, place_id: 2, filename: 'fixture-tickets.pdf', original_name: 'Tickets.pdf', file_size: 1024, mime_type: 'application/pdf', uploaded_by: 3, starred: 1 });
ins('trip_files', {
id: 1,
trip_id: 1,
reservation_id: 1,
filename: 'fixture-boarding-pass.pdf',
original_name: 'Boarding pass.pdf',
file_size: 48213,
mime_type: 'application/pdf',
uploaded_by: 2,
});
ins('trip_files', {
id: 2,
trip_id: 1,
place_id: 2,
filename: 'fixture-tickets.pdf',
original_name: 'Tickets.pdf',
file_size: 1024,
mime_type: 'application/pdf',
uploaded_by: 3,
starred: 1,
});
ins('tags', { id: 1, user_id: 2, name: 'Must see', color: '#ef4444' });
ins('place_tags', { place_id: 2, tag_id: 1 });
ins('journeys', { id: 1, user_id: 2, title: 'Portugal 2027', subtitle: 'Three days by the river', status: 'draft', created_at: T, updated_at: T });
ins('journeys', {
id: 1,
user_id: 2,
title: 'Portugal 2027',
subtitle: 'Three days by the river',
status: 'draft',
created_at: T,
updated_at: T,
});
ins('journey_trips', { journey_id: 1, trip_id: 1, added_at: T });
ins('journey_contributors', { journey_id: 1, user_id: 3, role: 'editor', added_at: T });
ins('journey_entries', { id: 1, journey_id: 1, source_trip_id: 1, source_place_id: 2, author_id: 2, type: 'entry', title: 'Belem in the sun', story: 'Pasteis first, tower second.', entry_date: '2027-05-11', entry_time: '11:00', location_name: 'Belem', location_lat: 38.6916, location_lng: -9.216, mood: 'amazing', visibility: 'shared', sort_order: 0, created_at: T, updated_at: T });
ins('journey_entries', {
id: 1,
journey_id: 1,
source_trip_id: 1,
source_place_id: 2,
author_id: 2,
type: 'entry',
title: 'Belem in the sun',
story: 'Pasteis first, tower second.',
entry_date: '2027-05-11',
entry_time: '11:00',
location_name: 'Belem',
location_lat: 38.6916,
location_lng: -9.216,
mood: 'amazing',
visibility: 'shared',
sort_order: 0,
created_at: T,
updated_at: T,
});
ins('notifications', { type: 'simple', scope: 'trip', target: 1, sender_id: 2, recipient_id: 3, title_key: 'notif.test.title', text_key: 'notif.test.text', is_read: 0 });
ins('notifications', {
type: 'simple',
scope: 'trip',
target: 1,
sender_id: 2,
recipient_id: 3,
title_key: 'notif.test.title',
text_key: 'notif.test.text',
is_read: 0,
});
}
/** Every timestamp the runner or a column default wrote is "now"; pin them so a rebuild is byte-stable. */
server/scripts/coverage-thresholds.mjs
+4 -1
View File
@@ -35,7 +35,10 @@ const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
const domains = new Map();
for (const [file, entry] of Object.entries(summary)) {
if (file === 'total') continue;
const match = file.split('\\').join('/').match(/\/src\/nest\/([a-z0-9-]+)\//);
const match = file
.split('\\')
.join('/')
.match(/\/src\/nest\/([a-z0-9-]+)\//);
if (!match) continue;
const totals = domains.get(match[1]) ?? new Map(METRICS.map(([k]) => [k, { covered: 0, total: 0 }]));
for (const [key, metric] of METRICS) {
server/scripts/dev.mjs
+8 -1
View File
@@ -4,7 +4,14 @@ console.log('[dev] initial build...');
execSync('node scripts/build.mjs', { stdio: 'inherit' });
const children = [];
const stop = () => { children.forEach((c) => { try { c.kill(); } catch {} }); process.exit(0); };
const stop = () => {
children.forEach((c) => {
try {
c.kill();
} catch {}
});
process.exit(0);
};
process.on('SIGINT', stop);
process.on('SIGTERM', stop);
server/scripts/dialect-lint.mjs
+19 -7
View File
@@ -29,8 +29,8 @@
*
* --dir=<path> points the check at another server root (the unit tests use it).
*/
import { createRequire } from 'node:module';
import { existsSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -225,9 +225,12 @@ export function readBaseline(path) {
throw new Error(`scripts/dialect-baseline.json: ${file} must map rule names to counts`);
}
for (const [rule, n] of Object.entries(rules)) {
if (!RULE_NAMES.has(rule)) throw new Error(`scripts/dialect-baseline.json: ${file} names an unknown rule "${rule}"`);
if (!RULE_NAMES.has(rule))
throw new Error(`scripts/dialect-baseline.json: ${file} names an unknown rule "${rule}"`);
if (!Number.isInteger(n) || n <= 0) {
throw new Error(`scripts/dialect-baseline.json: ${file} ${rule} holds ${JSON.stringify(n)}, expected a positive integer`);
throw new Error(
`scripts/dialect-baseline.json: ${file} ${rule} holds ${JSON.stringify(n)}, expected a positive integer`,
);
}
}
}
@@ -291,18 +294,27 @@ function main(argv) {
console.error(`FAIL ${file}: ${n} SQLite-only ${rule} hit(s), ${allowed} allowed. ${adviceFor(rule)}.`);
}
for (const { file, rule, allowed, now } of stale) {
console.error(`FAIL ${file} is held at ${allowed} ${rule} hit(s) in scripts/dialect-baseline.json, but has ${now} now.`);
console.error(
`FAIL ${file} is held at ${allowed} ${rule} hit(s) in scripts/dialect-baseline.json, but has ${now} now.`,
);
}
if (stale.length) {
console.error('Run npm run lint:dialect -- --update to lower the baseline with the change that removed them.');
}
const total = Object.values(baseline).reduce((sum, rules) => sum + Object.values(rules).reduce((a, b) => a + b, 0), 0);
console.log(`dialect: ${Object.keys(baseline).length} file(s) hold ${total} SQLite-only spelling(s) at their baseline`);
const total = Object.values(baseline).reduce(
(sum, rules) => sum + Object.values(rules).reduce((a, b) => a + b, 0),
0,
);
console.log(
`dialect: ${Object.keys(baseline).length} file(s) hold ${total} SQLite-only spelling(s) at their baseline`,
);
return grown.length || stale.length ? 1 : 0;
}
const isCli =
Boolean(process.argv[1]) && existsSync(process.argv[1]) && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));
server/scripts/eslint-ratchet.mjs
+3 -2
View File
@@ -23,10 +23,11 @@
* ESLint comes from this workspace, so the server lints with the major its
* config was written for.
*/
import { ESLint } from 'eslint';
import { fileURLToPath } from 'node:url';
import { check, lintWith } from '../../scripts/lib/eslint-warnings.mjs';
import { runCli } from '../../scripts/lib/ratchet.mjs';
import { ESLint } from 'eslint';
import { fileURLToPath } from 'node:url';
const root = fileURLToPath(new URL('..', import.meta.url));
await runCli('lint:warnings', (args) => check({ root, update: args.includes('--update'), lint: lintWith(ESLint) }));
server/scripts/extract-templates.cjs
+30 -26
View File
@@ -6,16 +6,16 @@
* out on an A4 landscape book as it does on the 210mm square it was designed
* for. Run from server/, output goes to the client.
*/
const Database = require('better-sqlite3')
const fs = require('fs')
const Database = require('better-sqlite3');
const fs = require('fs');
const JOURNEY = Number(process.argv[2] || 7)
const OUT = process.argv[3] || '../client/src/components/Studio/bookTemplates.data.ts'
const JOURNEY = Number(process.argv[2] || 7);
const OUT = process.argv[3] || '../client/src/components/Studio/bookTemplates.data.ts';
const db = new Database('data/travel.db', { readonly: true })
const row = db.prepare('SELECT document FROM journey_books WHERE journey_id = ?').get(JOURNEY)
db.close()
if (!row) throw new Error(`no book on journey ${JOURNEY}`)
const db = new Database('data/travel.db', { readonly: true });
const row = db.prepare('SELECT document FROM journey_books WHERE journey_id = ?').get(JOURNEY);
db.close();
if (!row) throw new Error(`no book on journey ${JOURNEY}`);
/*
* Through the contract on the way in, so a template carries every field the
@@ -23,40 +23,44 @@ if (!row) throw new Error(`no book on journey ${JOURNEY}`)
* Without it a template written today is missing tomorrow's defaults and does
* not typecheck as a BookElement.
*/
const { normalizeBookDocument } = require('@trek/shared')
const doc = normalizeBookDocument(JSON.parse(row.document))
const PW = doc.page.pageWidth
const PH = doc.page.pageHeight
const { normalizeBookDocument } = require('@trek/shared');
const doc = normalizeBookDocument(JSON.parse(row.document));
const PW = doc.page.pageWidth;
const PH = doc.page.pageHeight;
const r4 = n => Math.round(n * 10000) / 10000
const r4 = (n) => Math.round(n * 10000) / 10000;
/** A frame as fractions of one page's width and of the page height. */
function relFrame(f) {
return { x: r4(f.x / PW), y: r4(f.y / PH), w: r4(f.w / PW), h: r4(f.h / PH) }
return { x: r4(f.x / PW), y: r4(f.y / PH), w: r4(f.w / PW), h: r4(f.h / PH) };
}
/** Type size as a fraction of the page height, so it scales with the sheet. */
const relSize = n => r4(n / PH)
const relSize = (n) => r4(n / PH);
function element(e) {
// The id travels with the template and is replaced when it is applied —
// dropping it here would leave the element off the BookElement union.
const out = { ...e, frame: relFrame(e.frame) }
if (typeof out.size === 'number') out.size = relSize(out.size)
if (typeof out.radius === 'number') out.radius = r4(out.radius / PW)
if (typeof out.strokeWidth === 'number') out.strokeWidth = r4(out.strokeWidth / PW)
return out
const out = { ...e, frame: relFrame(e.frame) };
if (typeof out.size === 'number') out.size = relSize(out.size);
if (typeof out.radius === 'number') out.radius = r4(out.radius / PW);
if (typeof out.strokeWidth === 'number') out.strokeWidth = r4(out.strokeWidth / PW);
return out;
}
const spreads = doc.spreads
.map((s, i) => ({ index: i, role: s.role, background: s.background, elements: s.elements.map(element) }))
.filter(s => s.role === 'inner')
.filter((s) => s.role === 'inner');
const body = spreads.map((s, i) => ` {
const body = spreads
.map(
(s, i) => ` {
id: 'ref-${i + 1}',
background: ${JSON.stringify(s.background)},
elements: ${JSON.stringify(s.elements, null, 6).replace(/\n/g, '\n ')},
},`).join('\n')
},`,
)
.join('\n');
const file = `import type { BookElement } from '@trek/shared'
@@ -103,7 +107,7 @@ export interface SpreadTemplate {
export const SPREAD_TEMPLATES: SpreadTemplate[] = [
${body}
]
`
`;
fs.writeFileSync(OUT, file)
console.log(`wrote ${spreads.length} templates to ${OUT}`)
fs.writeFileSync(OUT, file);
console.log(`wrote ${spreads.length} templates to ${OUT}`);
server/scripts/format-baseline.json
+1 -1570
View File
File diff suppressed because it is too large Load Diff
server/scripts/format-lint.mjs
+3 -2
View File
@@ -22,11 +22,12 @@
* scripts/lib/format.mjs at the repository root; Prettier and its config come
* from this workspace.
*/
import { fileURLToPath } from 'node:url';
import * as prettier from 'prettier';
import { check } from '../../scripts/lib/format.mjs';
import { runCli } from '../../scripts/lib/ratchet.mjs';
import { fileURLToPath } from 'node:url';
import * as prettier from 'prettier';
const root = fileURLToPath(new URL('..', import.meta.url));
// The entity files are written by scripts/generate-entities.ts and held byte for
// byte to its output by check:entities, so their layout is the generator's, not
server/scripts/gen-plugin-facts.ts
+53 -23
View File
@@ -15,30 +15,60 @@
* plugin-sdk/ checkout with no server present, so a build-time artefact would ship an SDK
* with an empty permission list.
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import {
HOOK_PERMISSION, KNOWN_METHODS, KNOWN_PERMISSIONS, METHOD_PERMISSION,
EVENTS_PERMISSION, JOBS_PERMISSION, USER_DATA_PERMISSION, HTTP_OUTBOUND_PREFIX, PLUGIN_API_VERSION,
} from '../src/nest/plugins/protocol/envelope';
import {
ACTIONS_MAX, ADDON_ID_RE, CAPABILITY_NAME_RE, EGRESS_HOST_RE, MCP_TOOLS_MAX, PLUGIN_CHANNEL_EVENTS, PLUGIN_ID_RE,
PLUGIN_SEMVER_RE, PLUGIN_TYPES, REPLACEABLE_TABS, RESERVED_PLUGIN_IDS, RESERVED_SETTING_KEYS, ROUTE_PROFILE_ID_RE,
ROUTE_PROFILES_MAX, SETTING_FIELD_KEYS, SETTING_KEY_RE, TOOL_NAME_RE, TRIP_PAGE_POSITION_MAX, WIDGET_SLOTS,
} from '../src/nest/plugins/protocol/manifest-rules';
import { SNAPSHOT_GRANT, ENTITY_ID_KEYS } from '../src/plugin-event-sink';
import {
PLUGIN_ENTITY_CONTRACT, PLUGIN_ENTITY_NESTED, PLUGIN_METHOD_OUTPUT, pluginEntityFields, type PluginEntityName,
type PluginMethodOutput,
} from '../src/nest/plugins/protocol/output-contract';
import { ADDON_IDS } from '../src/addons';
// A relative import of the shared SOURCE, not @trek/shared: the plugin-facts CI job
// installs only the server, so shared's dist does not exist there. The file has no
// imports of its own for exactly this reason.
import {
PLUGIN_POI_ICONS, PLUGIN_POI_LABEL_MAX, PLUGIN_POI_MAX_CATEGORIES,
PLUGIN_POI_ICONS,
PLUGIN_POI_LABEL_MAX,
PLUGIN_POI_MAX_CATEGORIES,
} from '../../shared/src/plugins/plugin-poi-facts';
import { ADDON_IDS } from '../src/addons';
import {
HOOK_PERMISSION,
KNOWN_METHODS,
KNOWN_PERMISSIONS,
METHOD_PERMISSION,
EVENTS_PERMISSION,
JOBS_PERMISSION,
USER_DATA_PERMISSION,
HTTP_OUTBOUND_PREFIX,
PLUGIN_API_VERSION,
} from '../src/nest/plugins/protocol/envelope';
import {
ACTIONS_MAX,
ADDON_ID_RE,
CAPABILITY_NAME_RE,
EGRESS_HOST_RE,
MCP_TOOLS_MAX,
PLUGIN_CHANNEL_EVENTS,
PLUGIN_ID_RE,
PLUGIN_SEMVER_RE,
PLUGIN_TYPES,
REPLACEABLE_TABS,
RESERVED_PLUGIN_IDS,
RESERVED_SETTING_KEYS,
ROUTE_PROFILE_ID_RE,
ROUTE_PROFILES_MAX,
SETTING_FIELD_KEYS,
SETTING_KEY_RE,
TOOL_NAME_RE,
TRIP_PAGE_POSITION_MAX,
WIDGET_SLOTS,
} from '../src/nest/plugins/protocol/manifest-rules';
import {
PLUGIN_ENTITY_CONTRACT,
PLUGIN_ENTITY_NESTED,
PLUGIN_METHOD_OUTPUT,
pluginEntityFields,
type PluginEntityName,
type PluginMethodOutput,
} from '../src/nest/plugins/protocol/output-contract';
import { SNAPSHOT_GRANT, ENTITY_ID_KEYS } from '../src/plugin-event-sink';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
@@ -54,7 +84,9 @@ const HEADER = [
const list = (xs: readonly string[]) => xs.map((x) => ` '${x}',`).join('\n');
const pairs = (o: Readonly<Record<string, string>>) =>
Object.entries(o).map(([k, v]) => ` ${/^[A-Za-z_$][\w$]*$/.test(k) ? k : `'${k}'`}: '${v}',`).join('\n');
Object.entries(o)
.map(([k, v]) => ` ${/^[A-Za-z_$][\w$]*$/.test(k) ? k : `'${k}'`}: '${v}',`)
.join('\n');
/**
* Types here are deliberately WIDE — Readonly<Record<string, string>> and string[] —
@@ -261,9 +293,7 @@ function coverageProblems(): string[] {
const ui = fs.readFileSync(path.join(REPO, 'plugin-sdk/src/cli/ui.ts'), 'utf8');
const missingHint = KNOWN_PERMISSIONS.filter((p) => !ui.includes(`'${p}'`));
if (missingHint.length) {
problems.push(
`plugin-sdk/src/cli/ui.ts PERMISSION_FAMILIES does not cover: ${missingHint.join(', ')}`,
);
problems.push(`plugin-sdk/src/cli/ui.ts PERMISSION_FAMILIES does not cover: ${missingHint.join(', ')}`);
}
return problems;
server/scripts/generate-db-types.ts
+6 -7
View File
@@ -25,13 +25,11 @@
* node --import tsx scripts/generate-db-types.ts # rewrite src/db/kysely/
* node --import tsx scripts/generate-db-types.ts --check # exit 1 when src/db/kysely/ drifted (CI)
*/
import fs from 'node:fs';
import path from 'node:path';
import { migrateTempDb, tempDbPath } from './generate-entities';
import Database from 'better-sqlite3';
import { migrateTempDb, tempDbPath } from './generate-entities';
import fs from 'node:fs';
import path from 'node:path';
const SERVER_ROOT = path.join(__dirname, '..');
export const KYSELY_DIR = path.join(SERVER_ROOT, 'src/db/kysely');
@@ -39,7 +37,8 @@ export const KYSELY_DIR = path.join(SERVER_ROOT, 'src/db/kysely');
/** Tables the application never queries: the migrator's own bookkeeping. */
const SKIPPED_TABLES: ReadonlySet<string> = new Set(['mikro_orm_migrations']);
const HEADER = '// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:\n' +
const HEADER =
'// Generated by scripts/generate-db-types.ts from the migrated schema. Do not edit by hand:\n' +
'// change the migration, then run `npm run gen:db-types`.\n';
export interface ColumnInfo {
@@ -151,7 +150,7 @@ export function renderDb(tables: readonly TableInfo[]): string {
const body = tables.map((t) => ` ${propertyKey(t.name)}: ${interfaceName(t.name)};`).join('\n');
return (
`${HEADER}${imports}\n\n` +
'/** Every table of the migrated schema. A repository narrows it: `this.kysely<Pick<DB, \'trips\'>>()`. */\n' +
"/** Every table of the migrated schema. A repository narrows it: `this.kysely<Pick<DB, 'trips'>>()`. */\n" +
`export interface DB {\n${body}\n}\n`
);
}
server/scripts/import-boundaries.mjs
+27 -9
View File
@@ -68,15 +68,23 @@
*
* --dir=<path> points the check at another server root (the unit tests use it).
*/
import { createRequire } from 'node:module';
import { existsSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { dirname, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
const require = createRequire(import.meta.url);
const ts = require('typescript');
export const RULES = ['fileCycles', 'domainCycles', 'sharedImportsDomain', 'domainInternals', 'dbImportsNest', 'foreignRepositories', 'providersImportOrchestrator'];
export const RULES = [
'fileCycles',
'domainCycles',
'sharedImportsDomain',
'domainInternals',
'dbImportsNest',
'foreignRepositories',
'providersImportOrchestrator',
];
/**
* Folders under src/nest whose every file is shared infrastructure, open to all domains.
@@ -148,7 +156,9 @@ function isTypeOnlyImport(node) {
function isTypeOnlyExport(node) {
if (node.isTypeOnly) return true;
const clause = node.exportClause;
return Boolean(clause && ts.isNamedExports(clause) && clause.elements.length > 0 && clause.elements.every((el) => el.isTypeOnly));
return Boolean(
clause && ts.isNamedExports(clause) && clause.elements.length > 0 && clause.elements.every((el) => el.isTypeOnly),
);
}
/** Every relative import of a file: { spec, runtime } where runtime means eager and not erased. */
@@ -180,7 +190,8 @@ export function importsOf(text, fileName = 'file.ts') {
ts.forEachChild(node, (child) => visit(child, inner));
};
for (const stmt of sf.statements) {
if (!ts.isImportDeclaration(stmt) && !ts.isExportDeclaration(stmt) && !ts.isImportEqualsDeclaration(stmt)) visit(stmt, 0);
if (!ts.isImportDeclaration(stmt) && !ts.isExportDeclaration(stmt) && !ts.isImportEqualsDeclaration(stmt))
visit(stmt, 0);
}
return found;
}
@@ -412,7 +423,8 @@ export function analyse(edges) {
if (a && b && SHARED_DOMAINS.has(a) && !SHARED_DOMAINS.has(b)) sharedImportsDomain.add(`${e.from} -> ${e.to}`);
if (a && b && a !== b && !isPublic(e.to)) domainInternals.add(`${a} -> ${e.to.slice('nest/'.length)}`);
if (e.from.startsWith('db/') && e.to.startsWith('nest/')) dbImportsNest.add(`${e.from} -> ${e.to}`);
if (isProviderFile(e.from) && isOrchestratorOf(e.from, e.to)) providersImportOrchestrator.add(`${e.from} -> ${e.to}`);
if (isProviderFile(e.from) && isOrchestratorOf(e.from, e.to))
providersImportOrchestrator.add(`${e.from} -> ${e.to}`);
}
const sorted = (set) => [...set].sort();
return {
@@ -494,14 +506,17 @@ function main(argv) {
);
if (unowned.length) {
// An entity without an owner is an injection the rule cannot judge.
for (const u of unowned) console.error(`FAIL no owner for injected entity ${u}: add it to scripts/repository-owners.json`);
for (const u of unowned)
console.error(`FAIL no owner for injected entity ${u}: add it to scripts/repository-owners.json`);
return 1;
}
found.foreignRepositories = foreignRepositories;
let baseline = readBaseline(baselinePath);
if (update) {
const now = Object.fromEntries(RULES.map((rule) => [rule, new Set(found[rule])]));
baseline = Object.fromEntries(RULES.map((rule) => [rule, baseline[rule].filter((key) => now[rule].has(key)).sort()]));
baseline = Object.fromEntries(
RULES.map((rule) => [rule, baseline[rule].filter((key) => now[rule].has(key)).sort()]),
);
writeFileSync(baselinePath, JSON.stringify(baseline, null, 2) + '\n');
}
@@ -526,14 +541,17 @@ function main(argv) {
);
}
console.log(
`boundaries: ${edges.length} import(s); held at baseline: ` + RULES.map((rule) => `${rule} ${baseline[rule].length}`).join(', '),
`boundaries: ${edges.length} import(s); held at baseline: ` +
RULES.map((rule) => `${rule} ${baseline[rule].length}`).join(', '),
);
return failed ? 1 : 0;
}
// Compared by real path, so the check still runs when the script is started through a symlink.
const isCli =
Boolean(process.argv[1]) && existsSync(process.argv[1]) && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));
server/scripts/migrate-encryption.ts
+50 -27
View File
@@ -14,12 +14,11 @@
* The script will prompt for the old and new keys interactively so they never
* appear in shell history, process arguments, or log output.
*/
import Database from 'better-sqlite3';
import crypto from 'crypto';
import fs from 'fs';
import path from 'path';
import readline from 'readline';
import Database from 'better-sqlite3';
// ---------------------------------------------------------------------------
// Crypto helpers — mirrors apiKeyCrypto.ts and mfaCrypto.ts but with
@@ -143,9 +142,7 @@ async function main() {
console.log('A backup of the database will be created before any changes are made.\n');
// Resolve DB path
const dbPath = path.resolve(
process.env.DB_PATH ?? path.join(__dirname, '../data/travel.db')
);
const dbPath = path.resolve(process.env.DB_PATH ?? path.join(__dirname, '../data/travel.db'));
if (!fs.existsSync(dbPath)) {
console.error(`Database not found at: ${dbPath}`);
@@ -213,14 +210,18 @@ async function main() {
const wantedJournal = (process.env.TREK_DB_JOURNAL_MODE ?? '').trim().toUpperCase();
const journalMode = journalModes.includes(wantedJournal) ? wantedJournal : 'WAL';
if (wantedJournal && journalMode !== wantedJournal) {
console.warn(`TREK_DB_JOURNAL_MODE="${process.env.TREK_DB_JOURNAL_MODE}" is not a SQLite journal mode — using ${journalMode}.`);
console.warn(
`TREK_DB_JOURNAL_MODE="${process.env.TREK_DB_JOURNAL_MODE}" is not a SQLite journal mode — using ${journalMode}.`,
);
}
const wantedSync = (process.env.TREK_DB_SYNCHRONOUS ?? '').trim().toUpperCase();
const defaultSync = journalMode === 'WAL' ? 'NORMAL' : 'FULL';
const synchronous = syncLevels.includes(wantedSync) ? wantedSync : defaultSync;
if (wantedSync && synchronous !== wantedSync) {
console.warn(`TREK_DB_SYNCHRONOUS="${process.env.TREK_DB_SYNCHRONOUS}" is not a SQLite synchronous level — using ${synchronous}.`);
console.warn(
`TREK_DB_SYNCHRONOUS="${process.env.TREK_DB_SYNCHRONOUS}" is not a SQLite synchronous level — using ${synchronous}.`,
);
}
db.pragma(`journal_mode = ${journalMode}`);
@@ -257,7 +258,10 @@ async function main() {
// Helper: migrate a single MFA value (no prefix, raw base64)
function migrateMfaValue(raw: string, label: string): string | null {
if (!raw) { result.skipped++; return null; }
if (!raw) {
result.skipped++;
return null;
}
const plain = decryptMfa(raw, oldKey);
if (plain !== null) {
@@ -335,7 +339,10 @@ async function main() {
const users = db.prepare('SELECT id FROM users').all() as { id: number }[];
for (const user of users) {
const row = db.prepare(`SELECT ${apiKeyColumns.join(', ')} FROM users WHERE id = ?`).get(user.id) as Record<string, string>;
const row = db.prepare(`SELECT ${apiKeyColumns.join(', ')} FROM users WHERE id = ?`).get(user.id) as Record<
string,
string
>;
for (const col of apiKeyColumns) {
if (!row[col]) continue;
@@ -346,7 +353,8 @@ async function main() {
}
// mfa_secret (mfa crypto)
const mfaRow = db.prepare('SELECT mfa_secret FROM users WHERE id = ? AND mfa_secret IS NOT NULL').get(user.id) as { mfa_secret: string } | undefined;
const mfaRow = db.prepare('SELECT mfa_secret FROM users WHERE id = ? AND mfa_secret IS NOT NULL').get(user.id) as
{ mfa_secret: string } | undefined;
if (mfaRow?.mfa_secret) {
const newVal = migrateMfaValue(mfaRow.mfa_secret, `users[${user.id}].mfa_secret`);
if (newVal !== null) {
@@ -373,8 +381,7 @@ async function main() {
};
{
const row = db.prepare("SELECT value FROM app_settings WHERE key = 'storage.backends'").get() as
| { value: string }
| undefined;
{ value: string } | undefined;
if (row?.value) {
let backends: unknown;
try {
@@ -418,9 +425,11 @@ async function main() {
// --- settings: per-user encrypted keys ---
const encryptedSettingKeys = ['webhook_url', 'ntfy_token', 'mapbox_access_token', 'carto_api_key', 'llm_api_key'];
const settingRows = db.prepare(
`SELECT user_id, key, value FROM settings WHERE key IN (${encryptedSettingKeys.map(() => '?').join(', ')})`
).all(...encryptedSettingKeys) as { user_id: number; key: string; value: string }[];
const settingRows = db
.prepare(
`SELECT user_id, key, value FROM settings WHERE key IN (${encryptedSettingKeys.map(() => '?').join(', ')})`,
)
.all(...encryptedSettingKeys) as { user_id: number; key: string; value: string }[];
for (const row of settingRows) {
if (!row.value) continue;
@@ -465,7 +474,7 @@ async function main() {
// Which keys are secret is per plugin and per scope, and it lives in the DB
// rather than in this script, so nothing has to be pinned by hand here.
const secretFields = db
.prepare("SELECT plugin_id, field_key, scope FROM plugin_settings_fields WHERE secret = 1")
.prepare('SELECT plugin_id, field_key, scope FROM plugin_settings_fields WHERE secret = 1')
.all() as { plugin_id: string; field_key: string; scope: string | null }[];
const secretKeysFor = (pluginId: string, scope: string): string[] =>
secretFields
@@ -500,7 +509,10 @@ async function main() {
};
if (tableExists('plugins')) {
const pluginRows = db.prepare('SELECT id, config FROM plugins').all() as { id: string; config: string | null }[];
const pluginRows = db.prepare('SELECT id, config FROM plugins').all() as {
id: string;
config: string | null;
}[];
for (const row of pluginRows) {
const next = migrateConfigJson(row.config, secretKeysFor(row.id, 'instance'), `plugins[${row.id}].config`);
if (next !== null) {
@@ -510,9 +522,11 @@ async function main() {
}
if (tableExists('plugin_user_config')) {
const userConfigRows = db
.prepare('SELECT plugin_id, user_id, config FROM plugin_user_config')
.all() as { plugin_id: string; user_id: number; config: string | null }[];
const userConfigRows = db.prepare('SELECT plugin_id, user_id, config FROM plugin_user_config').all() as {
plugin_id: string;
user_id: number;
config: string | null;
}[];
for (const row of userConfigRows) {
const next = migrateConfigJson(
row.config,
@@ -535,9 +549,10 @@ async function main() {
// (llm-parse/llm-config.ts). Left out, LLM parsing quietly runs without a
// key after the rotation, since a failed decrypt reads as "no key".
if (tableExists('addons')) {
const addonRows = db
.prepare('SELECT id, config FROM addons WHERE config IS NOT NULL')
.all() as { id: string; config: string }[];
const addonRows = db.prepare('SELECT id, config FROM addons WHERE config IS NOT NULL').all() as {
id: string;
config: string;
}[];
for (const row of addonRows) {
let config: unknown;
try {
@@ -559,7 +574,10 @@ async function main() {
}
// --- trip_album_links: passphrase ---
const albumLinks = db.prepare('SELECT id, passphrase FROM trip_album_links WHERE passphrase IS NOT NULL').all() as { id: number; passphrase: string }[];
const albumLinks = db.prepare('SELECT id, passphrase FROM trip_album_links WHERE passphrase IS NOT NULL').all() as {
id: number;
passphrase: string;
}[];
for (const row of albumLinks) {
const newVal = migrateApiKeyValue(row.passphrase, `trip_album_links[${row.id}].passphrase`);
if (newVal !== null) {
@@ -584,7 +602,10 @@ async function main() {
}
// --- trek_photos: passphrase ---
const photos = db.prepare('SELECT id, passphrase FROM trek_photos WHERE passphrase IS NOT NULL').all() as { id: number; passphrase: string }[];
const photos = db.prepare('SELECT id, passphrase FROM trek_photos WHERE passphrase IS NOT NULL').all() as {
id: number;
passphrase: string;
}[];
for (const row of photos) {
const newVal = migrateApiKeyValue(row.passphrase, `trek_photos[${row.id}].passphrase`);
if (newVal !== null) {
@@ -640,7 +661,7 @@ async function main() {
console.log(` Skipped (empty): ${result.skipped}`);
if (result.errors.length > 0) {
console.warn(` Errors: ${result.errors.length}`);
result.errors.forEach(e => console.warn(` - ${e}`));
result.errors.forEach((e) => console.warn(` - ${e}`));
console.warn('\nSome secrets could not be migrated. Check the errors above.');
console.warn(`Your original database is backed up at: ${backupPath}`);
process.exit(1);
@@ -654,7 +675,9 @@ async function main() {
fs.writeFileSync(keyFile, newKey, { mode: 0o600 });
console.log(`Key file updated: ${keyFile}`);
} else {
console.log('No key file next to the database. Store the new key wherever TREK reads it from (ENCRYPTION_KEY or data/.encryption_key).');
console.log(
'No key file next to the database. Store the new key wherever TREK reads it from (ENCRYPTION_KEY or data/.encryption_key).',
);
}
console.log(`Backup retained at: ${backupPath}`);
}
server/scripts/pg-probe.ts
+55 -17
View File
@@ -50,17 +50,31 @@
* Exit codes: 0 passed, 1 a helper case or the ratchet failed, 2 the probe
* could not run (no database, a database that is not empty, a crash).
*/
import { planRepositories, readCompilerOptions, type MethodPlan } from './pg-probe/arg-samples';
import { compareWithBaseline, formatBaseline, lowerBaseline, readBaseline } from './pg-probe/baseline';
import {
connectProbeOrm,
connectSetupOrm,
postgresHelperEngine,
prepareDatabase,
probeRepositories,
sqliteHelperEngine,
} from './pg-probe/engines';
import { runHelperCases, type HelperResult } from './pg-probe/helper-cases';
import { StatementRecorder } from './pg-probe/recorder';
import {
formatConsole,
formatMarkdown,
passed,
probedNothing,
summarize,
type MethodResult,
type ReportInput,
} from './pg-probe/report';
import { appendFileSync, readdirSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { planRepositories, readCompilerOptions, type MethodPlan } from './pg-probe/arg-samples';
import { compareWithBaseline, formatBaseline, lowerBaseline, readBaseline } from './pg-probe/baseline';
import { connectProbeOrm, connectSetupOrm, postgresHelperEngine, prepareDatabase, probeRepositories, sqliteHelperEngine } from './pg-probe/engines';
import { runHelperCases, type HelperResult } from './pg-probe/helper-cases';
import { StatementRecorder } from './pg-probe/recorder';
import { formatConsole, formatMarkdown, passed, probedNothing, summarize, type MethodResult, type ReportInput } from './pg-probe/report';
const SERVER_ROOT = path.join(__dirname, '..');
export const BASELINE_PATH = path.join(SERVER_ROOT, 'scripts', 'pg-probe-baseline.json');
const REPOSITORIES_DIR = path.join(SERVER_ROOT, 'src', 'db', 'repositories');
@@ -87,7 +101,9 @@ export function parseArgs(argv: readonly string[], env: Readonly<Record<string,
summary: env.GITHUB_STEP_SUMMARY || null,
};
for (const arg of argv) {
const [flag, value] = arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, ''];
const [flag, value] = arg.includes('=')
? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)]
: [arg, ''];
if (flag === '--update' && value === '') args.update = true;
else if (flag === '--url' && value) args.url = value;
else if (flag === '--next-baseline' && value) args.nextBaseline = value;
@@ -96,7 +112,9 @@ export function parseArgs(argv: readonly string[], env: Readonly<Record<string,
else throw new Error(`pg-probe: unknown argument ${arg}`);
}
if (!/^postgres(ql)?:\/\//.test(args.url)) {
throw new Error('pg-probe: no Postgres URL. Set TREK_PG_PROBE_URL or pass --url=postgres://user:password@host:port/database.');
throw new Error(
'pg-probe: no Postgres URL. Set TREK_PG_PROBE_URL or pass --url=postgres://user:password@host:port/database.',
);
}
return args;
}
@@ -147,7 +165,9 @@ async function main(): Promise<number> {
try {
const schema = await timed('derive the schema from the entities', () => prepareDatabase(setup));
schemaFailures = schema.failures;
console.log(` ${schema.statements} DDL statements, ${schema.adjusted} columns adjusted, ${schema.failures.length} refused`);
console.log(
` ${schema.statements} DDL statements, ${schema.adjusted} columns adjusted, ${schema.failures.length} refused`,
);
helpers = await timed('dialect helpers on SQLite and Postgres', async () => {
const sqlite = sqliteHelperEngine();
@@ -159,14 +179,20 @@ async function main(): Promise<number> {
});
const plans: MethodPlan[] = await timed('plan the repository calls', async () =>
planRepositories(repositoryFiles(REPOSITORIES_DIR), readCompilerOptions(path.join(SERVER_ROOT, 'tsconfig.json')), SERVER_ROOT),
planRepositories(
repositoryFiles(REPOSITORIES_DIR),
readCompilerOptions(path.join(SERVER_ROOT, 'tsconfig.json')),
SERVER_ROOT,
),
);
console.log(` ${plans.length} public repository methods`);
const recorder = new StatementRecorder();
const probe = await connectProbeOrm(args.url, recorder, STATEMENT_TIMEOUT_MS);
try {
results = await timed('call every repository method', () => probeRepositories(probe, recorder, plans, SERVER_ROOT, METHOD_TIMEOUT_MS));
results = await timed('call every repository method', () =>
probeRepositories(probe, recorder, plans, SERVER_ROOT, METHOD_TIMEOUT_MS),
);
} finally {
await probe.close(true);
}
@@ -201,7 +227,10 @@ async function main(): Promise<number> {
annotate('error', `pg-probe helper case [${failure.engine}] ${failure.name}: ${failure.failure}`);
}
for (const entry of verdict.grown) {
annotate('error', `pg-probe: ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`);
annotate(
'error',
`pg-probe: ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`,
);
}
for (const entry of verdict.stale) {
annotate(
@@ -218,9 +247,13 @@ async function main(): Promise<number> {
);
}
for (const method of verdict.nowCovered) {
annotate('error', `pg-probe: ${method} is listed as uncovered but is measured now (or gone). Drop it (--update, or the pg-probe-baseline artifact).`);
annotate(
'error',
`pg-probe: ${method} is listed as uncovered but is measured now (or gone). Drop it (--update, or the pg-probe-baseline artifact).`,
);
}
if (probedNothing(summary)) annotate('error', `pg-probe: no repository method was called (${summary.methods} planned).`);
if (probedNothing(summary))
annotate('error', `pg-probe: no repository method was called (${summary.methods} planned).`);
if (verdict.unseeded) {
annotate(
'error',
@@ -236,12 +269,17 @@ if (require.main === module) {
// A method abandoned after its timeout may still reject later; that is
// already counted as a timeout and must not take the whole run down.
process.on('unhandledRejection', (reason: unknown) => {
annotate('warning', `pg-probe: a rejection nobody awaited: ${reason instanceof Error ? reason.message.split('\n')[0] : String(reason)}`);
annotate(
'warning',
`pg-probe: a rejection nobody awaited: ${reason instanceof Error ? reason.message.split('\n')[0] : String(reason)}`,
);
});
main().then(
(code) => process.exit(code),
(error: unknown) => {
console.error(`pg-probe could not run: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`);
console.error(
`pg-probe could not run: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`,
);
process.exit(2);
},
);
server/scripts/pg-probe/arg-samples.ts
+21 -6
View File
@@ -14,9 +14,7 @@
* method unprobeable: the report lists it instead of guessing, and the
* baseline's `uncovered` list must name it, so a new one fails the run.
*/
import path from 'node:path';
import ts from 'typescript';
export type Sample = { ok: true; value: unknown } | { ok: false; reason: string };
@@ -118,7 +116,13 @@ function sampleObjectType(checker: ts.TypeChecker, type: ts.ObjectType, name: st
return sampleObject(checker, type, name, depth);
}
function sampleCollection(checker: ts.TypeChecker, type: ts.ObjectType, kind: string, name: string, depth: number): Sample {
function sampleCollection(
checker: ts.TypeChecker,
type: ts.ObjectType,
kind: string,
name: string,
depth: number,
): Sample {
const args = checker.getTypeArguments(type as ts.TypeReference);
const samples = args.map((arg, i) => sampleFor(checker, arg, i === 0 ? singular(name) : name, depth + 1));
const failed = samples.find((sample) => !sample.ok);
@@ -152,7 +156,10 @@ function isPublicMethod(member: ts.ClassElement): member is ts.MethodDeclaration
}
/** The sample argument list for one method, or why there is none. */
export function planMethod(checker: ts.TypeChecker, method: ts.MethodDeclaration): { args: unknown[] } | { unprobeable: string } {
export function planMethod(
checker: ts.TypeChecker,
method: ts.MethodDeclaration,
): { args: unknown[] } | { unprobeable: string } {
const args: unknown[] = [];
for (const parameter of method.parameters) {
const name = ts.isIdentifier(parameter.name) ? parameter.name.text : 'options';
@@ -169,7 +176,11 @@ export function planMethod(checker: ts.TypeChecker, method: ts.MethodDeclaration
}
/** Every public method of every exported class declared in `files`, with its sample arguments. */
export function planRepositories(files: readonly string[], compilerOptions: ts.CompilerOptions, root: string): MethodPlan[] {
export function planRepositories(
files: readonly string[],
compilerOptions: ts.CompilerOptions,
root: string,
): MethodPlan[] {
const program = ts.createProgram({ rootNames: [...files], options: { ...compilerOptions, noEmit: true } });
const checker = program.getTypeChecker();
const wanted = new Set(files.map((file) => path.resolve(file)));
@@ -186,7 +197,11 @@ export function planRepositories(files: readonly string[], compilerOptions: ts.C
if (!isPublicMethod(member)) continue;
const method = (member.name as ts.Identifier).text;
const plan = planMethod(checker, member);
plans.push('args' in plan ? { className, file, method, args: plan.args } : { className, file, method, unprobeable: plan.unprobeable });
plans.push(
'args' in plan
? { className, file, method, args: plan.args }
: { className, file, method, unprobeable: plan.unprobeable },
);
}
}
}
server/scripts/pg-probe/baseline.ts
+4 -3
View File
@@ -31,7 +31,6 @@
* measurement, written by `--update` or taken from the CI artifact, is
* committed.
*/
import { readFileSync } from 'node:fs';
export interface ProbeBaseline {
@@ -104,7 +103,8 @@ function parseFailing(failing: unknown, source: string): Record<string, number>
/** A missing `uncovered` key reads as unmeasured, so a baseline written before the list existed gets it seeded. */
function parseUncovered(uncovered: unknown, source: string): string[] | null {
if (uncovered === undefined || uncovered === null) return null;
if (!Array.isArray(uncovered)) throw new Error(`${source}: "uncovered" must be null or an array of Class.method names`);
if (!Array.isArray(uncovered))
throw new Error(`${source}: "uncovered" must be null or an array of Class.method names`);
const seen = new Set<string>();
for (const method of uncovered as unknown[]) {
if (typeof method !== 'string' || !METHOD_KEY.test(method)) {
@@ -169,7 +169,8 @@ export function lowerBaseline(
if (now > 0) next[method] = now;
}
}
const uncovered = baseline.uncovered === null ? [...uncoveredNow] : baseline.uncovered.filter((method) => uncoveredNow.has(method));
const uncovered =
baseline.uncovered === null ? [...uncoveredNow] : baseline.uncovered.filter((method) => uncoveredNow.has(method));
return {
failing: Object.fromEntries(Object.entries(next).sort(([a], [b]) => a.localeCompare(b))),
uncovered: [...new Set(uncovered)].sort((a, b) => a.localeCompare(b)),
server/scripts/pg-probe/engines.ts
+61 -19
View File
@@ -5,21 +5,34 @@
* SQLite database the helper cases are compared on, and the loop that calls
* every repository method.
*/
import path from 'node:path';
import { RequestContext, type Configuration, type EntityManager, type EntityProperty } from '@mikro-orm/core';
import { MikroORM, PostgreSqlConnection, PostgreSqlDriver, type Options } from '@mikro-orm/postgresql';
import { SqlitePlatform } from '@mikro-orm/sql';
import Database from 'better-sqlite3';
import { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, type PostgresDialect } from 'kysely';
import { ALL_ENTITIES } from '../../src/db/entities';
import type { MethodPlan } from './arg-samples';
import { VALUES_TABLE, VALUES_TABLE_DDL, type HelperEngine, type KyselySelect, type RawSelect, type ValuesDB, type ValuesRow } from './helper-cases';
import {
VALUES_TABLE,
VALUES_TABLE_DDL,
type HelperEngine,
type KyselySelect,
type RawSelect,
type ValuesDB,
type ValuesRow,
} from './helper-cases';
import { ProbePostgresDialect, type StatementRecorder } from './recorder';
import type { MethodResult } from './report';
import { adjustColumns, NOCASE_COLLATION_SQL, splitStatements, type ColumnMeta } from './schema';
import { RequestContext, type Configuration, type EntityManager, type EntityProperty } from '@mikro-orm/core';
import { MikroORM, PostgreSqlConnection, PostgreSqlDriver, type Options } from '@mikro-orm/postgresql';
import { SqlitePlatform } from '@mikro-orm/sql';
import Database from 'better-sqlite3';
import {
DummyDriver,
Kysely,
SqliteAdapter,
SqliteIntrospector,
SqliteQueryCompiler,
type PostgresDialect,
} from 'kysely';
import path from 'node:path';
/** Marks a database the probe built, so a rerun may drop what it finds there and nothing else. */
const MARKER_TABLE = 'trek_pg_probe_marker';
@@ -42,9 +55,14 @@ export function connectSetupOrm(url: string): Promise<MikroORM> {
}
/** A Postgres driver whose Kysely dialect records every statement (see recorder.ts). */
export function createProbeDriverClass(recorder: StatementRecorder, statementTimeoutMs: number): typeof PostgreSqlDriver {
export function createProbeDriverClass(
recorder: StatementRecorder,
statementTimeoutMs: number,
): typeof PostgreSqlDriver {
class ProbeConnection extends PostgreSqlConnection {
override createKyselyDialect(overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0]): PostgresDialect {
override createKyselyDialect(
overrides: Parameters<PostgreSqlConnection['createKyselyDialect']>[0],
): PostgresDialect {
return new ProbePostgresDialect(super.createKyselyDialect(overrides), recorder, statementTimeoutMs);
}
}
@@ -58,7 +76,11 @@ export function createProbeDriverClass(recorder: StatementRecorder, statementTim
};
}
export function connectProbeOrm(url: string, recorder: StatementRecorder, statementTimeoutMs: number): Promise<MikroORM> {
export function connectProbeOrm(
url: string,
recorder: StatementRecorder,
statementTimeoutMs: number,
): Promise<MikroORM> {
return MikroORM.init({ ...baseOptions(url), driver: createProbeDriverClass(recorder, statementTimeoutMs) });
}
@@ -74,7 +96,9 @@ async function resetDatabase(orm: MikroORM): Promise<void> {
);
const names = tables.map((row) => row.table_name);
if (names.length > 0 && !names.includes(MARKER_TABLE)) {
throw new Error(`refusing to run: the database holds ${names.length} table(s) the probe did not create. Point it at an empty database.`);
throw new Error(
`refusing to run: the database holds ${names.length} table(s) the probe did not create. Point it at an empty database.`,
);
}
if (names.length > 0) {
await connection.execute('drop schema public cascade');
@@ -89,7 +113,9 @@ export interface SchemaFailure {
}
/** Resets the database and builds the probe schema; returns the DDL statements Postgres refused. */
export async function prepareDatabase(orm: MikroORM): Promise<{ failures: SchemaFailure[]; adjusted: number; statements: number }> {
export async function prepareDatabase(
orm: MikroORM,
): Promise<{ failures: SchemaFailure[]; adjusted: number; statements: number }> {
await resetDatabase(orm);
const connection = orm.em.getConnection();
const failures: SchemaFailure[] = [];
@@ -128,7 +154,10 @@ export function postgresHelperEngine(orm: MikroORM): HelperEngine {
},
async selectKysely(select: KyselySelect) {
const platform = orm.em.getPlatform();
const row = await kysely.selectFrom(VALUES_TABLE).select((eb) => select(platform, eb).as('v')).executeTakeFirst();
const row = await kysely
.selectFrom(VALUES_TABLE)
.select((eb) => select(platform, eb).as('v'))
.executeTakeFirst();
return row?.v;
},
};
@@ -153,7 +182,13 @@ export function sqliteHelperEngine(): HelperEngine & { close(): void } {
platform,
async writeRow(row: ValuesRow) {
db.prepare(`delete from ${VALUES_TABLE}`).run();
db.prepare(`insert into ${VALUES_TABLE} (id, a, b, n, x) values (?, ?, ?, ?, ?)`).run(row.id, row.a, row.b, row.n, row.x);
db.prepare(`insert into ${VALUES_TABLE} (id, a, b, n, x) values (?, ?, ?, ?, ?)`).run(
row.id,
row.a,
row.b,
row.n,
row.x,
);
},
async selectRaw(select: RawSelect) {
const sql = `select ${select.sql}${select.column ? '' : ' as v'} from ${VALUES_TABLE}`;
@@ -161,7 +196,10 @@ export function sqliteHelperEngine(): HelperEngine & { close(): void } {
return row?.[select.column ?? 'v'];
},
async selectKysely(select: KyselySelect) {
const compiled = compiler.selectFrom(VALUES_TABLE).select((eb) => select(platform, eb).as('v')).compile();
const compiled = compiler
.selectFrom(VALUES_TABLE)
.select((eb) => select(platform, eb).as('v'))
.compile();
const row = db.prepare(compiled.sql).get(...compiled.parameters.map(bind)) as Record<string, unknown> | undefined;
return row?.v;
},
@@ -172,7 +210,11 @@ export function sqliteHelperEngine(): HelperEngine & { close(): void } {
type RepositoryFactory = (em: EntityManager) => object;
/** How to reach each repository class: through its entity, or `new X(em)` for the two that have none. */
function repositoryFactories(orm: MikroORM, plans: readonly MethodPlan[], serverRoot: string): Map<string, RepositoryFactory | string> {
function repositoryFactories(
orm: MikroORM,
plans: readonly MethodPlan[],
serverRoot: string,
): Map<string, RepositoryFactory | string> {
const byEntity = new Map<string, RepositoryFactory>();
for (const meta of orm.getMetadata().getAll().values()) {
const repository = meta.repository?.();
server/scripts/pg-probe/helper-cases.ts
+203 -48
View File
@@ -15,10 +15,15 @@
* Unlike the repository statements, nothing here is baselined: every case
* must pass on both engines.
*/
import type { Platform } from '@mikro-orm/core';
import type { AliasableExpression, ExpressionBuilder } from 'kysely';
import {
castIntegerKysely,
concatKysely,
currentTimestampKysely,
nowPlusSecondsKysely,
startsWithIsoDateKysely,
substringKysely,
unixEpochToIsoKysely,
} from '../../src/db/dialect/kysely-functions';
import {
absDifference,
caseWhenEquals,
@@ -53,15 +58,9 @@ import {
substring,
trim,
} from '../../src/db/dialect/sql-functions';
import {
castIntegerKysely,
concatKysely,
currentTimestampKysely,
nowPlusSecondsKysely,
startsWithIsoDateKysely,
substringKysely,
unixEpochToIsoKysely,
} from '../../src/db/dialect/kysely-functions';
import type { Platform } from '@mikro-orm/core';
import type { AliasableExpression, ExpressionBuilder } from 'kysely';
export type Engine = 'sqlite' | 'postgres';
@@ -92,7 +91,10 @@ export interface RawSelect {
column?: string;
}
export type KyselySelect = (platform: Platform, eb: ExpressionBuilder<ValuesDB, 'trek_pg_probe_values'>) => AliasableExpression<unknown>;
export type KyselySelect = (
platform: Platform,
eb: ExpressionBuilder<ValuesDB, 'trek_pg_probe_values'>,
) => AliasableExpression<unknown>;
export type Expectation =
| { equals: string | number | boolean | null }
@@ -118,7 +120,10 @@ const rawOf = (fragment: { sql: string; params: readonly unknown[] }, column?: s
});
/** `(<key> = <value>)` for the helpers used as a filter key. */
const comparison = (key: { sql: string; params: readonly unknown[] }, value: { sql: string; params: readonly unknown[] }): RawSelect => ({
const comparison = (
key: { sql: string; params: readonly unknown[] },
value: { sql: string; params: readonly unknown[] },
): RawSelect => ({
sql: `(${key.sql} = ${value.sql})`,
params: [...key.params, ...value.params],
});
@@ -133,9 +138,24 @@ export function isoSeconds(ms: number): string {
const EPOCH_MS = 1_758_459_909_123;
export const HELPER_CASES: readonly HelperCase[] = [
{ name: 'dateOf', row: { a: '2026-09-21 13:05:09' }, raw: (p) => rawOf(dateOf(p, 'a')), expect: { equals: '2026-09-21' } },
{ name: 'dateAdd forward', row: { a: '2026-09-21 13:05:09' }, raw: (p) => rawOf(dateAdd(p, 'a', 10)), expect: { equals: '2026-10-01' } },
{ name: 'dateAdd back', row: { a: '2026-09-21 13:05:09' }, raw: (p) => rawOf(dateAdd(p, 'a', -10)), expect: { equals: '2026-09-11' } },
{
name: 'dateOf',
row: { a: '2026-09-21 13:05:09' },
raw: (p) => rawOf(dateOf(p, 'a')),
expect: { equals: '2026-09-21' },
},
{
name: 'dateAdd forward',
row: { a: '2026-09-21 13:05:09' },
raw: (p) => rawOf(dateAdd(p, 'a', 10)),
expect: { equals: '2026-10-01' },
},
{
name: 'dateAdd back',
row: { a: '2026-09-21 13:05:09' },
raw: (p) => rawOf(dateAdd(p, 'a', -10)),
expect: { equals: '2026-09-11' },
},
{ name: 'currentTimestamp', row: {}, raw: (p) => rawOf(currentTimestamp(p)), expect: { timestamp: 0 } },
{ name: 'nowMinusDays', row: {}, raw: (p) => rawOf(nowMinusDays(p, 30)), expect: { timestamp: -30 * DAY_MS } },
{ name: 'nowMinusHours', row: {}, raw: (p) => rawOf(nowMinusHours(p, 20)), expect: { timestamp: -20 * 3_600_000 } },
@@ -143,28 +163,98 @@ export const HELPER_CASES: readonly HelperCase[] = [
{ name: 'nowDateOffset back', row: {}, raw: (p) => rawOf(nowDateOffset(p, -400)), expect: { date: -400 } },
{ name: 'nowDateOffset forward', row: {}, raw: (p) => rawOf(nowDateOffset(p, 1)), expect: { date: 1 } },
{ name: 'columnRef', row: { n: 7 }, raw: (p) => rawOf(columnRef(p, 'n')), expect: { equals: 7 } },
{ name: 'columnIncrementedBy', row: { n: 7 }, raw: (p) => rawOf(columnIncrementedBy(p, 'n', -2)), expect: { equals: 5 } },
{
name: 'columnIncrementedBy',
row: { n: 7 },
raw: (p) => rawOf(columnIncrementedBy(p, 'n', -2)),
expect: { equals: 5 },
},
{ name: 'lower', row: { a: 'Probe@X.com' }, raw: (p) => rawOf(lower(p, 'a')), expect: { equals: 'probe@x.com' } },
{ name: 'lower = lowerParam', row: { a: 'Probe@X.com' }, raw: (p) => comparison(lower(p, 'a'), lowerParam(p, 'PROBE@x.COM')), expect: { equals: true } },
{
name: 'lower = lowerParam',
row: { a: 'Probe@X.com' },
raw: (p) => comparison(lower(p, 'a'), lowerParam(p, 'PROBE@x.COM')),
expect: { equals: true },
},
{ name: 'trim', row: { a: ' probe ' }, raw: (p) => rawOf(trim(p, 'a')), expect: { equals: 'probe' } },
{ name: 'lowerTrim', row: { a: ' AbC ' }, raw: (p) => rawOf(lowerTrim(p, 'a')), expect: { equals: 'abc' } },
{ name: 'lowerTrim = lowerTrimParam', row: { a: ' AbC ' }, raw: (p) => comparison(lowerTrim(p, 'a'), lowerTrimParam(p, ' aBc')), expect: { equals: true } },
{ name: 'coalesce', row: { a: null, b: 'fallback' }, raw: (p) => rawOf(coalesce(p, 'a', 'b')), expect: { equals: 'fallback' } },
{ name: 'coalesceParam keeps the column', row: { a: 'stored' }, raw: (p) => rawOf(coalesceParam(p, 'a', 'new')), expect: { equals: 'stored' } },
{ name: 'coalesceParam fills a null', row: { a: null }, raw: (p) => rawOf(coalesceParam(p, 'a', 'new')), expect: { equals: 'new' } },
{ name: 'coalesceOverride prefers the value', row: { a: 'stored' }, raw: (p) => rawOf(coalesceOverride(p, 'new', 'a')), expect: { equals: 'new' } },
{ name: 'coalesceOverride keeps on null', row: { a: 'stored' }, raw: (p) => rawOf(coalesceOverride(p, null, 'a')), expect: { equals: 'stored' } },
{
name: 'lowerTrim = lowerTrimParam',
row: { a: ' AbC ' },
raw: (p) => comparison(lowerTrim(p, 'a'), lowerTrimParam(p, ' aBc')),
expect: { equals: true },
},
{
name: 'coalesce',
row: { a: null, b: 'fallback' },
raw: (p) => rawOf(coalesce(p, 'a', 'b')),
expect: { equals: 'fallback' },
},
{
name: 'coalesceParam keeps the column',
row: { a: 'stored' },
raw: (p) => rawOf(coalesceParam(p, 'a', 'new')),
expect: { equals: 'stored' },
},
{
name: 'coalesceParam fills a null',
row: { a: null },
raw: (p) => rawOf(coalesceParam(p, 'a', 'new')),
expect: { equals: 'new' },
},
{
name: 'coalesceOverride prefers the value',
row: { a: 'stored' },
raw: (p) => rawOf(coalesceOverride(p, 'new', 'a')),
expect: { equals: 'new' },
},
{
name: 'coalesceOverride keeps on null',
row: { a: 'stored' },
raw: (p) => rawOf(coalesceOverride(p, null, 'a')),
expect: { equals: 'stored' },
},
{ name: 'absDifference', row: { x: 4 }, raw: (p) => rawOf(absDifference(p, 'x', 6.5)), expect: { equals: 2.5 } },
{ name: 'countAll', row: {}, raw: (p) => rawOf(countAll(p, 'c'), 'c'), expect: { equals: 1 } },
{ name: 'countAllRef', row: {}, raw: (p) => rawOf(countAllRef(p)), expect: { equals: 1 } },
{ name: 'minOf', row: { n: 3 }, raw: (p) => rawOf(minOf(p, 'n', 'lo'), 'lo'), expect: { equals: 3 } },
{ name: 'maxOf', row: { n: 3 }, raw: (p) => rawOf(maxOf(p, 'n', 'hi'), 'hi'), expect: { equals: 3 } },
{ name: 'caseWhenEquals', row: { n: 7 }, raw: (p) => rawOf(caseWhenEquals(p, 'n', 7, 'owner', 'member')), expect: { equals: 'owner' } },
{ name: 'caseWhenEquals else', row: { n: 8 }, raw: (p) => rawOf(caseWhenEquals(p, 'n', 7, 'owner', 'member')), expect: { equals: 'member' } },
{ name: 'startsWithIsoDate yes', row: { a: '2026-09-21T10:00' }, raw: (p) => rawOf(startsWithIsoDate(p, 'a')), expect: { equals: true } },
{ name: 'startsWithIsoDate no', row: { a: 'abcd-ef-gh' }, raw: (p) => rawOf(startsWithIsoDate(p, 'a')), expect: { equals: false } },
{ name: 'substring with length', row: { a: '2026-09-21T10:00' }, raw: (p) => rawOf(substring(p, 'a', 1, 10)), expect: { equals: '2026-09-21' } },
{ name: 'substring to the end', row: { a: '2026-09-21T10:00' }, raw: (p) => rawOf(substring(p, 'a', 12)), expect: { equals: '10:00' } },
{
name: 'caseWhenEquals',
row: { n: 7 },
raw: (p) => rawOf(caseWhenEquals(p, 'n', 7, 'owner', 'member')),
expect: { equals: 'owner' },
},
{
name: 'caseWhenEquals else',
row: { n: 8 },
raw: (p) => rawOf(caseWhenEquals(p, 'n', 7, 'owner', 'member')),
expect: { equals: 'member' },
},
{
name: 'startsWithIsoDate yes',
row: { a: '2026-09-21T10:00' },
raw: (p) => rawOf(startsWithIsoDate(p, 'a')),
expect: { equals: true },
},
{
name: 'startsWithIsoDate no',
row: { a: 'abcd-ef-gh' },
raw: (p) => rawOf(startsWithIsoDate(p, 'a')),
expect: { equals: false },
},
{
name: 'substring with length',
row: { a: '2026-09-21T10:00' },
raw: (p) => rawOf(substring(p, 'a', 1, 10)),
expect: { equals: '2026-09-21' },
},
{
name: 'substring to the end',
row: { a: '2026-09-21T10:00' },
raw: (p) => rawOf(substring(p, 'a', 12)),
expect: { equals: '10:00' },
},
{
name: 'concat',
row: { a: '2026-09-21', b: '10:00' },
@@ -173,15 +263,30 @@ export const HELPER_CASES: readonly HelperCase[] = [
},
{ name: 'castInteger of 14.0', row: { a: '14.0' }, raw: (p) => rawOf(castInteger(p, 'a')), expect: { equals: 14 } },
{ name: 'castInteger of 14', row: { a: '14' }, raw: (p) => rawOf(castInteger(p, 'a')), expect: { equals: 14 } },
{ name: 'dayDistance', row: { a: '2026-09-21' }, raw: (p) => rawOf(dayDistance(p, 'a', '2026-09-24')), expect: { equals: 3 } },
{ name: 'collateNoCase', row: { a: 'Probe' }, raw: (p) => comparison(collateNoCase(p, 'a'), bound('pROBE')), expect: { equals: true } },
{
name: 'dayDistance',
row: { a: '2026-09-21' },
raw: (p) => rawOf(dayDistance(p, 'a', '2026-09-24')),
expect: { equals: 3 },
},
{
name: 'collateNoCase',
row: { a: 'Probe' },
raw: (p) => comparison(collateNoCase(p, 'a'), bound('pROBE')),
expect: { equals: true },
},
{
name: 'foundAgainState recovers',
row: { a: 'remote_missing', b: 'file-1' },
raw: (p) => rawOf(foundAgainState(p, 'a', 'b')),
expect: { equals: 'synced' },
},
{ name: 'foundAgainState passes through', row: { a: 'error', b: 'file-1' }, raw: (p) => rawOf(foundAgainState(p, 'a', 'b')), expect: { equals: 'error' } },
{
name: 'foundAgainState passes through',
row: { a: 'error', b: 'file-1' },
raw: (p) => rawOf(foundAgainState(p, 'a', 'b')),
expect: { equals: 'error' },
},
{
name: 'coalesceOverrideWhileSame keeps for the same key',
row: { a: 'https://a', n: 1 },
@@ -194,24 +299,61 @@ export const HELPER_CASES: readonly HelperCase[] = [
raw: (p) => rawOf(coalesceOverrideWhileSame(p, null, 'n', 'a', 'https://b')),
expect: { equals: 0 },
},
{ name: 'caseWhenNotNull', row: { a: 'x', b: 'route' }, raw: (p) => rawOf(caseWhenNotNull(p, 'a', 'b')), expect: { equals: 'route' } },
{ name: 'caseWhenNotNull null', row: { a: null, b: 'route' }, raw: (p) => rawOf(caseWhenNotNull(p, 'a', 'b')), expect: { equals: null } },
{
name: 'caseWhenNotNull',
row: { a: 'x', b: 'route' },
raw: (p) => rawOf(caseWhenNotNull(p, 'a', 'b')),
expect: { equals: 'route' },
},
{
name: 'caseWhenNotNull null',
row: { a: null, b: 'route' },
raw: (p) => rawOf(caseWhenNotNull(p, 'a', 'b')),
expect: { equals: null },
},
{
name: 'startsWithIsoDateKysely',
row: { a: '2026-09-21 10:00:00' },
kysely: (p, eb) => eb.case().when(startsWithIsoDateKysely(p, eb, 'a')).then('yes').else('no').end(),
kysely: (p, eb) =>
eb
.case()
.when(startsWithIsoDateKysely(p, eb, 'a'))
.then('yes')
.else('no')
.end(),
expect: { equals: 'yes' },
},
{ name: 'substringKysely', row: { a: '2026-09-21T10:00' }, kysely: (p, eb) => substringKysely(p, eb, 'a', 1, 10), expect: { equals: '2026-09-21' } },
{
name: 'substringKysely',
row: { a: '2026-09-21T10:00' },
kysely: (p, eb) => substringKysely(p, eb, 'a', 1, 10),
expect: { equals: '2026-09-21' },
},
{
name: 'concatKysely',
row: { a: '2026-09-21', b: 'xx10:00' },
kysely: (p, eb) => concatKysely(p, eb, { column: 'a' }, { value: 'T' }, { expression: substringKysely(p, eb, 'b', 3) }),
kysely: (p, eb) =>
concatKysely(p, eb, { column: 'a' }, { value: 'T' }, { expression: substringKysely(p, eb, 'b', 3) }),
expect: { equals: '2026-09-21T10:00' },
},
{ name: 'castIntegerKysely', row: { a: '14.0' }, kysely: (p, eb) => castIntegerKysely(p, eb, 'a'), expect: { equals: 14 } },
{ name: 'unixEpochToIsoKysely', row: { n: EPOCH_MS }, kysely: (p, eb) => unixEpochToIsoKysely(p, eb, 'n'), expect: { equals: isoSeconds(EPOCH_MS) } },
{ name: 'nowPlusSecondsKysely', row: {}, kysely: (p, eb) => nowPlusSecondsKysely(p, eb, 45), expect: { timestamp: 45_000 } },
{
name: 'castIntegerKysely',
row: { a: '14.0' },
kysely: (p, eb) => castIntegerKysely(p, eb, 'a'),
expect: { equals: 14 },
},
{
name: 'unixEpochToIsoKysely',
row: { n: EPOCH_MS },
kysely: (p, eb) => unixEpochToIsoKysely(p, eb, 'n'),
expect: { equals: isoSeconds(EPOCH_MS) },
},
{
name: 'nowPlusSecondsKysely',
row: {},
kysely: (p, eb) => nowPlusSecondsKysely(p, eb, 45),
expect: { timestamp: 45_000 },
},
{ name: 'currentTimestampKysely', row: {}, kysely: (p) => currentTimestampKysely(p), expect: { timestamp: 0 } },
];
@@ -222,14 +364,17 @@ const DATE_TEXT = /^\d{4}-\d{2}-\d{2}$/;
export function checkValue(expectation: Expectation, value: unknown, engine: Engine, now: number): string | null {
if ('perEngine' in expectation) return checkValue(expectation.perEngine[engine], value, engine, now);
if ('timestamp' in expectation) {
if (typeof value !== 'string' || !TIMESTAMP_TEXT.test(value)) return `expected timestamp text, got ${JSON.stringify(value)}`;
if (typeof value !== 'string' || !TIMESTAMP_TEXT.test(value))
return `expected timestamp text, got ${JSON.stringify(value)}`;
const off = Date.parse(`${value.replace(' ', 'T')}Z`) - (now + expectation.timestamp);
return Math.abs(off) <= CLOCK_TOLERANCE_MS ? null : `${value} is ${Math.round(off / 1000)}s off`;
}
if ('date' in expectation) {
if (typeof value !== 'string' || !DATE_TEXT.test(value)) return `expected date text, got ${JSON.stringify(value)}`;
const wanted = Date.parse(new Date(now + expectation.date * DAY_MS).toISOString().slice(0, 10));
return Math.abs(Date.parse(value) - wanted) <= DAY_MS ? null : `${value} is not ${expectation.date} day(s) from today`;
return Math.abs(Date.parse(value) - wanted) <= DAY_MS
? null
: `${value} is not ${expectation.date} day(s) from today`;
}
const expected = expectation.equals;
if (expected === null) return value === null ? null : `expected null, got ${JSON.stringify(value)}`;
@@ -239,8 +384,15 @@ export function checkValue(expectation: Expectation, value: unknown, engine: Eng
return (expected ? truthy : falsy) ? null : `expected ${expected}, got ${JSON.stringify(value)}`;
}
if (typeof expected === 'number') {
const n = typeof value === 'bigint' ? Number(value) : typeof value === 'string' || typeof value === 'number' ? Number(value) : Number.NaN;
return Math.abs(n - expected) < 1e-9 ? null : `expected ${expected}, got ${JSON.stringify(typeof value === 'bigint' ? String(value) : value)}`;
const n =
typeof value === 'bigint'
? Number(value)
: typeof value === 'string' || typeof value === 'number'
? Number(value)
: Number.NaN;
return Math.abs(n - expected) < 1e-9
? null
: `expected ${expected}, got ${JSON.stringify(typeof value === 'bigint' ? String(value) : value)}`;
}
return value === expected ? null : `expected ${JSON.stringify(expected)}, got ${JSON.stringify(value)}`;
}
@@ -260,7 +412,10 @@ export interface HelperResult {
failure: string | null;
}
export async function runHelperCases(engine: HelperEngine, cases: readonly HelperCase[] = HELPER_CASES): Promise<HelperResult[]> {
export async function runHelperCases(
engine: HelperEngine,
cases: readonly HelperCase[] = HELPER_CASES,
): Promise<HelperResult[]> {
const results: HelperResult[] = [];
for (const helperCase of cases) {
let failure: string | null;
server/scripts/pg-probe/recorder.ts
+7 -5
View File
@@ -21,9 +21,6 @@
* AsyncLocalStorage, so a straggler from a timed-out method can never be
* counted against the next one.
*/
import { AsyncLocalStorage } from 'node:async_hooks';
import {
CompiledQuery,
type DatabaseConnection,
@@ -35,6 +32,7 @@ import {
type QueryCompiler,
type QueryResult,
} from 'kysely';
import { AsyncLocalStorage } from 'node:async_hooks';
export type StatementOutcome = { ok: true } | { ok: false; code: string; message: string };
@@ -44,7 +42,8 @@ export interface StatementRecord {
}
/** Transaction control a probed method sends; each statement already runs in its own rolled-back transaction. */
const TRANSACTION_CONTROL = /^\s*(begin|commit|rollback|end|abort|start\s+transaction|savepoint|release(\s+savepoint)?)\b/i;
const TRANSACTION_CONTROL =
/^\s*(begin|commit|rollback|end|abort|start\s+transaction|savepoint|release(\s+savepoint)?)\b/i;
export function isTransactionControl(sql: string): boolean {
return TRANSACTION_CONTROL.test(sql);
@@ -114,7 +113,10 @@ export class ProbeConnection implements DatabaseConnection {
this.recorder.record(key, { sql: compiled.sql, outcome: { ok: true } });
return result;
} catch (error) {
this.recorder.record(key, { sql: compiled.sql, outcome: { ok: false, code: sqlState(error), message: errorLine(error) } });
this.recorder.record(key, {
sql: compiled.sql,
outcome: { ok: false, code: sqlState(error), message: errorLine(error) },
});
return { rows: [] };
} finally {
await this.inner.executeQuery(CompiledQuery.raw('rollback'));
server/scripts/pg-probe/report.ts
+64 -21
View File
@@ -2,7 +2,6 @@
* Turns what the Postgres probe saw into counts, a console log and the
* Markdown the CI job writes to its step summary.
*/
import { countsTowardRatchet, type BaselineVerdict } from './baseline';
import type { HelperResult } from './helper-cases';
import type { StatementRecord } from './recorder';
@@ -77,7 +76,10 @@ export function summarize(results: readonly MethodResult[]): ProbeSummary {
summary.timedOut += 1;
summary.uncovered.set(result.key, 'timed out');
} else if (result.statements.length === 0) {
summary.uncovered.set(result.key, result.threw === undefined ? 'sent no SQL' : `threw before any statement: ${result.threw}`);
summary.uncovered.set(
result.key,
result.threw === undefined ? 'sent no SQL' : `threw before any statement: ${result.threw}`,
);
}
if (result.statements.length === 0) summary.withoutSql += 1;
const seen = new Set<string>();
@@ -93,7 +95,13 @@ export function summarize(results: readonly MethodResult[]): ProbeSummary {
if (!('code' in outcome)) continue;
const dialect = countsTowardRatchet(outcome.code);
if (dialect) counted += 1;
summary.failedStatements.push({ method: result.key, code: outcome.code, message: outcome.message, sql: statement.sql, counted: dialect });
summary.failedStatements.push({
method: result.key,
code: outcome.code,
message: outcome.message,
sql: statement.sql,
counted: dialect,
});
summary.failuresByCode.set(outcome.code, (summary.failuresByCode.get(outcome.code) ?? 0) + 1);
}
summary.failingByMethod.set(result.key, counted);
@@ -102,7 +110,10 @@ export function summarize(results: readonly MethodResult[]): ProbeSummary {
}
/** Refused statements the ratchet holds (dialect errors) and the ones it only reports. */
export function splitRefusals(summary: Pick<ProbeSummary, 'failedStatements'>): { counted: FailedStatement[]; reported: FailedStatement[] } {
export function splitRefusals(summary: Pick<ProbeSummary, 'failedStatements'>): {
counted: FailedStatement[];
reported: FailedStatement[];
} {
return {
counted: summary.failedStatements.filter((failed) => failed.counted),
reported: summary.failedStatements.filter((failed) => !failed.counted),
@@ -163,11 +174,16 @@ export function formatConsole(input: ReportInput): string {
const { summary, verdict } = input;
const lines: string[] = [];
const failures = helperFailures(input.helpers);
lines.push(`Dialect helpers: ${input.helpers.length - failures.length}/${input.helpers.length} cases pass on SQLite and Postgres.`);
lines.push(
`Dialect helpers: ${input.helpers.length - failures.length}/${input.helpers.length} cases pass on SQLite and Postgres.`,
);
for (const failure of failures) lines.push(` FAIL [${failure.engine}] ${failure.name}: ${failure.failure}`);
if (input.schemaFailures.length > 0) {
lines.push(`Schema: ${input.schemaFailures.length} DDL statement(s) Postgres refused (their tables are missing below):`);
for (const failure of input.schemaFailures) lines.push(` ${failure.message} <- ${oneLine(failure.statement, 120)}`);
lines.push(
`Schema: ${input.schemaFailures.length} DDL statement(s) Postgres refused (their tables are missing below):`,
);
for (const failure of input.schemaFailures)
lines.push(` ${failure.message} <- ${oneLine(failure.statement, 120)}`);
}
lines.push(
`Repositories: ${summary.methods} methods, ${summary.called} called, ${summary.unprobeable} unprobeable, ` +
@@ -187,10 +203,13 @@ export function formatConsole(input: ReportInput): string {
}
if (summary.uncovered.size > 0) {
lines.push(`Unmeasured: ${summary.uncovered.size} method(s) whose SQL the probe did not see:`);
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) lines.push(` ${method}: ${why}`);
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b)))
lines.push(` ${method}: ${why}`);
}
if (probedNothing(summary)) {
lines.push(`FAIL No repository method was called (${summary.methods} planned); there is nothing to hold to the baseline.`);
lines.push(
`FAIL No repository method was called (${summary.methods} planned); there is nothing to hold to the baseline.`,
);
}
if (verdict.unseeded) {
lines.push(
@@ -202,7 +221,9 @@ export function formatConsole(input: ReportInput): string {
lines.push(`FAIL ${entry.method} sends ${entry.now} statement(s) Postgres refuses, ${entry.allowed} allowed.`);
}
for (const entry of verdict.stale) {
lines.push(`FAIL ${entry.method} is held at ${entry.allowed} failing statement(s) but fails ${entry.now} now; lower the baseline.`);
lines.push(
`FAIL ${entry.method} is held at ${entry.allowed} failing statement(s) but fails ${entry.now} now; lower the baseline.`,
);
}
for (const method of verdict.newlyUncovered) {
lines.push(
@@ -223,16 +244,21 @@ export function formatMarkdown(input: ReportInput): string {
const out: string[] = ['## Postgres probe', ''];
out.push(passed(input) ? 'Result: **passed**' : 'Result: **failed**', '');
out.push('| | |', '|---|---|');
out.push(`| Dialect helper cases (SQLite and Postgres) | ${input.helpers.length - failures.length} of ${input.helpers.length} pass |`);
out.push(
`| Dialect helper cases (SQLite and Postgres) | ${input.helpers.length - failures.length} of ${input.helpers.length} pass |`,
);
out.push(`| Schema statements refused | ${input.schemaFailures.length} |`);
const refusals = splitRefusals(summary);
out.push(`| Repository methods | ${summary.methods} (${summary.called} called, ${summary.unprobeable} unprobeable) |`);
out.push(
`| Repository methods | ${summary.methods} (${summary.called} called, ${summary.unprobeable} unprobeable) |`,
);
out.push(`| Methods whose SQL the probe did not see | ${summary.uncovered.size} |`);
out.push(`| Distinct statements | ${summary.statements} |`);
out.push(`| Refused by Postgres as SQL (held) | ${refusals.counted.length} |`);
out.push(`| Refused over a sample value or the empty tables (reported only) | ${refusals.reported.length} |`);
out.push('');
if (probedNothing(summary)) out.push('No repository method was called, so there is nothing to hold to the baseline.', '');
if (probedNothing(summary))
out.push('No repository method was called, so there is nothing to hold to the baseline.', '');
if (verdict.unseeded) {
out.push(
'The baseline has not been measured yet, which fails the run. Commit `pg-probe-baseline.json` from the `pg-probe-baseline` artifact as `server/scripts/pg-probe-baseline.json`.',
@@ -241,33 +267,50 @@ export function formatMarkdown(input: ReportInput): string {
}
if (verdict.grown.length > 0 || verdict.stale.length > 0) {
out.push('### Ratchet', '', '| Method | Allowed | Now |', '|---|---|---|');
for (const entry of [...verdict.grown, ...verdict.stale]) out.push(`| ${cell(entry.method)} | ${entry.allowed} | ${entry.now} |`);
for (const entry of [...verdict.grown, ...verdict.stale])
out.push(`| ${cell(entry.method)} | ${entry.allowed} | ${entry.now} |`);
out.push('');
}
if (verdict.newlyUncovered.length > 0 || verdict.nowCovered.length > 0) {
out.push('### Coverage ratchet', '', '| Method | Baseline | Now |', '|---|---|---|');
for (const method of verdict.newlyUncovered) out.push(`| ${cell(method)} | measured | ${cell(summary.uncovered.get(method) ?? 'unmeasured')} |`);
for (const method of verdict.newlyUncovered)
out.push(`| ${cell(method)} | measured | ${cell(summary.uncovered.get(method) ?? 'unmeasured')} |`);
for (const method of verdict.nowCovered) out.push(`| ${cell(method)} | uncovered | measured or gone |`);
out.push('');
}
if (failures.length > 0) {
out.push('### Helper cases that failed', '', '| Engine | Case | Why |', '|---|---|---|');
for (const failure of failures) out.push(`| ${failure.engine} | ${cell(failure.name)} | ${cell(failure.failure ?? '')} |`);
for (const failure of failures)
out.push(`| ${failure.engine} | ${cell(failure.name)} | ${cell(failure.failure ?? '')} |`);
out.push('');
}
if (summary.failuresByCode.size > 0) {
out.push('### Refused statements by SQLSTATE', '', '| SQLSTATE | Statements |', '|---|---|');
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1])) out.push(`| ${code} | ${count} |`);
for (const [code, count] of [...summary.failuresByCode.entries()].sort((a, b) => b[1] - a[1]))
out.push(`| ${code} | ${count} |`);
out.push('');
out.push('<details><summary>Every refused statement</summary>', '', '| Method | SQLSTATE | Held | Message |', '|---|---|---|---|');
out.push(
'<details><summary>Every refused statement</summary>',
'',
'| Method | SQLSTATE | Held | Message |',
'|---|---|---|---|',
);
for (const failed of summary.failedStatements) {
out.push(`| ${cell(failed.method)} | ${failed.code} | ${failed.counted ? 'yes' : 'no'} | ${cell(failed.message)} |`);
out.push(
`| ${cell(failed.method)} | ${failed.code} | ${failed.counted ? 'yes' : 'no'} | ${cell(failed.message)} |`,
);
}
out.push('', '</details>', '');
}
if (summary.uncovered.size > 0) {
out.push('<details><summary>Methods whose SQL the probe did not see</summary>', '', '| Method | Why |', '|---|---|');
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b))) out.push(`| ${cell(method)} | ${cell(why)} |`);
out.push(
'<details><summary>Methods whose SQL the probe did not see</summary>',
'',
'| Method | Why |',
'|---|---|',
);
for (const [method, why] of [...summary.uncovered.entries()].sort(([a], [b]) => a.localeCompare(b)))
out.push(`| ${cell(method)} | ${cell(why)} |`);
out.push('', '</details>', '');
}
return `${out.join('\n')}\n`;
server/scripts/pg-probe/schema.ts
+6 -3
View File
@@ -19,7 +19,6 @@
*
* Unique and partial unique indexes stay, because `ON CONFLICT` needs them.
*/
import { pgTimestampText, PG_UTC_NOW } from '../../src/db/dialect/platform';
/** SQLite's timestamp text on Postgres, for a column default. */
@@ -31,9 +30,13 @@ export const NOCASE_COLLATION_SQL =
/** The Postgres spelling of a column default the entities carry in SQLite's. */
export function portableDefault(raw: string): string {
const text = raw.trim().replace(/^\((.*)\)$/s, '$1').trim();
const text = raw
.trim()
.replace(/^\((.*)\)$/s, '$1')
.trim();
if (/^current_timestamp$/i.test(text) || /^datetime\(\s*'now'\s*\)$/i.test(text)) return PG_NOW_TEXT_DEFAULT;
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text)) return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
if (/^strftime\(\s*'%s'\s*,\s*'now'\s*\)$/i.test(text))
return 'CAST(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) AS bigint)';
return raw;
}
server/scripts/query-api-ratchet.mjs
+6 -2
View File
@@ -138,13 +138,17 @@ function main(argv) {
console.error('Run npm run lint:query-api -- --update to lower the baseline with the change that made it smaller.');
}
const total = Object.values(counts).reduce((sum, n) => sum + n, 0);
console.log(`query-api: ${total} QueryBuilder call(s) in ${Object.keys(counts).length} file(s) held at their baseline`);
console.log(
`query-api: ${total} QueryBuilder call(s) in ${Object.keys(counts).length} file(s) held at their baseline`,
);
return grown.length || stale.length ? 1 : 0;
}
// Compared by real path, so the check still runs when the script is started through a symlink.
const isCli =
Boolean(process.argv[1]) && existsSync(process.argv[1]) && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));
server/scripts/size-lint.mjs
+5 -3
View File
@@ -33,8 +33,6 @@
* Walking the tree and handling the baseline live in scripts/lib/ratchet.mjs
* at the repository root.
*/
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
listFiles,
lowerCounts,
@@ -47,6 +45,9 @@ import {
writeBaseline,
} from '../../scripts/lib/ratchet.mjs';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
/** The most lines a source file without a baseline entry may hold. */
const LIMIT = 1000;
@@ -112,7 +113,8 @@ const sizeEntries = (limit) => (value) => {
return 'it must be an object of file paths to line counts';
}
for (const [file, n] of Object.entries(value)) {
if (!Number.isInteger(n) || n <= limit) return `${file} holds ${JSON.stringify(n)}, expected an integer above ${limit}`;
if (!Number.isInteger(n) || n <= limit)
return `${file} holds ${JSON.stringify(n)}, expected an integer above ${limit}`;
}
return null;
};
server/scripts/strict-lint.mjs
+15 -10
View File
@@ -24,11 +24,6 @@
* use both). The counting lives in scripts/lib/tsc-errors.mjs, the baseline
* handling in scripts/lib/ratchet.mjs at the repository root.
*/
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import {
countMap,
lowerCounts,
@@ -40,15 +35,25 @@ import {
} from '../../scripts/lib/ratchet.mjs';
import { countErrors } from './lib/tsc-errors.mjs';
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
/** tsc over tsconfig.strict.json, as its output and exit status. */
function runTsc(serverDir) {
const tsc = createRequire(join(serverDir, 'package.json')).resolve('typescript/bin/tsc');
try {
const output = execFileSync(process.execPath, [tsc, '-p', 'tsconfig.strict.json', '--noEmit', '--pretty', 'false'], {
cwd: serverDir,
encoding: 'utf8',
maxBuffer: 256 * 1024 * 1024,
});
const output = execFileSync(
process.execPath,
[tsc, '-p', 'tsconfig.strict.json', '--noEmit', '--pretty', 'false'],
{
cwd: serverDir,
encoding: 'utf8',
maxBuffer: 256 * 1024 * 1024,
},
);
return { output, status: 0 };
} catch (err) {
if (typeof err.status !== 'number') throw err;
server/scripts/test-new-service-ratchet.mjs
+2 -1
View File
@@ -24,9 +24,10 @@
* npm run lint:test-new-service check against the baseline (CI)
* npm run lint:test-new-service -- --update lower the counts; never raises or adds an entry
*/
import { cliMain, stripComments } from './lib/count-ratchet.mjs';
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
import { cliMain, stripComments } from './lib/count-ratchet.mjs';
const CLASS_DECL = /\bclass\s+([A-Za-z_$][\w$]*)(?:\s*<[^{]*?>)?(?:\s+extends\s+([A-Za-z_$][\w$.]*))?[^{]*\{/g;
server/scripts/test-sql-ratchet.mjs
+3 -1
View File
@@ -225,7 +225,9 @@ function main(argv) {
// Compared by real path, so the check still runs when the script is started through a symlink.
const isCli =
Boolean(process.argv[1]) && existsSync(process.argv[1]) && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));
server/scripts/tx-writes.mjs
+37 -11
View File
@@ -266,26 +266,41 @@ export function analyse(classes) {
return true;
}
const first = call.arguments[0];
return Boolean(first && (ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first)) && RAW_WRITE.test(first.text));
return Boolean(
first && (ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first)) && RAW_WRITE.test(first.text),
);
}
/** Write units of `node`, read inside `cls`. `covered` is true inside a transactional callback. */
function count(cls, node, covered) {
if (!node) return 0;
if (ts.isFunctionDeclaration(node) || ts.isClassDeclaration(node)) return 0;
if (ts.isBlock(node) || ts.isSourceFile(node) || ts.isModuleBlock(node)) return statements(cls, node.statements, 0, covered);
if (ts.isBlock(node) || ts.isSourceFile(node) || ts.isModuleBlock(node))
return statements(cls, node.statements, 0, covered);
if (ts.isIfStatement(node)) {
return count(cls, node.expression, covered) + Math.max(count(cls, node.thenStatement, covered), count(cls, node.elseStatement, covered));
return (
count(cls, node.expression, covered) +
Math.max(count(cls, node.thenStatement, covered), count(cls, node.elseStatement, covered))
);
}
if (ts.isConditionalExpression(node)) {
return count(cls, node.condition, covered) + Math.max(count(cls, node.whenTrue, covered), count(cls, node.whenFalse, covered));
return (
count(cls, node.condition, covered) +
Math.max(count(cls, node.whenTrue, covered), count(cls, node.whenFalse, covered))
);
}
if (ts.isSwitchStatement(node)) {
let max = 0;
for (const clause of node.caseBlock.clauses) max = Math.max(max, statements(cls, clause.statements, 0, covered));
return count(cls, node.expression, covered) + max;
}
if (ts.isForStatement(node) || ts.isForOfStatement(node) || ts.isForInStatement(node) || ts.isWhileStatement(node) || ts.isDoStatement(node)) {
if (
ts.isForStatement(node) ||
ts.isForOfStatement(node) ||
ts.isForInStatement(node) ||
ts.isWhileStatement(node) ||
ts.isDoStatement(node)
) {
let head = 0;
if (ts.isForStatement(node)) head = count(cls, node.initializer, covered);
else if (!ts.isDoStatement(node) && !ts.isWhileStatement(node)) head = count(cls, node.expression, covered);
@@ -325,7 +340,9 @@ export function analyse(classes) {
function call(cls, node, covered) {
const callee = node.expression;
const method = ts.isPropertyAccessExpression(callee) ? callee.name.text : null;
const receiver = ts.isPropertyAccessExpression(callee) ? count(cls, callee.expression, covered) : count(cls, callee, covered);
const receiver = ts.isPropertyAccessExpression(callee)
? count(cls, callee.expression, covered)
: count(cls, callee, covered);
if (method === 'transactional') {
let inner = 0;
frames.push([]); // what runs inside is one unit, not a list of its own
@@ -381,7 +398,7 @@ export function analyse(classes) {
// transactional callback or comes from a method that is bound itself. Grown from
// nothing to a fixed point, so a cycle of methods calling each other binds none.
const bound = new Set();
for (let grew = true; grew; ) {
for (let grew = true; grew;) {
grew = false;
for (const [key, list] of callSites) {
if (bound.has(key) || list.length === 0) continue;
@@ -404,7 +421,10 @@ export function analyse(classes) {
explained[`${cls.file}#${key}`] = sites.get(key);
}
}
return { found: Object.fromEntries(Object.entries(reported).sort(([a], [b]) => a.localeCompare(b))), sites: explained };
return {
found: Object.fromEntries(Object.entries(reported).sort(([a], [b]) => a.localeCompare(b))),
sites: explained,
};
}
/** The baseline as committed. Missing or malformed stops the run instead of reading as empty. */
@@ -420,7 +440,9 @@ export function readBaseline(path) {
}
for (const [key, n] of Object.entries(parsed)) {
if (!Number.isInteger(n) || n < 2) {
throw new Error(`scripts/tx-writes-baseline.json: ${key} holds ${JSON.stringify(n)}, expected an integer of at least 2`);
throw new Error(
`scripts/tx-writes-baseline.json: ${key} holds ${JSON.stringify(n)}, expected an integer of at least 2`,
);
}
}
return parsed;
@@ -479,12 +501,16 @@ function main(argv) {
if (stale.length) {
console.error('Run npm run lint:tx -- --update to lower the baseline with the change that fixed it.');
}
console.log(`tx: ${Object.keys(found).length} method(s) with writes outside one transaction, ${Object.keys(baseline).length} held at their baseline`);
console.log(
`tx: ${Object.keys(found).length} method(s) with writes outside one transaction, ${Object.keys(baseline).length} held at their baseline`,
);
return grown.length || stale.length ? 1 : 0;
}
const isCli =
Boolean(process.argv[1]) && existsSync(process.argv[1]) && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));