Files
TREK/server/scripts/dialect-lint.mjs
T
Maurice 80fca10784 feat(db): hold migrations after the Postgres cut to engineDdl in lint:dialect
A migration newer than the baseline's through (read from
assets/postgres/baseline.json) fails on a schema statement outside engineDdl,
a plain string column for addColumnIfMissing or a SQLite-only helper, and on a
SQLite spelling in an engineDdl postgres member. The advice names the two
steps, and the baseline cannot hold an allowance for these rules.
2026-11-17 23:03:49 +01:00

479 lines
19 KiB
JavaScript

#!/usr/bin/env node
/*
* lint:dialect: SQLite-only SQL may only shrink.
*
* The server runs on SQLite today and is meant to run on Postgres next. What
* stands in the way is SQL that only SQLite understands, spelled out inline:
* INSERT OR IGNORE, datetime('now'), strftime, json_extract, GLOB,
* last_insert_rowid, PRAGMA, AUTOINCREMENT and `||` string concatenation.
* Each of them has a dialect-neutral form: the query builder's onConflict(),
* returning('id'), or a helper in src/db/dialect/ (sql-functions.ts and its
* Kysely twin kysely-functions.ts), which dispatches on the live platform
* and is the one place allowed to spell an engine's own forms.
*
* The check reads every string and template literal under src/ (comments
* never count) and every `.insertId` read, counts the hits per file and rule,
* and holds them to scripts/dialect-baseline.json. A new hit fails, and so
* does a baseline entry above what its file holds now, so the change that
* removes one lowers the baseline with --update in the same commit.
*
* Not scanned: src/db/dialect/ (the dialect layer itself) and
* the migrations that have shipped (append-only, never edited again; the last
* one is FROZEN_MIGRATIONS_THROUGH). A newer migration is scanned for the
* DML rules. PRAGMA is allowed in the src/db/ lifecycle files (src/db/*.ts
* and the migrations), nowhere else.
*
* A migration after the Postgres baseline's cut (`through` in
* assets/postgres/baseline.json) runs on both engines, so its schema
* statements are held too (POST_CUT_RULES): each one goes through
* engineDdl(this, { sqlite: [...], postgres: [...] }), each column handed to
* addColumnIfMissing is { sqlite, postgres }, the SQLite-only helpers of
* migration-utils are not called, and a postgres member holds no SQLite
* spelling. These rules have no allowance: the baseline file may not hold an
* entry for them.
*
* npm run lint:dialect check against the baseline (CI)
* npm run lint:dialect -- --update lower the baseline to what the files hold now;
* it never raises or adds an entry
*
* --dir=<path> points the check at another server root (the unit tests use it).
*/
import { existsSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, relative, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const require = createRequire(import.meta.url);
const ts = require('typescript');
/** The last migration that shipped before this check existed. Later ones are scanned. */
export const FROZEN_MIGRATIONS_THROUGH = 'Migration20200101042400';
/** Where the cut is read from: the migrations after its `through` run on Postgres too. */
export const POSTGRES_BASELINE_MANIFEST = 'assets/postgres/baseline.json';
/** The dialect layer: the one place that may spell an engine's own SQL. */
const DIALECT_LAYER = 'src/db/dialect/';
const SQL_KEYWORDS = /\b(SELECT|UPDATE|INSERT|DELETE|WHERE|SET|CASE|WHEN|COALESCE|CAST|VALUES|FROM)\b/i;
/**
* Each rule: a name, a test over one literal's text, the advice the failure
* prints, and whether it applies to a migration newer than the frozen ones.
*/
export const RULES = [
{
name: 'insert-or',
test: (text) => /\bINSERT\s+OR\s+(IGNORE|REPLACE)\b|\bREPLACE\s+INTO\b/i.test(text),
advice: "use the query builder's onConflict((oc) => oc.columns([...]).doNothing() / .doUpdateSet(...))",
migrations: true,
},
{
name: 'datetime',
// The call in SQL text, or the bare name the query builder takes in eb.fn('datetime', [...]).
test: (text) =>
/\b(datetime|julianday|unixepoch)\s*\(/i.test(text) || /^(datetime|julianday|unixepoch)$/i.test(text.trim()),
advice: 'use currentTimestamp/nowMinusDays/nowPlusSeconds and friends from sql-functions.ts',
migrations: true,
},
{
name: 'strftime',
test: (text) => /\bstrftime\b/i.test(text),
advice: 'use unixEpochToIsoKysely or another sql-functions.ts helper',
migrations: true,
},
{
name: 'json',
test: (text) => /\bjson_(extract|each|set|patch|insert|replace|remove|group_array|group_object)\b/i.test(text),
advice: 'read the JSON column and parse it in the service, or add a helper to sql-functions.ts',
migrations: true,
},
{
name: 'glob',
test: (text) => /\bGLOB\b/.test(text) || text.trim().toLowerCase() === 'glob',
advice: 'use LIKE, or startsWithIsoDate and its siblings in sql-functions.ts',
migrations: true,
},
{
name: 'last-insert-rowid',
test: (text) => /\blast_insert_rowid\b/i.test(text),
advice: "read the new id with .returning('id')",
migrations: true,
},
{
name: 'autoincrement',
test: (text) => /\bAUTOINCREMENT\b/i.test(text),
advice: 'tables are created by migrations, not by query-builder code',
migrations: false,
},
{
name: 'pragma',
// The statement, `PRAGMA name`: the bare word is also an HTTP header.
test: (text) => /\bPRAGMA\s+[a-z_]/i.test(text),
advice: 'PRAGMA belongs in the src/db/ lifecycle files (connection, durability, migrations)',
migrations: false,
},
{
name: 'concat',
test: (text, inSqlTag) => text.includes('||') && (inSqlTag || SQL_KEYWORDS.test(text)),
advice: 'use concat/concatKysely from sql-functions.ts',
migrations: true,
},
];
/** `.insertId` reads, the JS face of last_insert_rowid. */
const INSERT_ID_RULE = 'insert-id';
/** What every post-cut rule's advice ends with: the two steps that satisfy it. */
const POST_CUT_ROUTINE =
'(1) write the schema change as engineDdl(this, { sqlite: [...], postgres: [] }) from src/db/migration-utils.ts ' +
"with the SQLite statements by hand (a column for addColumnIfMissing as { sqlite: '...', postgres: '...' }); " +
'(2) fill postgres with what npm run gen:pg-migration prints (server/CLAUDE.md, SQL dialect, Migrations)';
/** A schema statement: CREATE/ALTER/DROP of a table, index, trigger or view, or a RENAME. */
const DDL_STATEMENT =
/\b(?:CREATE|ALTER|DROP)\s+(?:UNIQUE\s+)?(?:TEMP(?:ORARY)?\s+)?(?:TABLE|INDEX|TRIGGER|VIEW)\b|\bRENAME\s+(?:TO|COLUMN)\b/i;
/** SQLite's own column types and clauses, which Postgres refuses or reads differently. */
const SQLITE_ONLY_DDL = /\bDATETIME\b|\bAUTOINCREMENT\b|\bINTEGER\s+PRIMARY\s+KEY\b|\bWITHOUT\s+ROWID\b|\bPRAGMA\b/i;
/** The migration-utils helpers that only speak SQLite. */
const SQLITE_ONLY_HELPERS = new Set(['tableSql', 'createPlaceRegionsFollowPlaceTrigger']);
/** The calls whose { sqlite, postgres } argument holds the DDL of each engine. */
const ENGINE_DDL_CALLS = new Set(['engineDdl', 'addColumnIfMissing']);
/**
* The rules for a migration after the Postgres baseline's cut. A new
* migration starts at zero: the baseline file may not hold an entry for them.
*/
export const POST_CUT_RULES = [
{
name: 'ddl',
label: 'schema statement(s) outside engineDdl',
advice: `a migration after the Postgres baseline's cut runs on both engines: ${POST_CUT_ROUTINE}`,
},
{
name: 'sqlite-ddl',
label: 'SQLite spelling(s) in an engineDdl postgres member',
advice:
'the postgres member is what Postgres runs, and DATETIME, AUTOINCREMENT, INTEGER PRIMARY KEY, WITHOUT ROWID, ' +
'PRAGMA and the SQLite-only functions are not Postgres: replace it with what npm run gen:pg-migration prints',
},
];
const POST_CUT_RULE_NAMES = new Set(POST_CUT_RULES.map((rule) => rule.name));
const SOURCE = /\.ts$/;
const TEST = /\.(?:test|spec)\./;
/** The `Migration<timestamp>` a migration's class name starts with, or null. */
function migrationId(name) {
return /^(Migration\d+)(?:_|$)/.exec(name)?.[1] ?? null;
}
/**
* 'skip', 'migration' (a newer one, DML rules only), 'post-cut' (one after
* the Postgres baseline's `through`: the DML rules and POST_CUT_RULES) or
* 'source'.
*/
export function classify(file, through) {
if (file.startsWith(DIALECT_LAYER)) return 'skip';
const migration = /^src\/db\/migrations\/(Migration\d+)_/.exec(file);
if (migration) {
if (migration[1] <= FROZEN_MIGRATIONS_THROUGH) return 'skip';
return migration[1] > migrationId(through) ? 'post-cut' : 'migration';
}
if (file.startsWith('src/db/migrations/')) return 'skip';
return 'source';
}
/**
* The Postgres baseline's cut, from its manifest. A missing or malformed one
* stops the run: without it no migration would be held to POST_CUT_RULES.
*/
export function readThrough(serverDir) {
let through;
try {
through = JSON.parse(readFileSync(join(serverDir, POSTGRES_BASELINE_MANIFEST), 'utf8')).through;
} catch (err) {
throw new Error(`${POSTGRES_BASELINE_MANIFEST} cannot be read: ${err.message}`);
}
if (typeof through !== 'string' || migrationId(through) === null) {
throw new Error(`${POSTGRES_BASELINE_MANIFEST} must name the last migration it stands for in "through"`);
}
return through;
}
/** The name a call is made by: `engineDdl(...)` and `utils.engineDdl(...)` alike. */
function calleeName(node) {
const callee = node.expression;
if (ts.isIdentifier(callee)) return callee.text;
if (ts.isPropertyAccessExpression(callee)) return callee.name.text;
return null;
}
/** Whether `node` is an object literal naming both engines, the shape engineDdl and addColumnIfMissing take. */
function isPerEngineObject(node) {
if (!node || !ts.isObjectLiteralExpression(node)) return false;
const names = node.properties
.filter((property) => ts.isPropertyAssignment(property) && ts.isIdentifier(property.name))
.map((property) => property.name.text);
return names.includes('sqlite') && names.includes('postgres');
}
/**
* PRAGMA is fine in src/db/*.ts (connection, durability, baseline, snapshot),
* the migrations and the SQLite engine's own folder (src/db/engine/sqlite/).
*/
export function pragmaAllowed(file) {
return (
/^src\/db\/[^/]+\.ts$/.test(file) ||
file.startsWith('src/db/migrations/') ||
file.startsWith('src/db/engine/sqlite/')
);
}
function isSqlTag(node) {
let tag = node.tag;
if (ts.isPropertyAccessExpression(tag)) tag = tag.expression;
return ts.isIdentifier(tag) && tag.text === 'sql';
}
/**
* Hits per rule in one file's source. `through` is the Postgres baseline's
* cut; a migration after it is held to POST_CUT_RULES as well.
*/
export function scanSource(file, text, through) {
const kind = classify(file, through);
const counts = {};
if (kind === 'skip') return counts;
const postCut = kind === 'post-cut';
const add = (rule) => {
counts[rule] = (counts[rule] ?? 0) + 1;
};
const sf = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);
// `engine` is the member of an engineDdl or addColumnIfMissing argument a
// literal sits in: 'sqlite' (SQLite's own DDL, held to nothing), 'postgres'
// (no SQLite spelling) or null (anywhere else).
const checkText = (literal, inSqlTag, engine) => {
if (engine === 'sqlite') return;
if (engine === 'postgres') {
// Postgres spells `||` itself; every other rule names a form only SQLite has.
const sqliteOnly =
SQLITE_ONLY_DDL.test(literal) || RULES.some((rule) => rule.name !== 'concat' && rule.test(literal, inSqlTag));
if (sqliteOnly) add('sqlite-ddl');
return;
}
if (postCut && DDL_STATEMENT.test(literal)) add('ddl');
for (const rule of RULES) {
if (kind !== 'source' && !rule.migrations) continue;
if (rule.name === 'pragma' && pragmaAllowed(file)) continue;
if (rule.test(literal, inSqlTag)) add(rule.name);
}
};
const visitPerEngine = (object, inSqlTag) => {
for (const property of object.properties) {
const name = ts.isPropertyAssignment(property) && ts.isIdentifier(property.name) ? property.name.text : null;
if (name === 'sqlite' || name === 'postgres') visit(property.initializer, inSqlTag, name);
else visit(property, inSqlTag, null);
}
};
const visit = (node, inSqlTag, engine) => {
if (ts.isTaggedTemplateExpression(node) && isSqlTag(node)) {
ts.forEachChild(node, (child) => visit(child, true, engine));
return;
}
if (postCut && engine === null && ts.isCallExpression(node)) {
const name = calleeName(node);
if (SQLITE_ONLY_HELPERS.has(name)) add('ddl');
// A plain string would hand Postgres the SQLite spelling of the column.
if (name === 'addColumnIfMissing' && !isPerEngineObject(node.arguments[3])) add('ddl');
if (ENGINE_DDL_CALLS.has(name)) {
visit(node.expression, inSqlTag, null);
for (const argument of node.arguments) {
if (isPerEngineObject(argument)) visitPerEngine(argument, inSqlTag);
else visit(argument, inSqlTag, null);
}
return;
}
}
if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) {
checkText(node.text, inSqlTag, engine);
} else if (ts.isTemplateExpression(node)) {
// The literal parts only: an interpolated value is code, scanned on its own.
checkText([node.head.text, ...node.templateSpans.map((span) => span.literal.text)].join(' '), inSqlTag, engine);
} else if (ts.isPropertyAccessExpression(node) && node.name.text === 'insertId') {
add(INSERT_ID_RULE);
} else if (
ts.isBindingElement(node) &&
((node.propertyName && ts.isIdentifier(node.propertyName) && node.propertyName.text === 'insertId') ||
(!node.propertyName && ts.isIdentifier(node.name) && node.name.text === 'insertId'))
) {
add(INSERT_ID_RULE);
}
ts.forEachChild(node, (child) => visit(child, inSqlTag, engine));
};
visit(sf, false, null);
return counts;
}
function walk(dir, files = []) {
for (const name of readdirSync(dir)) {
if (name === 'node_modules') continue;
const path = join(dir, name);
if (statSync(path).isDirectory()) walk(path, files);
else if (SOURCE.test(name) && !TEST.test(name) && !name.endsWith('.d.ts')) files.push(path);
}
return files;
}
/** `{ file: { rule: n } }` for every file under src/ with at least one hit. */
export function scan(serverDir) {
const src = join(serverDir, 'src');
if (!existsSync(src) || !statSync(src).isDirectory()) {
throw new Error(`src/ does not exist under ${serverDir}: the check would pass without looking at anything`);
}
const through = readThrough(serverDir);
const result = {};
for (const path of walk(src)) {
const file = relative(serverDir, path).split('\\').join('/');
const counts = scanSource(file, readFileSync(path, 'utf8'), through);
if (Object.keys(counts).length > 0) result[file] = sortKeys(counts);
}
return sortKeys(result);
}
function sortKeys(obj) {
return Object.fromEntries(Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)));
}
const RULE_NAMES = new Set([...RULES.map((r) => r.name), INSERT_ID_RULE]);
/** The committed baseline. Missing or malformed stops the run rather than reading as empty. */
export function readBaseline(path) {
let parsed;
try {
parsed = JSON.parse(readFileSync(path, 'utf8'));
} catch (err) {
throw new Error(`scripts/dialect-baseline.json cannot be read: ${err.message}`);
}
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error('scripts/dialect-baseline.json must be an object of file paths to { rule: count }');
}
for (const [file, rules] of Object.entries(parsed)) {
if (rules === null || typeof rules !== 'object' || Array.isArray(rules)) {
throw new Error(`scripts/dialect-baseline.json: ${file} must map rule names to counts`);
}
for (const [rule, n] of Object.entries(rules)) {
if (POST_CUT_RULE_NAMES.has(rule)) {
throw new Error(
`scripts/dialect-baseline.json: ${file} holds an allowance for ${rule}, which a migration after the ` +
"Postgres baseline's cut never gets",
);
}
if (!RULE_NAMES.has(rule))
throw new Error(`scripts/dialect-baseline.json: ${file} names an unknown rule "${rule}"`);
if (!Number.isInteger(n) || n <= 0) {
throw new Error(
`scripts/dialect-baseline.json: ${file} ${rule} holds ${JSON.stringify(n)}, expected a positive integer`,
);
}
}
}
return parsed;
}
/** The baseline lowered to what the files hold now. Never raises an entry, never adds one. */
export function lowerBaseline(baseline, counts) {
const lowered = {};
for (const [file, rules] of Object.entries(baseline)) {
const kept = {};
for (const [rule, allowed] of Object.entries(rules)) {
const now = counts[file]?.[rule] ?? 0;
if (now > 0) kept[rule] = Math.min(allowed, now);
}
if (Object.keys(kept).length > 0) lowered[file] = sortKeys(kept);
}
return sortKeys(lowered);
}
/** New hits over the baseline, and baseline entries above what their file holds now. */
export function compare(baseline, counts) {
const grown = [];
for (const [file, rules] of Object.entries(counts)) {
for (const [rule, n] of Object.entries(rules)) {
const allowed = baseline[file]?.[rule] ?? 0;
if (n > allowed) grown.push({ file, rule, n, allowed });
}
}
const lowered = lowerBaseline(baseline, counts);
const stale = [];
for (const [file, rules] of Object.entries(baseline)) {
for (const [rule, allowed] of Object.entries(rules)) {
const now = lowered[file]?.[rule] ?? 0;
if (now !== allowed) stale.push({ file, rule, allowed, now: counts[file]?.[rule] ?? 0 });
}
}
return { grown, stale };
}
function adviceFor(rule) {
if (rule === INSERT_ID_RULE) return "read the new id with .returning('id'); insertId is undefined on Postgres";
return [...RULES, ...POST_CUT_RULES].find((r) => r.name === rule).advice;
}
function labelFor(rule) {
return POST_CUT_RULES.find((r) => r.name === rule)?.label ?? `SQLite-only ${rule} hit(s)`;
}
function main(argv) {
const dirArg = argv.find((a) => a.startsWith('--dir='));
const serverDir = dirArg ? resolve(dirArg.slice('--dir='.length)) : fileURLToPath(new URL('..', import.meta.url));
const baselinePath = join(serverDir, 'scripts', 'dialect-baseline.json');
const update = argv.includes('--update');
const counts = scan(serverDir);
let baseline = readBaseline(baselinePath);
if (update) {
baseline = lowerBaseline(baseline, counts);
writeFileSync(baselinePath, JSON.stringify(baseline, null, 2) + '\n');
}
const { grown, stale } = compare(baseline, counts);
for (const { file, rule, n, allowed } of grown) {
console.error(`FAIL ${file}: ${n} ${labelFor(rule)}, ${allowed} allowed. ${adviceFor(rule)}.`);
}
for (const { file, rule, allowed, now } of stale) {
console.error(
`FAIL ${file} is held at ${allowed} ${rule} hit(s) in scripts/dialect-baseline.json, but has ${now} now.`,
);
}
if (stale.length) {
console.error('Run npm run lint:dialect -- --update to lower the baseline with the change that removed them.');
}
const total = Object.values(baseline).reduce(
(sum, rules) => sum + Object.values(rules).reduce((a, b) => a + b, 0),
0,
);
console.log(
`dialect: ${Object.keys(baseline).length} file(s) hold ${total} SQLite-only spelling(s) at their baseline`,
);
return grown.length || stale.length ? 1 : 0;
}
const isCli =
Boolean(process.argv[1]) &&
existsSync(process.argv[1]) &&
realpathSync(process.argv[1]) === fileURLToPath(import.meta.url);
if (isCli) {
try {
process.exitCode = main(process.argv.slice(2));
} catch (err) {
console.error(`FAIL ${err.message}`);
process.exitCode = 1;
}
}