mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-11-20 19:16:08 +00:00
fix(db): refuse a Postgres boot that would leave a SQLite install behind
Setting TREK_DB_URL on an existing SQLite install booted a new, empty instance beside its data. The Postgres bootstrap now refuses an empty database, or a migrated one without a single account, while the data directory's travel.db holds accounts, and names the move command. A file that cannot be read counts as holding them. TREK_DB_IGNORE_SQLITE_FILE=true starts anyway, with a warning; the file is opened read-only and only when the database holds no TREK data.
This commit is contained in:
@@ -5,14 +5,10 @@ import { knownMigrationNames } from '../../known-migrations';
|
||||
import { migrationLabel, REFUSAL, refuseNewerDatabase, type MigrationSafetyNet } from '../../legacy-baseline';
|
||||
import type { AnyOrm } from '../../orm';
|
||||
import { listSnapshots, PREFIX, pruneSnapshots, stamp } from '../../pre-migrate-snapshot';
|
||||
import { sqliteInstallHoldsAccounts } from '../sqlite/sqlite-source';
|
||||
import { readPostgresBaseline, type PostgresBaseline } from './baseline';
|
||||
import { pgDump } from './pg-tools';
|
||||
import {
|
||||
PRIVILEGED_ROLE_SQL,
|
||||
PRIVILEGED_ROLE_WARNING,
|
||||
refuseUnsupportedServer,
|
||||
serverMajor,
|
||||
} from './server-checks';
|
||||
import { PRIVILEGED_ROLE_SQL, PRIVILEGED_ROLE_WARNING, refuseUnsupportedServer, serverMajor } from './server-checks';
|
||||
import type { Connection } from '@mikro-orm/core';
|
||||
import type { Migrator } from '@mikro-orm/migrations';
|
||||
import { Logger } from '@nestjs/common';
|
||||
@@ -30,6 +26,50 @@ export interface PostgresBootstrapDeps {
|
||||
baseline: () => PostgresBaseline;
|
||||
known: Set<string>;
|
||||
safetyNet: MigrationSafetyNet;
|
||||
/**
|
||||
* The SQLite install a boot on a database without TREK data would leave
|
||||
* behind: the path of a SQLite file in the data directory that holds
|
||||
* accounts, or null when there is none (or the operator said to start
|
||||
* anyway). Read only when the database holds no TREK data, so a boot on a
|
||||
* database in use never opens the file.
|
||||
*/
|
||||
sqliteLeftBehind: () => string | null;
|
||||
}
|
||||
|
||||
/** The command that moves a SQLite install into Postgres, as the refusal names it. */
|
||||
const MOVE_COMMAND_HINT =
|
||||
'docker compose run --rm --user node -w /app/server app node --require tsconfig-paths/register ' +
|
||||
'dist/move-to-postgres.js (npm run db:move-to-postgres in server/ outside Docker)';
|
||||
|
||||
/**
|
||||
* The refusal of a boot that would come up empty on Postgres while `file`
|
||||
* holds the install's data.
|
||||
*/
|
||||
function leftBehindRefusal(file: string): string {
|
||||
return (
|
||||
`${REFUSAL} TREK_DB_URL points at a Postgres database that holds no TREK data, while ${file} holds an ` +
|
||||
'existing SQLite install with accounts. Starting would bring up a new, empty TREK beside it. To move that ' +
|
||||
`install to Postgres, stop TREK and run the move command: ${MOVE_COMMAND_HINT}. The wiki page ` +
|
||||
'Database-Postgres ("Moving an existing install to Postgres") has the steps. To start on the empty ' +
|
||||
'database on purpose, set TREK_DB_IGNORE_SQLITE_FILE=true; to keep running on SQLite, unset TREK_DB_URL.'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the database holds an account. A database TREK created always does
|
||||
* once its first boot finished; one without is what a move that stopped part
|
||||
* of the way, or a first boot that died before seeding, leaves.
|
||||
*/
|
||||
async function holdsAccounts(connection: Connection, tables: readonly string[]): Promise<boolean> {
|
||||
if (!tables.includes('users')) return false;
|
||||
const rows: unknown[] = await connection.execute('select 1 as present from users limit 1');
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
/** Refuses the boot when `sqliteLeftBehind` names a SQLite install this boot would leave behind. */
|
||||
function refuseLeavingSqliteBehind(deps: PostgresBootstrapDeps): void {
|
||||
const file = deps.sqliteLeftBehind();
|
||||
if (file !== null) throw new Error(leftBehindRefusal(file));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -147,9 +187,13 @@ export async function bootstrapPostgresSchema(orm: AnyOrm, deps: PostgresBootstr
|
||||
const executed = tables.includes(historyTable) ? (await migrator.getExecuted()).map((row) => row.name) : [];
|
||||
|
||||
if (executed.length === 0) {
|
||||
// An empty database (a database holding other tables is refused below, with its own message).
|
||||
if (tables.every((table) => table === historyTable)) refuseLeavingSqliteBehind(deps);
|
||||
await createFromBaseline(connection, migrator.getStorage(), tables, historyTable, deps.baseline);
|
||||
} else {
|
||||
await refuseNewerHistory(executed, deps);
|
||||
// A schema without a single account holds no TREK data either; only then is the SQLite file looked at.
|
||||
if (!(await holdsAccounts(connection, tables))) refuseLeavingSqliteBehind(deps);
|
||||
}
|
||||
|
||||
const pending = await migrator.getPending();
|
||||
@@ -229,7 +273,37 @@ export function postgresSafetyNet(
|
||||
};
|
||||
}
|
||||
|
||||
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net above. */
|
||||
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
|
||||
return { baseline: readPostgresBaseline, known: knownMigrationNames(), safetyNet: postgresSafetyNet(url) };
|
||||
/**
|
||||
* The SQLite install a Postgres boot would leave behind (`PostgresBootstrapDeps.sqliteLeftBehind`):
|
||||
* the SQLite file the data directory resolves to (`data/travel.db`), when it
|
||||
* holds accounts. A file that cannot be read counts as holding them, so the
|
||||
* check fails closed. `TREK_DB_IGNORE_SQLITE_FILE=true` lets the boot through
|
||||
* with a warning. Off under NODE_ENV=test, where the data directory is the
|
||||
* developer's own.
|
||||
*/
|
||||
export function sqliteLeftBehindCheck(
|
||||
env: () => AppEnv = readEnv,
|
||||
holdsAccounts: (file: string) => boolean = sqliteInstallHoldsAccounts,
|
||||
): () => string | null {
|
||||
return () => {
|
||||
const current = env();
|
||||
if (current.app.isTest) return null;
|
||||
const file = resolveDataPaths(current).dbFile;
|
||||
if (!holdsAccounts(file)) return null;
|
||||
if (current.db.ignoreSqliteFile) {
|
||||
logger.warn(`TREK_DB_IGNORE_SQLITE_FILE is set: starting on Postgres beside the SQLite install in ${file}`);
|
||||
return null;
|
||||
}
|
||||
return file;
|
||||
};
|
||||
}
|
||||
|
||||
/** What a production boot passes: the committed baseline, the migrations on disk, the safety net and the check above. */
|
||||
export function defaultPostgresBootstrapDeps(url: string): PostgresBootstrapDeps {
|
||||
return {
|
||||
baseline: readPostgresBaseline,
|
||||
known: knownMigrationNames(),
|
||||
safetyNet: postgresSafetyNet(url),
|
||||
sqliteLeftBehind: sqliteLeftBehindCheck(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import fs from 'node:fs';
|
||||
|
||||
/**
|
||||
* A SQLite file opened for reading only, for the move to Postgres and the
|
||||
* Postgres boot's check for an install left behind. It never writes: the
|
||||
* handle is read-only, and every read runs in one read transaction, so the
|
||||
* whole move sees the file as it was at one moment.
|
||||
*
|
||||
* Integers come back as `bigint` (better-sqlite3's safe integers), so a value
|
||||
* past 2^53 keeps every digit.
|
||||
*/
|
||||
|
||||
/** A column as `pragma_table_info` describes it. */
|
||||
export interface SqliteColumn {
|
||||
name: string;
|
||||
/** The declared type, as written in the CREATE TABLE (may be empty). */
|
||||
declaredType: string;
|
||||
notNull: boolean;
|
||||
/** Its position in the primary key, from 1; 0 when it is not part of it. */
|
||||
primaryKey: number;
|
||||
/** The default as SQL text, or null. */
|
||||
defaultSql: string | null;
|
||||
}
|
||||
|
||||
/** An index of a table, as `pragma_index_list` and `pragma_index_info` describe it. */
|
||||
export interface SqliteIndex {
|
||||
name: string;
|
||||
unique: boolean;
|
||||
/** Where it comes from: `c` a CREATE INDEX, `u` a UNIQUE constraint, `pk` the primary key. */
|
||||
origin: string;
|
||||
partial: boolean;
|
||||
/** The indexed columns in order; null when one of them is an expression. */
|
||||
columns: string[] | null;
|
||||
}
|
||||
|
||||
/** A row whose foreign key points at nothing, from `pragma_foreign_key_check`. */
|
||||
interface OrphanRow {
|
||||
table: string;
|
||||
rowid: bigint | null;
|
||||
parent: string;
|
||||
}
|
||||
|
||||
/** `name` as a SQLite identifier. */
|
||||
function quoteSqliteIdentifier(name: string): string {
|
||||
return `"${name.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
export class SqliteSource {
|
||||
private constructor(
|
||||
private readonly db: Database.Database,
|
||||
readonly file: string,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Opens `file` read-only and starts the read transaction. A file that does
|
||||
* not exist is refused rather than created.
|
||||
*/
|
||||
static open(file: string): SqliteSource {
|
||||
if (!fs.existsSync(file)) throw new Error(`There is no SQLite file at ${file}.`);
|
||||
const db = new Database(file, { readonly: true, fileMustExist: true });
|
||||
try {
|
||||
db.defaultSafeIntegers(true);
|
||||
db.exec('BEGIN');
|
||||
return new SqliteSource(db, file);
|
||||
} catch (err) {
|
||||
db.close();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (!this.db.open) return;
|
||||
if (this.db.inTransaction) this.db.exec('ROLLBACK');
|
||||
this.db.close();
|
||||
}
|
||||
|
||||
private names(sql: string, ...params: unknown[]): string[] {
|
||||
return this.db
|
||||
.prepare(sql)
|
||||
.pluck()
|
||||
.all(...params)
|
||||
.map((value) => String(value));
|
||||
}
|
||||
|
||||
/** The user tables, by name, without SQLite's own (`sqlite_*`). */
|
||||
tables(): string[] {
|
||||
return this.names(
|
||||
"select name from sqlite_master where type = 'table' and name not like 'sqlite\\_%' escape '\\' order by name",
|
||||
);
|
||||
}
|
||||
|
||||
/** The views and triggers, which the move does not carry for a plugin's database. */
|
||||
viewsAndTriggers(): Array<{ type: string; name: string }> {
|
||||
return this.db
|
||||
.prepare("select type, name from sqlite_master where type in ('view', 'trigger') order by type, name")
|
||||
.all() as Array<{ type: string; name: string }>;
|
||||
}
|
||||
|
||||
hasTable(name: string): boolean {
|
||||
return this.names("select name from sqlite_master where type = 'table' and name = ?", name).length > 0;
|
||||
}
|
||||
|
||||
/** The names TREK's migration history holds; null when the file has no history table. */
|
||||
migrationNames(): string[] | null {
|
||||
if (!this.hasTable('mikro_orm_migrations')) return null;
|
||||
return this.names('select name from mikro_orm_migrations order by name');
|
||||
}
|
||||
|
||||
columns(table: string): SqliteColumn[] {
|
||||
const rows = this.db
|
||||
.prepare('select name, type, "notnull", pk, dflt_value from pragma_table_info(?) order by cid')
|
||||
.all(table) as Array<{
|
||||
name: string;
|
||||
type: string | null;
|
||||
notnull: bigint;
|
||||
pk: bigint;
|
||||
dflt_value: string | null;
|
||||
}>;
|
||||
return rows.map((row) => ({
|
||||
name: row.name,
|
||||
declaredType: row.type ?? '',
|
||||
notNull: row.notnull !== 0n,
|
||||
primaryKey: Number(row.pk),
|
||||
defaultSql: row.dflt_value,
|
||||
}));
|
||||
}
|
||||
|
||||
indexes(table: string): SqliteIndex[] {
|
||||
const list = this.db
|
||||
.prepare('select name, "unique", origin, partial from pragma_index_list(?)')
|
||||
.all(table) as Array<{
|
||||
name: string;
|
||||
unique: bigint;
|
||||
origin: string;
|
||||
partial: bigint;
|
||||
}>;
|
||||
return list.map((index) => {
|
||||
const keys = this.db
|
||||
.prepare('select cid, name from pragma_index_info(?) order by seqno')
|
||||
.all(index.name) as Array<{
|
||||
cid: bigint;
|
||||
name: string | null;
|
||||
}>;
|
||||
const columns = keys.every((key) => key.cid >= 0n && key.name !== null)
|
||||
? keys.map((key) => String(key.name))
|
||||
: null;
|
||||
return {
|
||||
name: index.name,
|
||||
unique: index.unique !== 0n,
|
||||
origin: index.origin,
|
||||
partial: index.partial !== 0n,
|
||||
columns,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
hasRows(table: string): boolean {
|
||||
return this.db.prepare(`select 1 from ${quoteSqliteIdentifier(table)} limit 1`).get() !== undefined;
|
||||
}
|
||||
|
||||
count(table: string): number {
|
||||
return Number(
|
||||
this.db
|
||||
.prepare(`select count(*) from ${quoteSqliteIdentifier(table)}`)
|
||||
.pluck()
|
||||
.get(),
|
||||
);
|
||||
}
|
||||
|
||||
/** What `typeof()` answers for the column's values that are not NULL: `integer`, `real`, `text`, `blob`. */
|
||||
storageClasses(table: string, column: string): Set<string> {
|
||||
const quoted = quoteSqliteIdentifier(column);
|
||||
return new Set(
|
||||
this.names(`select distinct typeof(${quoted}) from ${quoteSqliteIdentifier(table)} where ${quoted} is not null`),
|
||||
);
|
||||
}
|
||||
|
||||
/** The rows of `table`, one array of `columns` per row, read one at a time. */
|
||||
rows(table: string, columns: readonly string[]): IterableIterator<unknown[]> {
|
||||
const list = columns.map(quoteSqliteIdentifier).join(', ');
|
||||
return this.db
|
||||
.prepare(`select ${list} from ${quoteSqliteIdentifier(table)}`)
|
||||
.raw(true)
|
||||
.iterate() as IterableIterator<unknown[]>;
|
||||
}
|
||||
|
||||
/** The highest id each AUTOINCREMENT table ever handed out, which can be above the largest id left. */
|
||||
autoincrementSeen(): Map<string, bigint> {
|
||||
if (!this.hasTable('sqlite_sequence')) return new Map();
|
||||
const rows = this.db.prepare('select name, seq from sqlite_sequence').all() as Array<{ name: string; seq: bigint }>;
|
||||
return new Map(rows.map((row) => [row.name, BigInt(row.seq)]));
|
||||
}
|
||||
|
||||
/** Up to `limit` rows whose foreign key points at a row that does not exist. */
|
||||
orphans(limit: number): OrphanRow[] {
|
||||
const rows = this.db
|
||||
.prepare('select "table", rowid, parent from pragma_foreign_key_check limit ?')
|
||||
.all(limit) as Array<{
|
||||
table: string;
|
||||
rowid: bigint | null;
|
||||
parent: string;
|
||||
}>;
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `file` is a SQLite install holding accounts: a `users` table with a
|
||||
* row. A missing or empty file holds none. A file that cannot be read is
|
||||
* reported as holding some, so a check built on this fails closed.
|
||||
*/
|
||||
export function sqliteInstallHoldsAccounts(file: string): boolean {
|
||||
let size: number;
|
||||
try {
|
||||
size = fs.statSync(file).size;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false;
|
||||
return true;
|
||||
}
|
||||
if (size === 0) return false;
|
||||
let source: SqliteSource | null = null;
|
||||
try {
|
||||
source = SqliteSource.open(file);
|
||||
return source.hasTable('users') && source.hasRows('users');
|
||||
} catch {
|
||||
return true;
|
||||
} finally {
|
||||
source?.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user