mirror of
https://github.com/mauriceboe/TREK.git
synced 2026-10-21 06:26:26 +00:00
perf(server): write the file log in batches off the event loop
Every log line ran existsSync, statSync and appendFileSync on the event loop before it returned. On a slow data volume (an SMB or NFS bind mount on a NAS) that stalled every request that logged, and rotation renamed files without checking whether another process sharing data/logs had rotated already. The file half of the logger is now BufferedLogFile: a line is queued in memory and the queue goes out with one appendFile per batch, through a single serial chain, so no append runs while files are being renamed. The files and the rotation are the same as before (trek.log rotates to .1 at 10 MB, five files in total), but the size is re-read before a rotation and a file another process already rotated is left alone. A stalled disk costs dropped lines with a count instead of unbounded memory, a failed write is reported instead of thrown, and index.ts writes whatever is still queued synchronously when the process exits.
This commit is contained in:
Maurice
@@ -18,6 +18,7 @@ import type { DatabaseLifecycle } from './nest/database/database-lifecycle.servi
|
||||
import { getAppUrl, getMcpSafeUrl, readEnv } from './app-config';
|
||||
import { resolveDataPaths } from './app-config/data-paths';
|
||||
import { resolveDbPath } from './db/db-path';
|
||||
import { flushLogFileSync } from './nest/audit/audit-log.logger';
|
||||
|
||||
// data/tmp is the driver-agnostic global scratch dir (restore-upload spool,
|
||||
// mirror stream staging) and stays boot-created here. Driver-owned roots — the
|
||||
@@ -206,5 +207,10 @@ function shutdown(signal: string): void {
|
||||
});
|
||||
}
|
||||
|
||||
// trek.log is written in batches off the event loop. Whatever is still queued
|
||||
// when the process ends, by any of the exits above or below, goes out
|
||||
// synchronously here: an 'exit' listener cannot wait for a promise.
|
||||
process.on('exit', () => flushLogFileSync());
|
||||
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
import { readEnv } from '../../app-config';
|
||||
import { resolveDataPaths } from '../../app-config/data-paths';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { BufferedLogFile } from './log-file';
|
||||
|
||||
/**
|
||||
* The server's rotating file logger — a plain module, NOT an injectable
|
||||
* (index.ts lazy-requires it before any Nest container exists). Directory
|
||||
* creation is lazy (first write), so importing the module has no disk side
|
||||
* effect; the LOG_LEVEL freeze below is the one deliberate import-time
|
||||
* behavior and is load-bearing for tests/setup.ts.
|
||||
* The server's rotating file logger, a plain module and NOT an injectable
|
||||
* (index.ts lazy-requires it before any Nest container exists). Every line goes
|
||||
* to the console at once and to `data/logs/trek.log` through a buffered,
|
||||
* asynchronous sink (log-file.ts), so logging never waits for the disk.
|
||||
* Nothing touches the disk at import: the directory appears with the first
|
||||
* flush. The LOG_LEVEL freeze below is the one deliberate import-time behavior
|
||||
* and is load-bearing for tests/setup.ts.
|
||||
*/
|
||||
|
||||
// Frozen at import on purpose (legacy timing; tests/setup.ts sets it pre-import).
|
||||
@@ -29,48 +30,26 @@ const C = {
|
||||
reset: '\x1b[0m',
|
||||
};
|
||||
|
||||
// ── File logger with rotation ─────────────────────────────────────────────
|
||||
// ── File sink ─────────────────────────────────────────────────────────────
|
||||
|
||||
// From the data layout, like every other data path. It used to be cwd-based,
|
||||
// which agreed only because the image and `npm run dev` both start in server/.
|
||||
const logsDir = resolveDataPaths().logsDir;
|
||||
const logFilePath = path.join(logsDir, 'trek.log');
|
||||
let logsDirReady = false;
|
||||
const logFile = new BufferedLogFile({
|
||||
// From the data layout, like every other data path. It used to be cwd-based,
|
||||
// which agreed only because the image and `npm run dev` both start in server/.
|
||||
dir: resolveDataPaths().logsDir,
|
||||
file: 'trek.log',
|
||||
maxBytes: MAX_LOG_SIZE,
|
||||
maxFiles: MAX_LOG_FILES,
|
||||
onError: (message) => console.error(`[logger] ${message}`),
|
||||
});
|
||||
|
||||
function ensureLogsDir(): void {
|
||||
if (logsDirReady) return;
|
||||
try {
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
logsDirReady = true;
|
||||
} catch (e) {
|
||||
console.error(`[logger] could not create ${logsDir}: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
/** Resolves once every line logged so far is in trek.log. */
|
||||
export function flushLogFile(): Promise<void> {
|
||||
return logFile.flush();
|
||||
}
|
||||
|
||||
function rotateIfNeeded(): void {
|
||||
try {
|
||||
if (!fs.existsSync(logFilePath)) return;
|
||||
const stat = fs.statSync(logFilePath);
|
||||
if (stat.size < MAX_LOG_SIZE) return;
|
||||
|
||||
for (let i = MAX_LOG_FILES - 1; i >= 1; i--) {
|
||||
const src = i === 1 ? logFilePath : `${logFilePath}.${i - 1}`;
|
||||
const dst = `${logFilePath}.${i}`;
|
||||
if (fs.existsSync(src)) fs.renameSync(src, dst);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`[logger] log rotation failed: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
}
|
||||
|
||||
function writeToFile(line: string): void {
|
||||
try {
|
||||
ensureLogsDir();
|
||||
rotateIfNeeded();
|
||||
fs.appendFileSync(logFilePath, line + '\n');
|
||||
} catch (e) {
|
||||
console.error(`[logger] log file write failed: ${e instanceof Error ? e.message : e}`);
|
||||
}
|
||||
/** Writes what is still queued, synchronously: for the process exit handler only. */
|
||||
export function flushLogFileSync(): void {
|
||||
logFile.flushSync();
|
||||
}
|
||||
|
||||
// ── Public log helpers ────────────────────────────────────────────────────
|
||||
@@ -84,27 +63,27 @@ export function logInfo(msg: string): void {
|
||||
if (LOG_THRESHOLD < LEVEL_RANKS.info) return;
|
||||
const ts = formatTs();
|
||||
console.log(`${C.blue}[INFO]${C.reset} ${ts} ${msg}`);
|
||||
writeToFile(`[INFO] ${ts} ${msg}`);
|
||||
logFile.write(`[INFO] ${ts} ${msg}`);
|
||||
}
|
||||
|
||||
export function logDebug(msg: string): void {
|
||||
if (LOG_THRESHOLD < LEVEL_RANKS.debug) return;
|
||||
const ts = formatTs();
|
||||
console.log(`${C.cyan}[DEBUG]${C.reset} ${ts} ${msg}`);
|
||||
writeToFile(`[DEBUG] ${ts} ${msg}`);
|
||||
logFile.write(`[DEBUG] ${ts} ${msg}`);
|
||||
}
|
||||
|
||||
export function logError(msg: string): void {
|
||||
const ts = formatTs();
|
||||
console.error(`${C.red}[ERROR]${C.reset} ${ts} ${msg}`);
|
||||
writeToFile(`[ERROR] ${ts} ${msg}`);
|
||||
logFile.write(`[ERROR] ${ts} ${msg}`);
|
||||
}
|
||||
|
||||
export function logWarn(msg: string): void {
|
||||
if (LOG_THRESHOLD < LEVEL_RANKS.warn) return;
|
||||
const ts = formatTs();
|
||||
console.warn(`${C.yellow}[WARN]${C.reset} ${ts} ${msg}`);
|
||||
writeToFile(`[WARN] ${ts} ${msg}`);
|
||||
logFile.write(`[WARN] ${ts} ${msg}`);
|
||||
}
|
||||
|
||||
export { LOG_LEVEL };
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
export interface LogFileOptions {
|
||||
/** Directory the log lives in; created on the first write, never at construction. */
|
||||
dir: string;
|
||||
/** File name inside `dir`. */
|
||||
file?: string;
|
||||
/** Size at which the file is rotated to `<file>.1`. */
|
||||
maxBytes?: number;
|
||||
/** How many files exist at most, the live one included (`<file>`, `.1` ... `.<maxFiles - 1>`). */
|
||||
maxFiles?: number;
|
||||
/**
|
||||
* Ceiling on what may wait in memory while the disk is slow. Lines past it
|
||||
* are counted and dropped, and the count is written once there is room
|
||||
* again, so a stalled volume costs log lines rather than the process.
|
||||
*/
|
||||
maxBufferedBytes?: number;
|
||||
/** How long a line may wait for others to share its write. */
|
||||
flushDelayMs?: number;
|
||||
/** Where the sink reports its own failures; it never throws into a caller. */
|
||||
onError: (message: string) => void;
|
||||
}
|
||||
|
||||
const DEFAULTS = {
|
||||
file: 'trek.log',
|
||||
maxBytes: 10 * 1024 * 1024,
|
||||
maxFiles: 5,
|
||||
maxBufferedBytes: 4 * 1024 * 1024,
|
||||
flushDelayMs: 50,
|
||||
};
|
||||
|
||||
function errorMessage(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
|
||||
function isMissing(err: unknown): boolean {
|
||||
return (err as NodeJS.ErrnoException | null)?.code === 'ENOENT';
|
||||
}
|
||||
|
||||
/**
|
||||
* The file half of the server log: lines are queued in memory and appended in
|
||||
* batches, off the event loop, by one write at a time.
|
||||
*
|
||||
* The previous logger did `existsSync` + `statSync` + `appendFileSync` for
|
||||
* every line, on the event loop, which a slow volume (an SMB or NFS bind
|
||||
* mount) turned into a stall of every request. Here a line costs an array
|
||||
* push; the disk sees one `appendFile` per batch.
|
||||
*
|
||||
* Rotation keeps its old shape (`trek.log` moves to `trek.log.1`, the oldest
|
||||
* of `maxFiles` falls off) and is made safe in two ways. It runs inside the
|
||||
* same serial chain as the writes, so no batch is appended while files are
|
||||
* being renamed. And before rotating it re-reads the size on disk: when
|
||||
* another process sharing the directory rotated first, the file is small
|
||||
* again and this one only adopts the new size instead of rotating a second
|
||||
* time and pushing a fresh file out.
|
||||
*/
|
||||
export class BufferedLogFile {
|
||||
private readonly filePath: string;
|
||||
private readonly opts: Required<Omit<LogFileOptions, 'onError'>> & Pick<LogFileOptions, 'onError'>;
|
||||
private pending: string[] = [];
|
||||
private pendingBytes = 0;
|
||||
/** The batch whose append has not resolved yet; flushSync() writes it too, so an exit does not lose it. */
|
||||
private inFlight: string | null = null;
|
||||
private dropped = 0;
|
||||
/** Bytes in the live file as far as this process knows; read from disk before the first append. */
|
||||
private size: number | null = null;
|
||||
private dirReady = false;
|
||||
private chain: Promise<void> = Promise.resolve();
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
|
||||
constructor(options: LogFileOptions) {
|
||||
this.opts = { ...DEFAULTS, ...options };
|
||||
this.filePath = path.join(this.opts.dir, this.opts.file);
|
||||
}
|
||||
|
||||
get path(): string {
|
||||
return this.filePath;
|
||||
}
|
||||
|
||||
/** Queue one line (without its newline). Never blocks and never throws. */
|
||||
write(line: string): void {
|
||||
const chunk = `${line}\n`;
|
||||
if (this.pendingBytes + chunk.length > this.opts.maxBufferedBytes) {
|
||||
this.dropped++;
|
||||
return;
|
||||
}
|
||||
if (this.dropped > 0) this.reportDropped();
|
||||
this.pending.push(chunk);
|
||||
this.pendingBytes += chunk.length;
|
||||
this.schedule();
|
||||
}
|
||||
|
||||
/** Resolves once everything queued before the call is on disk (or reported as failed). */
|
||||
flush(): Promise<void> {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
this.chain = this.chain.then(() => this.drain());
|
||||
return this.chain;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write whatever is still in memory, synchronously. For the last moment of
|
||||
* a process (the exit handler), where a promise would never settle. A batch
|
||||
* whose async append was still running is written again rather than risked:
|
||||
* a duplicated line at exit is better than a lost one.
|
||||
*/
|
||||
flushSync(): void {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
const data = (this.inFlight ?? '') + this.takePending();
|
||||
if (!data) return;
|
||||
try {
|
||||
fs.mkdirSync(this.opts.dir, { recursive: true });
|
||||
fs.appendFileSync(this.filePath, data);
|
||||
} catch (err) {
|
||||
this.opts.onError(`log file write failed: ${errorMessage(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private schedule(): void {
|
||||
if (this.timer) return;
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null;
|
||||
void this.flush();
|
||||
}, this.opts.flushDelayMs);
|
||||
// A queued log line must never be what keeps a finished process alive.
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
private takePending(): string {
|
||||
if (this.dropped > 0) this.reportDropped();
|
||||
const data = this.pending.join('');
|
||||
this.pending = [];
|
||||
this.pendingBytes = 0;
|
||||
return data;
|
||||
}
|
||||
|
||||
private reportDropped(): void {
|
||||
const note = `[WARN] [logger] ${this.dropped} log line(s) dropped while the log file could not keep up\n`;
|
||||
this.dropped = 0;
|
||||
this.pending.push(note);
|
||||
this.pendingBytes += note.length;
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
const data = this.takePending();
|
||||
if (!data) return;
|
||||
this.inFlight = data;
|
||||
try {
|
||||
await this.ensureDir();
|
||||
const bytes = Buffer.byteLength(data);
|
||||
if (this.size === null) this.size = await this.sizeOnDisk();
|
||||
// Same trigger as before: a file already at the cap is rotated before the next append.
|
||||
if (this.size >= this.opts.maxBytes) await this.rotate();
|
||||
await fs.promises.appendFile(this.filePath, data);
|
||||
this.size += bytes;
|
||||
} catch (err) {
|
||||
// The size we track may be wrong after a failure; learn it again next time.
|
||||
this.size = null;
|
||||
this.opts.onError(`log file write failed: ${errorMessage(err)}`);
|
||||
} finally {
|
||||
this.inFlight = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureDir(): Promise<void> {
|
||||
if (this.dirReady) return;
|
||||
await fs.promises.mkdir(this.opts.dir, { recursive: true });
|
||||
this.dirReady = true;
|
||||
}
|
||||
|
||||
private async sizeOnDisk(): Promise<number> {
|
||||
try {
|
||||
return (await fs.promises.stat(this.filePath)).size;
|
||||
} catch (err) {
|
||||
if (isMissing(err)) return 0;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async rotate(): Promise<void> {
|
||||
try {
|
||||
// Somebody else may have rotated since we last looked.
|
||||
const actual = await this.sizeOnDisk();
|
||||
if (actual < this.opts.maxBytes) {
|
||||
this.size = actual;
|
||||
return;
|
||||
}
|
||||
for (let i = this.opts.maxFiles - 1; i >= 1; i--) {
|
||||
const src = i === 1 ? this.filePath : `${this.filePath}.${i - 1}`;
|
||||
try {
|
||||
await fs.promises.rename(src, `${this.filePath}.${i}`);
|
||||
} catch (err) {
|
||||
if (!isMissing(err)) throw err;
|
||||
}
|
||||
}
|
||||
this.size = 0;
|
||||
} catch (err) {
|
||||
this.opts.onError(`log rotation failed: ${errorMessage(err)}`);
|
||||
// Keep appending to the current file rather than losing the batch.
|
||||
this.size = await this.sizeOnDisk().catch(() => 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user