fix(server): release sockets on shutdown so docker stop no longer ends in SIGKILL

Every `docker stop` of a TREK instance with a browser attached exited 137,
exactly ten seconds after the stop.

`http.Server.close()` stops new connections and then waits for the open ones
to end by themselves — and a WebSocket never does. TREK holds one per open
tab, so the close callback never ran. `closeDb()` and `process.exit(0)` lived
inside that callback, which left the only remaining exit a fallback timer set
to 10000ms: exactly when Docker sends SIGKILL. Docker starts counting when it
sends the signal and Node only once the handler is dispatched, so that timer
started strictly later and could never win. It was unreachable code in every
default deployment.

Waiting for Nest was not an option either: `WsAdapter.close()` is the one
place that terminates clients, and Nest reaches it only after every
onModuleDestroy hook has resolved — including the plugin supervisor's SIGTERM
grace for its forked children. The sockets holding the server open would be
hostage to the slowest hook in the container.

So the sockets are now released deliberately: ws clients get 1001 "going
away", idle keep-alive sockets are dropped at once, and anything still
attached after a 2s drain is destroyed. Nest is closed alongside that, the
database is closed on both exit paths rather than only the unreachable one,
and the fallback fires at 5s — inside Docker's grace, so a forced exit is
still ours.

The sequence moved into src/shutdown.ts behind injected dependencies. It was
untestable by construction before, which is part of why this went unnoticed;
it now has nine tests, including the signal that beats the bootstrap.

Closes #2193

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S5VwmiBQ9sPCJwfmpv64r3

server/src/index.ts

@@ -116,24 +116,41 @@ bootstrap().catch((err) => {
 });
 
 // Graceful shutdown
+//
+// The sequence itself lives in ./shutdown so it can be tested without a process
+// to exit; this is only the wiring. See that file for why the order matters —
+// in short, #2193: nothing here could ever release a WebSocket, so `docker
+// stop` always ended in SIGKILL and exit 137.
+let shuttingDown = false;
 function shutdown(signal: string): void {
+  // A second signal — the SIGINT that follows a Ctrl-C, or an impatient
+  // orchestrator sending SIGTERM twice — must not start a second teardown on
+  // top of the first one.
+  if (shuttingDown) return;
+  shuttingDown = true;
   const { logInfo: sLogInfo, logError: sLogError } = require('./nest/audit/audit-log.logger');
   const { closeMcpSessions } = require('./mcp');
-  sLogInfo(`${signal} received — shutting down gracefully...`);
-  closeMcpSessions();
-  // nestApp.close() stops every cron via the scheduling registrar's shutdown hook.
-  void nestApp?.close();
-  server.close(() => {
-    sLogInfo('HTTP server closed');
-    const { closeDb } = require('./db/database');
-    closeDb();
-    sLogInfo('Shutdown complete');
-    process.exit(0);
-  });
-  setTimeout(() => {
-    sLogError('Forced shutdown after timeout');
+  const { getServer } = require('./nest/realtime/ws-state');
+  const { runShutdown } = require('./shutdown');
+  runShutdown(signal, {
+    // Both are still undefined if the signal beat bootstrap(); the sequence
+    // handles that and still exits rather than hanging or throwing.
+    server,
+    // nestApp.close() stops every cron via the scheduling registrar's shutdown
+    // hook, and tears the plugin supervisor's forked children down.
+    closeNestApp: async () => { await nestApp?.close(); },
+    getWsClients: () => getServer()?.clients ?? null,
+    closeMcpSessions,
+    closeDb: () => { require('./db/database').closeDb(); },
+    logInfo: sLogInfo,
+    logError: sLogError,
+    exit: (code: number) => process.exit(code),
+  }).catch((err: unknown) => {
+    // Fire-and-forget would make this an unhandled rejection, which on Node 22
+    // is a crash — a worse ending than the one we are here to fix.
+    sLogError(`Shutdown failed: ${err instanceof Error ? err.message : String(err)}`);
     process.exit(1);
-  }, 10000);
+  });
 }
 
 process.on('SIGTERM', () => shutdown('SIGTERM'));

server/src/shutdown.ts

@@ -0,0 +1,165 @@
+import type { Server as HttpServer } from 'node:http';
+
+/**
+ * A ws client, as much of one as the shutdown sequence needs.
+ *
+ * Typed structurally rather than as `ws`'s WebSocket so the sequence can be
+ * tested with plain objects — the point of this module is that the thing that
+ * kept #2193 alive is finally reachable from a test.
+ */
+export interface ClosableSocket {
+  close(code: number, reason: string): void;
+  terminate(): void;
+}
+
+/** Everything a shutdown touches, handed in rather than imported. */
+export interface ShutdownDeps {
+  /**
+   * The http.Server this process listens on (bootstrap's, via getHttpServer).
+   *
+   * Undefined when the signal beats the bootstrap — an orchestrator that stops a
+   * container mid-boot, which is exactly the kind of deployment #2193 came from.
+   * There is nothing to drain in that case, and the shutdown must still reach
+   * process.exit rather than throwing into a fire-and-forget promise.
+   */
+  server: Pick<HttpServer, 'close' | 'closeIdleConnections' | 'closeAllConnections'> | undefined;
+  /** Runs every onModuleDestroy hook — crons, plugin supervisor, ws adapter. */
+  closeNestApp: () => Promise<void>;
+  /** The live ws clients, or null once the adapter has torn the server down. */
+  getWsClients: () => Iterable<ClosableSocket> | null;
+  closeMcpSessions: () => void;
+  closeDb: () => void;
+  logInfo: (message: string) => void;
+  logError: (message: string) => void;
+  exit: (code: number) => void;
+  /** How long a socket may take the polite way out before it is destroyed. */
+  drainMs?: number;
+  /** Last-resort exit, deliberately below Docker's 10s stop grace. */
+  forcedMs?: number;
+}
+
+/**
+ * How long a client gets to answer the closing handshake before its socket is
+ * destroyed. Long enough for a browser on a bad connection to reply, short
+ * enough to leave the forced exit below plenty of room.
+ */
+export const SOCKET_DRAIN_MS = 2_000;
+
+/**
+ * The fallback exit, at half of Docker's default 10s stop grace.
+ *
+ * The old value was exactly 10_000 — the same moment Docker sends SIGKILL.
+ * Docker starts counting when it sends the signal and Node only once the
+ * handler is dispatched, so that timer started strictly later and could never
+ * win. It was unreachable code in every default deployment, which is why #2193
+ * saw exit 137 rather than "Forced shutdown after timeout".
+ */
+export const FORCED_EXIT_MS = 5_000;
+
+/**
+ * Shut the process down in the order the runtime actually requires.
+ *
+ * #2193: every `docker stop` of a TREK instance with a browser attached ended
+ * in SIGKILL and exit 137. `http.Server.close()` stops NEW connections and then
+ * waits for the open ones to end by themselves — and a WebSocket never does.
+ * Node's own `closeAllConnections()` does not help either: an upgraded socket
+ * is no longer the http server's to destroy. So the close callback stayed
+ * pending forever, `closeDb()` (which lived inside it) never ran, and the only
+ * remaining exit was a timer set to the very second Docker fires SIGKILL.
+ *
+ * Waiting for Nest to do it does not work either: `WsAdapter.close()` is the
+ * one place that terminates clients, and Nest reaches it only after every
+ * `onModuleDestroy` hook has resolved — including the plugin supervisor's own
+ * SIGTERM grace for its forked children. The sockets holding the server open
+ * would be hostage to the slowest hook in the container.
+ *
+ * Hence this order, and it is the whole fix:
+ *
+ *  1. ws clients get 1001 "going away" first, so a browser sees a real close
+ *     frame and reconnects the quiet way. The client ignores the code and
+ *     reconnects on any close (client/src/api/websocket.ts), so this is a
+ *     nicety, not a contract change.
+ *  2. `closeIdleConnections()` drops keep-alive sockets parked between
+ *     requests — a reverse proxy always holds a few.
+ *  3. After `drainMs`, anything still attached is destroyed: `terminate()` for
+ *     the ws clients that never answered the handshake, `closeAllConnections()`
+ *     for requests still in flight. A shutdown that waits on an unbounded
+ *     upload is the same hang somewhere else.
+ *
+ * Nest is closed alongside that, and the DB is closed exactly once on BOTH exit
+ * paths rather than only on the one that used to be unreachable.
+ */
+export async function runShutdown(signal: string, deps: ShutdownDeps): Promise<void> {
+  const {
+    server, closeNestApp, getWsClients, closeMcpSessions, closeDb,
+    logInfo, logError, exit,
+    drainMs = SOCKET_DRAIN_MS, forcedMs = FORCED_EXIT_MS,
+  } = deps;
+
+  logInfo(`${signal} received — shutting down gracefully...`);
+
+  const guard = (what: string, fn: () => void): void => {
+    try {
+      fn();
+    } catch (err) {
+      // One step failing must not strand the process in a half-shutdown: log it
+      // and carry on to the next, because the alternative is the SIGKILL again.
+      logError(`${what} failed during shutdown: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  };
+
+  let finished = false;
+  /** The single exit path, so the DB is closed exactly once and never skipped. */
+  const finish = (code: number, message: string): void => {
+    if (finished) return;
+    finished = true;
+    // WAL-mode SQLite recovers on the next boot, so a DB that refuses to close
+    // is worth a line in the log and nothing more — never a reason to hang.
+    guard('closeDb', closeDb);
+    logInfo(message);
+    exit(code);
+  };
+
+  // Armed first, so a hang anywhere below still ends in OUR exit rather than in
+  // Docker's SIGKILL. unref'd: it must never be why the loop stays alive.
+  const forced = setTimeout(() => {
+    logError('Forced shutdown after timeout');
+    finish(1, 'Shutdown complete (forced)');
+  }, forcedMs);
+  forced.unref?.();
+
+  guard('closeMcpSessions', closeMcpSessions);
+
+  // Before Nest closes: its adapter's close() drops the server reference, and
+  // after that there is nothing left to say goodbye to.
+  const clients: ClosableSocket[] = [];
+  guard('ws teardown', () => {
+    for (const client of getWsClients() ?? []) clients.push(client);
+  });
+  for (const client of clients) {
+    guard('ws close', () => client.close(1001, 'Server shutting down'));
+  }
+
+  const httpClosed = server
+    ? new Promise<void>((resolve) => {
+        server.close(() => {
+          logInfo('HTTP server closed');
+          resolve();
+        });
+      })
+    : Promise.resolve();
+  if (server) guard('closeIdleConnections', () => server.closeIdleConnections());
+
+  const drain = setTimeout(() => {
+    // Whatever is still holding the server open at this point is not going to
+    // let go on its own.
+    for (const client of clients) guard('ws terminate', () => client.terminate());
+    if (server) guard('closeAllConnections', () => server.closeAllConnections());
+  }, drainMs);
+  drain.unref?.();
+
+  await Promise.allSettled([httpClosed, closeNestApp()]);
+  clearTimeout(drain);
+  clearTimeout(forced);
+  finish(0, 'Shutdown complete');
+}