Event Loop Block: 4-Second Template Compile in Node.js

const fs = require('fs');

// Demonstrates the phase ordering — run this and study the output
fs.readFile(__filename, () => {
  // We are now inside an I/O callback (poll phase)

  setTimeout(() => console.log('3: setTimeout'), 0);
  // Will run in the NEXT iteration's timer phase

  setImmediate(() => console.log('2: setImmediate'));
  // Will run in THIS iteration's check phase — before setTimeout

  Promise.resolve().then(() => console.log('1: Promise.then'));
  // Microtask — runs before check phase

  process.nextTick(() => console.log('0: nextTick'));
  // Microtask — runs before Promise.then, before any phase
});

// Output order:
// 0: nextTick       (microtask, highest priority — before any phase transition)
// 1: Promise.then   (microtask, runs after nextTick queue is empty)
// 2: setImmediate   (check phase — this iteration)
// 3: setTimeout     (timer phase — NEXT iteration)


// ── Production pattern: yield the event loop during batch processing ───────
// Without yielding, processing 100,000 items blocks the loop for the full duration.
// With setImmediate between batches, I/O and other requests can interleave.
function processBatchWithYield(items, batchSize, processFn, onComplete) {
  let index = 0;

  function processNextBatch() {
    const batchEnd = Math.min(index + batchSize, items.length);

    // Process one batch synchronously
    for (; index < batchEnd; index++) {
      processFn(items[index]);
    }

    if (index < items.length) {
      // Yield to the event loop — allows pending I/O and HTTP requests to run
      // setImmediate is better than setTimeout(fn, 0) here:
      // no artificial 1ms delay, runs in the check phase of the current iteration
      setImmediate(processNextBatch);
    } else {
      onComplete();
    }
  }

  processNextBatch();
}

// Usage:
processBatchWithYield(
  largeArray,
  500,           // Process 500 items per batch before yielding
  (item) => transform(item),
  () => console.log('All items processed')
);


// ── Measuring actual event loop lag in production ──────────────────────────
// Schedule a task, measure how long it actually takes to execute vs expected.
// The delta is your event loop lag.
function measureEventLoopLag() {
  const INTERVAL_MS = 1000;
  let lastCheck = Date.now();

  setInterval(() => {
    const now = Date.now();
    const lag = now - lastCheck - INTERVAL_MS;
    lastCheck = now;

    if (lag > 50) {
      console.warn(`Event loop lag: ${lag}ms — investigate blocking operations`);
    }
    // In production: expose this as a Prometheus gauge
  }, INTERVAL_MS);
}

measureEventLoopLag();

const cluster = require('cluster');
const http = require('http');
const os = require('os');

// In K8s: WEB_CONCURRENCY can be injected as an env var matching allocated CPUs.
// In VMs/bare metal: default to os.cpus().length.
// Never hard-code a number — it breaks when the container size changes.
const WORKER_COUNT = parseInt(process.env.WEB_CONCURRENCY, 10) || os.cpus().length;
const PORT = parseInt(process.env.PORT, 10) || 3000;

if (cluster.isPrimary) {
  console.log(`Primary ${process.pid} — forking ${WORKER_COUNT} workers`);

  // Track each worker's lifecycle separately
  const workerMeta = new Map();

  for (let i = 0; i < WORKER_COUNT; i++) {
    spawnWorker();
  }

  function spawnWorker() {
    const worker = cluster.fork();
    workerMeta.set(worker.id, {
      pid: worker.process.pid,
      startedAt: Date.now(),
      crashCount: 0,
    });
    console.log(`Worker ${worker.id} (pid ${worker.process.pid}) started`);
    return worker;
  }

  cluster.on('exit', (worker, code, signal) => {
    const meta = workerMeta.get(worker.id);
    meta.crashCount++;

    const reason = signal || `exit code ${code}`;
    console.error(`Worker ${worker.id} exited (${reason}). Crash count: ${meta.crashCount}`);

    // Crash loop protection: if a worker crashes more than 5 times,
    // something is fundamentally wrong. Don't keep restarting it.
    // In K8s, the pod will be replaced. In a VM, alert and investigate.
    if (meta.crashCount > 5) {
      console.error(`Worker ${worker.id} has crashed ${meta.crashCount} times — stopping restarts`);
      // Remove from tracking so we don't endlessly accumulate stale entries
      workerMeta.delete(worker.id);
      return;
    }

    // Brief delay before respawning to avoid crash storms on startup failures
    setTimeout(() => spawnWorker(), 500);
  });

  // Rolling restart: send shutdown signal to each worker in sequence,
  // wait for it to drain and exit, then restart it.
  // This achieves zero-downtime restarts without PM2.
  async function rollingRestart() {
    const workerIds = Object.keys(cluster.workers);
    for (const id of workerIds) {
      await new Promise((resolve) => {
        const worker = cluster.workers[id];
        if (!worker) { resolve(); return; }

        worker.send('graceful-shutdown');
        worker.once('exit', () => {
          spawnWorker();
          // Give the new worker 2 seconds to initialise before draining the next one
          setTimeout(resolve, 2000);
        });
      });
    }
    console.log('Rolling restart complete');
  }

  process.on('SIGUSR1', rollingRestart);

  process.on('SIGTERM', () => {
    console.log('Primary received SIGTERM — initiating graceful shutdown');
    for (const id in cluster.workers) {
      cluster.workers[id].send('graceful-shutdown');
      cluster.workers[id].disconnect();
    }
  });

} else {
  // Worker process — this is where your actual application runs
  const server = http.createServer((req, res) => {
    res.writeHead(200, { 'Content-Type': 'text/plain' });
    res.end(`Worker ${cluster.worker.id} (pid ${process.pid}) handled this request`);
  });

  server.listen(PORT, () => {
    console.log(`Worker ${cluster.worker.id} listening on port ${PORT}`);
  });

  // Handle graceful shutdown: stop accepting new connections,
  // finish in-flight requests, then exit cleanly.
  process.on('message', (msg) => {
    if (msg === 'graceful-shutdown') {
      console.log(`Worker ${cluster.worker.id} shutting down gracefully`);
      server.close(() => {
        console.log(`Worker ${cluster.worker.id} shutdown complete`);
        process.exit(0);
      });

      // Force exit if graceful shutdown takes too long (e.g., a stuck WebSocket)
      setTimeout(() => {
        console.error(`Worker ${cluster.worker.id} forced exit after timeout`);
        process.exit(1);
      }, 30_000);
    }
  });

  // Catch unhandled rejections at the worker level
  // Log them fully before the process exits — silent rejections lose stack traces
  process.on('unhandledRejection', (reason, promise) => {
    console.error('Unhandled rejection in worker', cluster.worker.id, reason);
    // Depending on the severity, you may want to exit here:
    // process.exit(1);
  });
}

// ── cpu-worker.js ─────────────────────────────────────────────────────────
// This file runs inside each worker thread.
// It receives work via workerData, does computation, and posts the result back.

const { parentPort, workerData, threadId } = require('worker_threads');
const crypto = require('crypto');

function computeHashes(count) {
  // This is genuinely CPU-bound — no I/O, pure computation.
  // Running this on the main thread would block the event loop for the full duration.
  const results = new Array(count);
  for (let i = 0; i < count; i++) {
    results[i] = crypto
      .createHash('sha256')
      .update(`payload-${i}-${Date.now()}`)
      .digest('hex');
  }
  return results;
}

parentPort.postMessage({
  threadId,
  count: workerData.iterations,
  sample: computeHashes(workerData.iterations)[0],
});


// ── thread-pool.js ────────────────────────────────────────────────────────
// Production pattern: fixed-size thread pool using piscina.
// Never spawn a thread per request. Queue excess work.

const Piscina = require('piscina');
const path = require('path');

// Create the pool once at application startup — not per request.
// maxThreads should roughly match CPU core count.
// idleTimeout: threads that have been idle for 30s are terminated to free memory.
const pool = new Piscina({
  filename: path.resolve(__dirname, 'cpu-worker.js'),
  maxThreads: parseInt(process.env.WORKER_THREADS, 10) || (require('os').cpus().length - 1),
  idleTimeout: 30_000,
});

// In your Express/Fastify route handler:
async function handleHashRequest(req, res) {
  const { iterations = 1000 } = req.query;

  // Cap iterations to prevent intentional DoS via this endpoint
  const safeIterations = Math.min(parseInt(iterations, 10), 10_000);

  try {
    // This runs in a worker thread — the event loop is completely free
    // to handle other requests while this computation runs.
    const result = await pool.run({ iterations: safeIterations });
    res.json({ threadId: result.threadId, count: result.count, sample: result.sample });
  } catch (err) {
    // piscina surfaces worker errors as rejected promises
    console.error('Worker thread error:', err);
    res.status(500).json({ error: 'Computation failed' });
  }
}

// Monitor pool queue depth — if it grows unboundedly, you need more threads
// or you need to reject requests earlier with a 503
setInterval(() => {
  if (pool.queueSize > 50) {
    console.warn(`Thread pool queue depth: ${pool.queueSize} — consider increasing maxThreads or load shedding`);
  }
}, 5_000);


// ── SharedArrayBuffer pattern for zero-copy data sharing ──────────────────
// When passing large datasets to worker threads, avoid serialisation overhead
// by sharing the underlying ArrayBuffer directly.

const { Worker } = require('worker_threads');

function processLargeDataset(data) {
  // Convert data to a typed array for sharing
  const sharedBuffer = new SharedArrayBuffer(data.length * 4);
  const sharedView = new Int32Array(sharedBuffer);

  // Copy data into shared buffer — this is the only copy
  data.forEach((val, i) => { sharedView[i] = val; });

  return new Promise((resolve, reject) => {
    const worker = new Worker(`
      const { workerData, parentPort } = require('worker_threads');
      const view = new Int32Array(workerData.buffer);
      // Process in-place — no additional memory allocation for the data itself
      let sum = 0;
      for (let i = 0; i < view.length; i++) sum += view[i];
      parentPort.postMessage({ sum });
    `, {
      eval: true,
      workerData: { buffer: sharedBuffer } // Transferred, not copied
    });

    worker.on('message', resolve);
    worker.on('error', reject);
  });
}

// ══════════════════════════════════════════════════════════════════
// COMMON PRODUCTION LEAK PATTERNS — what they look like and how to fix them
// ══════════════════════════════════════════════════════════════════

// ── PATTERN 1: Unbounded in-memory cache ──────────────────────────
// The most common leak in production Node.js services.
// Works fine in staging (low traffic, frequent restarts).
// Reaches OOM in production after 12-48 hours of sustained traffic.

const BAD_cache = {};
function getUserBad(id) {
  if (!BAD_cache[id]) {
    BAD_cache[id] = fetchFromDb(id); // Added forever, never evicted
  }
  return BAD_cache[id];
}
// After 100,000 unique users: BAD_cache has 100,000 entries eating memory

// FIX: LRU cache with both max size and TTL
const { LRUCache } = require('lru-cache');
const userCache = new LRUCache({
  max: 10_000,           // Evict least-recently-used entries beyond this count
  ttl: 1000 * 60 * 5,   // Each entry expires after 5 minutes regardless of access
  allowStale: false,     // Don't serve expired entries even while revalidating
});

function getUser(id) {
  const cached = userCache.get(id);
  if (cached) return cached;
  const user = fetchFromDb(id);
  userCache.set(id, user);
  return user;
}


// ── PATTERN 2: Event listener accumulation ────────────────────────
// Each reconnection adds a new listener. removeListener is never called.
// process.on('warning') will emit MaxListenersExceededWarning at 11 listeners,
// but by then you might already have hundreds.

function handleConnectionBad(socket) {
  const onData = (chunk) => processChunk(socket, chunk);
  socket.on('data', onData);
  // BUG: socket 'close' event never removes 'data' listener
  // If this socket is reused or the same emitter receives multiple calls,
  // listeners pile up indefinitely
}

// FIX: always pair addListener with removeListener
function handleConnection(socket) {
  const onData = (chunk) => processChunk(socket, chunk);
  socket.on('data', onData);

  socket.once('close', () => {
    socket.removeListener('data', onData);
    // 'once' ensures this cleanup handler itself doesn't accumulate
  });
}

// For high-connection-volume services, set explicit listener limits:
socket.setMaxListeners(3); // data + error + close — that's all this socket needs


// ── PATTERN 3: Closure retaining large objects ────────────────────
// The closure captures the entire scope, including objects it doesn't use.
// Returned functions keep those objects alive in the old generation forever.

function BAD_createMiddleware() {
  const requestLog = []; // Grows with every request — never cleared

  return function middleware(req, res, next) {
    requestLog.push(req); // Holds every req object ever received
    next();
  };
}

// FIX: only retain what you actually need
function createMiddleware() {
  const requestCount = { value: 0 }; // Tiny counter, not the whole req object

  return function middleware(req, res, next) {
    requestCount.value++;
    next();
  };
}


// ── PATTERN 4: setInterval without clearInterval ──────────────────
// Common in module-level setup code — the interval callback fires forever
// and holds references to everything in its closure scope.

let pollingInterval;
function startPolling(config) {
  // GOOD: store the reference so we can clear it
  pollingInterval = setInterval(async () => {
    await pollRemoteService(config);
  }, 5000);
}

function stopPolling() {
  if (pollingInterval) {
    clearInterval(pollingInterval);
    pollingInterval = null;
  }
}

// Always hook into process shutdown to clean up timers:
process.on('SIGTERM', () => {
  stopPolling();
  // then close server, drain DB pool, etc.
});


// ── Heap snapshot helper for production debugging ─────────────────
// Add this to a debug-only route or triggered by a signal.
// Never leave heap snapshot generation on the hot path — it pauses the event loop.
const v8 = require('v8');

process.on('SIGUSR2', () => {
  const filename = `heap-${Date.now()}.heapsnapshot`;
  v8.writeHeapSnapshot(filename);
  console.log(`Heap snapshot written to ${filename}`);
  // Copy the file off the instance and open in Chrome DevTools Memory tab
});

# ── Install clinic globally (requires Node.js 18+, works on 22 LTS) ────────
npm install -g clinic

# Verify installation:
clinic --version


# ═════════════════════════════════════════════════════════════════════
# STEP 1: Full diagnostic overview — start here, always
# ═════════════════════════════════════════════════════════════════════

clinic doctor -- node server.js
# While this runs, apply load in a separate terminal:
npx autocannon -c 100 -d 30 http://localhost:3000
# clinic doctor generates an HTML report showing:
#   - Event loop delay over time (spikes = blocking operations)
#   - CPU usage across the profile period
#   - Memory growth trend
#   - Recommendations based on detected patterns


# ═════════════════════════════════════════════════════════════════════
# STEP 2: CPU hotspot identification (when doctor shows high CPU)
# ═════════════════════════════════════════════════════════════════════

clinic flame -- node server.js
# Apply load while running.
# The flamegraph shows call stacks with width proportional to CPU time.
# Wide boxes at the top of the stack are your hotspots.
# Look for synchronous JavaScript in the middle of the stack — anything
# that should be async but isn't shows up as a wide synchronous column.


# ═════════════════════════════════════════════════════════════════════
# STEP 3: Memory allocation profiling (when memory grows unexpectedly)
# ═════════════════════════════════════════════════════════════════════

clinic heapprofiler -- node server.js
# Shows an allocation timeline — not just what IS in the heap,
# but what is being actively allocated over time.
# Useful distinction: a large retained object is a leak.
# High allocation rate that GC keeps up with is a GC pressure issue, not a leak.


# ═════════════════════════════════════════════════════════════════════
# STEP 4: Async operation flow analysis (when timing looks wrong)
# ═════════════════════════════════════════════════════════════════════

clinic bubbleprof -- node server.js
# Visualises async operation chains — shows where time is spent waiting
# between async operations. Useful for finding:
#   - Database query chains that could be parallelised with Promise.all()
#   - Missing connection pool capacity (lots of time waiting for a connection)
#   - Unnecessary sequential async operations


# ═════════════════════════════════════════════════════════════════════
# STEP 5: Manual event loop lag measurement (production-safe)
# ═════════════════════════════════════════════════════════════════════

# Option A: quick terminal check
node -e "
  const INTERVAL = 1000;
  let last = Date.now();
  setInterval(() => {
    const now = Date.now();
    const lag = now - last - INTERVAL;
    last = now;
    console.log('Event loop lag:', lag.toFixed(0), 'ms');
    if (lag > 100) console.warn('WARNING: event loop saturation detected');
  }, INTERVAL);
"

# Option B: expose as a Prometheus metric (recommended for production)
# In your application startup:
# const promClient = require('prom-client');
# promClient.collectDefaultMetrics(); // includes nodejs_eventloop_lag_seconds


# ═════════════════════════════════════════════════════════════════════
# STEP 6: V8 CPU profile via --inspect (for interactive investigation)
# ═════════════════════════════════════════════════════════════════════

# Start the process with the inspector enabled:
node --inspect=0.0.0.0:9229 server.js

# Then:
# 1. Open Chrome and navigate to chrome://inspect
# 2. Click 'inspect' on your Node.js target
# 3. Go to the Profiler tab
# 4. Click 'Start' — apply load — click 'Stop'
# 5. Analyse the CPU profile flamegraph

# For production canary: bind inspector to localhost only
node --inspect=127.0.0.1:9229 server.js
# Tunnel via SSH to access from your laptop:
# ssh -L 9229:localhost:9229 user@canary-host


# ═════════════════════════════════════════════════════════════════════
# STEP 7: Heap snapshot on demand (production debugging)
# ═════════════════════════════════════════════════════════════════════

# If you added process.on('SIGUSR2') for heap snapshots:
kill -USR2 <node-pid>
# File appears in the working directory as heap-<timestamp>.heapsnapshot
# Load it in Chrome DevTools > Memory tab > Load profile

const async_hooks = require('async_hooks');
const fs = require('fs');

const contexts = new Map();

const hook = async_hooks.createHook({
  init(asyncId, type, triggerAsyncId) {
    if (type === 'MIDDLEWARE') {
      contexts.set(asyncId, Date.now());
    }
  },
  destroy(asyncId) {
    if (contexts.has(asyncId)) {
      const elapsed = Date.now() - contexts.get(asyncId);
      fs.appendFileSync('middleware-profile.log', `Middleware ${asyncId}: ${elapsed}ms\n`);
      contexts.delete(asyncId);
    }
  }
});
hook.enable();

// Usage in your app
app.use((req, res, next) => {
  // Trigger async hook with type 'MIDDLEWARE'
  const asyncId = async_hooks.executionAsyncId();
  next();
});

app.get('/api/users', (req, res) => {
  res.json({ message: 'Profile your middleware chain' });
});

// Sample output:
// Middleware 12: 3ms
// Middleware 13: 12ms  <-- THIS ONE IS THE PROBLEM
// Middleware 14: 1ms

const { Pool } = require('pg');

const pool = new Pool({
  host: process.env.DB_HOST,
  port: 5432,
  database: 'myapp',
  user: process.env.DB_USER,
  password: process.env.DB_PASS,
  max: 20,                // Never exceed DB's max_connections - 20%
  idleTimeoutMillis: 30000, // Close idle connections after 30s
  connectionTimeoutMillis: 5000, // Fail fast if DB is down
  maxUses: 7500,           // Recycle connection after 7500 queries
  queueLimit: 0,           // Allow unlimited queue
  allowExitOnIdle: true    // Let Node.js exit if pool is idle
});

// Monitor queue depth
setInterval(async () => {
  const stats = {
    totalCount: pool.totalCount,
    idleCount: pool.idleCount,
    waitingCount: pool.waitingCount,
  };
  console.log('Pool stats:', stats);
  // If waitingCount > 0 for 30 seconds, alert!
}, 10000);

// Sample output:
// Pool stats: { totalCount: 5, idleCount: 3, waitingCount: 0 }
// Pool stats: { totalCount: 12, idleCount: 0, waitingCount: 8 }  <-- PROBLEM