Load Balancing Components Explained — How Traffic Gets Distributed at Scale

# Upstream pool with mixed weights and a backup server
upstream forge_backend_cluster {
    # Least Connections: best for workloads with variable request processing times
    # Round Robin (default) assumes all requests take roughly the same time — often wrong
    least_conn;

    server 10.0.0.1:8080 weight=3;  # Larger instance — gets 3x the traffic
    server 10.0.0.2:8080 weight=1;  # Standard instance
    server 10.0.0.3:8080 backup;    # Only receives traffic when primary servers are down

    # Keepalive: reuse connections to backends instead of opening new TCP connections per request
    # Without this, high-traffic scenarios create connection exhaustion
    keepalive 32;

    # Passive health checks: mark server down after 3 consecutive failures, retry after 30s
    # Active health checks (requires nginx_upstream_check_module or NGINX Plus)
    # server 10.0.0.1:8080 max_fails=3 fail_timeout=30s;
}

server {
    listen 80;
    server_name thecodeforge.io;

    location /api/ {
        proxy_pass http://forge_backend_cluster;

        # Preserve real client IP through the proxy
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $host;

        # Timeout configuration — tune for your backend's actual SLA
        proxy_connect_timeout 5s;
        proxy_read_timeout 60s;
        proxy_send_timeout 60s;

        # Retry on failure — but only for idempotent methods to avoid double-posting
        proxy_next_upstream error timeout http_503;
        proxy_next_upstream_tries 2;
    }
}

package io.thecodeforge.loadbalancer;

import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.CopyOnWriteArrayList;

/**
 * Production-grade Weighted Round-Robin implementation.
 *
 * Design decisions worth understanding:
 * - AtomicInteger for the index counter: load balancers handle concurrent requests.
 *   A plain int here is a data race waiting to happen.
 * - CopyOnWriteArrayList for the server pool: allows safe dynamic weight updates
 *   without locking the hot path (getNextServer).
 * - Collections.shuffle() on construction: prevents all initial traffic from
 *   hitting server[0] in a predictable burst during startup.
 * - Math.abs() on the modulo result: AtomicInteger.getAndIncrement() eventually
 *   overflows to negative values. Without abs(), you get an ArrayIndexOutOfBoundsException
 *   at 2^31 requests — a production bug you will not find in testing.
 */
public class WeightedRoundRobinBalancer {

    private final List<String> serverPool;
    private final AtomicInteger currentIndex = new AtomicInteger(0);

    public WeightedRoundRobinBalancer(Map<String, Integer> serversWithWeights) {
        List<String> pool = new ArrayList<>();
        for (Map.Entry<String, Integer> entry : serversWithWeights.entrySet()) {
            for (int i = 0; i < entry.getValue(); i++) {
                pool.add(entry.getKey());
            }
        }
        // Shuffle to prevent predictable startup burst on the first server
        Collections.shuffle(pool);
        // CopyOnWriteArrayList: safe reads on the hot path, supports dynamic updates
        this.serverPool = new CopyOnWriteArrayList<>(pool);
    }

    public String getNextServer() {
        if (serverPool.isEmpty()) {
            throw new IllegalStateException(
                "No active servers in the pool. Check health checks and server registration."
            );
        }
        // Math.abs handles integer overflow at 2^31 requests
        int index = Math.abs(currentIndex.getAndIncrement() % serverPool.size());
        return serverPool.get(index);
    }

    /**
     * Dynamically update a server's weight — e.g., temporarily reduce weight
     * for a server showing elevated GC pause times or increased error rate.
     * In production, this would be called by your health monitoring system.
     */
    public synchronized void updateWeight(String server, int newWeight, int oldWeight) {
        // Remove existing entries for this server
        serverPool.removeIf(s -> s.equals(server));
        // Re-add with new weight
        for (int i = 0; i < newWeight; i++) {
            serverPool.add(server);
        }
        System.out.printf("Weight updated: %s  %d → %d (pool size: %d)%n",
            server, oldWeight, newWeight, serverPool.size());
    }

    public static void main(String[] args) {
        Map<String, Integer> config = new LinkedHashMap<>();
        config.put("app-server-large-01", 5);  // High-capacity instance
        config.put("app-server-std-01",   2);  // Standard instance
        config.put("app-server-std-02",   2);  // Standard instance

        WeightedRoundRobinBalancer balancer = new WeightedRoundRobinBalancer(config);

        System.out.println("Initial distribution across 18 requests:");
        Map<String, Integer> distribution = new LinkedHashMap<>();
        for (int i = 0; i < 18; i++) {
            String server = balancer.getNextServer();
            distribution.merge(server, 1, Integer::sum);
        }
        distribution.forEach((server, count) ->
            System.out.printf("  %-25s → %d requests%n", server, count));

        // Simulate degraded server — reduce its weight
        System.out.println("\nSimulating GC pressure on app-server-large-01...");
        balancer.updateWeight("app-server-large-01", 1, 5);
        System.out.println("Server temporarily downweighted. Rebalancing traffic.");
    }
}

const express = require('express');
const { createClient } = require('redis');
const { Pool } = require('pg');

const app = express();

// Initialize dependencies
const redisClient = createClient({ url: process.env.REDIS_URL });
const pgPool = new Pool({ connectionString: process.env.DATABASE_URL });

redisClient.connect().catch(console.error);

/**
 * Shallow health check — fast, for high-frequency LB probing.
 * Returns 200 if the process is alive. Does not check dependencies.
 * Use this for the LB's frequent interval check (every 5s).
 */
app.get('/health/live', (req, res) => {
  res.status(200).json({
    status: 'alive',
    pid: process.pid,
    uptime: process.uptime(),
  });
});

/**
 * Deep readiness check — validates all dependencies.
 * Returns 200 only when the application can actually serve traffic.
 * Use this as the LB's readiness gate during startup and deployment.
 * Check interval should be longer (every 10–15s) due to dependency I/O.
 */
app.get('/health/ready', async (req, res) => {
  const checks = {};
  let allHealthy = true;

  // Check database connectivity
  try {
    const client = await pgPool.connect();
    await client.query('SELECT 1');
    client.release();
    checks.database = { status: 'healthy' };
  } catch (err) {
    checks.database = { status: 'unhealthy', error: err.message };
    allHealthy = false;
  }

  // Check Redis connectivity
  try {
    await redisClient.ping();
    checks.redis = { status: 'healthy' };
  } catch (err) {
    checks.redis = { status: 'unhealthy', error: err.message };
    allHealthy = false;
  }

  // Memory pressure check — prevent routing to a server about to OOM
  const memUsage = process.memoryUsage();
  const heapUsedPercent = memUsage.heapUsed / memUsage.heapTotal;
  if (heapUsedPercent > 0.90) {
    checks.memory = {
      status: 'degraded',
      heapUsedPercent: (heapUsedPercent * 100).toFixed(1) + '%',
    };
    allHealthy = false;
  } else {
    checks.memory = {
      status: 'healthy',
      heapUsedPercent: (heapUsedPercent * 100).toFixed(1) + '%',
    };
  }

  const statusCode = allHealthy ? 200 : 503;
  res.status(statusCode).json({
    status: allHealthy ? 'ready' : 'not_ready',
    checks,
    pid: process.pid,
    timestamp: new Date().toISOString(),
  });
});

app.listen(8080, () => console.log('Health server on :8080'));