Docker Overlay Network MTU — Silent Failures at Scale

#!/bin/bash
# Production architecture setup and verification

# ── Docker Swarm: production setup ───────────────────────────────────────────

# Initialize the swarm on the manager node
docker swarm init --advertise-addr <manager-ip>
# Output: join token for worker nodes

# Join worker nodes
docker swarm join --token <token> <manager-ip>:2377

# Verify cluster status
docker node ls
# ID    HOSTNAME    STATUS    AVAILABILITY    MANAGER STATUS
# abc*  manager1    Ready     Active          Leader
# def   worker1     Ready     Active
# ghi   worker2     Ready     Active

# Create a production service with resource limits
docker service create \
  --name io-thecodeforge-api \
  --replicas 3 \
  --limit-cpu 1.0 \
  --limit-memory 512m \
  --reserve-cpu 0.5 \
  --reserve-memory 256m \
  --publish published=80,target=3000 \
  --update-parallelism 1 \
  --update-delay 10s \
  --update-failure-action rollback \
  --update-max-failure-ratio 0.25 \
  --health-cmd 'curl -f http://localhost:3000/health || exit 1' \
  --health-interval 10s \
  --health-timeout 5s \
  --health-retries 3 \
  --network app-overlay \
  registry.example.com/io-thecodeforge/api:1.0.0

# Verify service status
docker service ls
# ID    NAME                    MODE        REPLICAS  IMAGE
# abc   io-thecodeforge-api     replicated  3/3       registry.example.com/io-thecodeforge/api:1.0.0

# Check service logs across all replicas
docker service logs io-thecodeforge-api --tail 50 --follow

# ── Kubernetes: production deployment manifest ───────────────────────────────
cat <<'EOF' > /tmp/io-thecodeforge-api-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: io-thecodeforge-api
  namespace: production
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: io-thecodeforge-api
  template:
    metadata:
      labels:
        app: io-thecodeforge-api
    spec:
      containers:
      - name: api
        image: registry.example.com/io-thecodeforge/api:1.0.0
        ports:
        - containerPort: 3000
        resources:
          requests:
            cpu: 500m
            memory: 256Mi
          limits:
            cpu: 1000m
            memory: 512Mi
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 15
          periodSeconds: 10
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 5
          periodSeconds: 5
          failureThreshold: 3
EOF

kubectl apply -f /tmp/io-thecodeforge-api-deployment.yaml
kubectl -n production rollout status deployment/io-thecodeforge-api

# ── AWS ECS: production task definition ───────────────────────────────────────
cat <<'EOF' > /tmp/io-thecodeforge-api-task.json
{
  "family": "io-thecodeforge-api",
  "networkMode": "awsvpc",
  "requiresCompatibilities": ["FARGATE"],
  "cpu": "1024",
  "memory": "2048",
  "containerDefinitions": [
    {
      "name": "api",
      "image": "123456789.dkr.ecr.us-east-1.amazonaws.com/io-thecodeforge/api:1.0.0",
      "portMappings": [{"containerPort": 3000, "protocol": "tcp"}],
      "healthCheck": {
        "command": ["CMD-SHELL", "curl -f http://localhost:3000/health || exit 1"],
        "interval": 30,
        "timeout": 5,
        "retries": 3,
        "startPeriod": 60
      },
      "logConfiguration": {
        "logDriver": "awsfirelens",
        "options": {
          "Name": "cloudwatch",
          "region": "us-east-1",
          "log_group_name": "/ecs/io-thecodeforge-api",
          "auto_create_group": "true"
        }
      }
    }
  ]
}
EOF

aws ecs register-task-definition --cli-input-json file:///tmp/io-thecodeforge-api-task.json

#!/bin/bash
# Production resource management configuration and monitoring

# ── CPU limits ───────────────────────────────────────────────────────────────

# Run with 1 CPU core limit
docker run --cpus=1.0 --name cpu-test alpine:3.19 stress --cpu 2 --timeout 10s
# The container is throttled to 1 CPU even if stress spawns 2 workers

# Check CPU throttling
cat /sys/fs/cgroup/cpu/docker/<container-id>/cpu.stat
# nr_periods: total scheduling periods
# nr_throttled: periods where the container was throttled
# throttled_time: total time throttled (nanoseconds)

# Check CPU shares (relative priority)
cat /sys/fs/cgroup/cpu/docker/<container-id>/cpu.shares
# Default: 1024. Set with --cpu-shares=512 for lower priority

# ── Memory limits ────────────────────────────────────────────────────────────

# Run with 256MB memory limit
docker run --memory=256m --memory-swap=256m --name mem-test alpine:3.19 stress --vm 1 --vm-bytes 300M --timeout 10s
# Container is OOM-killed because it exceeds 256MB limit

# Check memory usage before OOM
cat /sys/fs/cgroup/memory/docker/<container-id>/memory.usage_in_bytes
cat /sys/fs/cgroup/memory/docker/<container-id>/memory.limit_in_bytes
cat /sys/fs/cgroup/memory/docker/<container-id>/memory.max_usage_in_bytes

# Check OOM events
dmesg | grep -i 'oom\|killed process' | tail -10
# [12345.678] Out of memory: Killed process 5678 (node) total-vm:123456kB, anon-rss:98765kB

# ── OOM score management ────────────────────────────────────────────────────

# Check a container's OOM score
CONTAINER_PID=$(docker inspect <container> --format '{{.State.Pid}}')
cat /proc/$CONTAINER_PID/oom_score
# 0-1000: higher = more likely to be killed

cat /proc/$CONTAINER_PID/oom_score_adj
# -1000 to 1000: adjust the score

# Set OOM priority for critical services (database)
docker run --oom-score-adj=-999 --name critical-db postgres:16
# This container is almost never killed by the OOM killer

# Set OOM priority for non-critical services (cache)
docker run --oom-score-adj=1000 --name expendable-cache redis:7
# This container is killed first in an OOM situation

# ── Kubernetes resource management ──────────────────────────────────────────

# Guaranteed QoS: requests == limits (never evicted for resource reasons)
cat <<'EOF'
resources:
  requests:
    cpu: 1000m
    memory: 512Mi
  limits:
    cpu: 1000m
    memory: 512Mi
EOF

# Burstable QoS: requests < limits (can burst but may be throttled/killed)
cat <<'EOF'
resources:
  requests:
    cpu: 500m
    memory: 256Mi
  limits:
    cpu: 1000m
    memory: 512Mi
EOF

# ── Monitor resource usage across all containers ─────────────────────────────

# Real-time resource usage
docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}'

# Find containers without resource limits
docker ps -q | xargs -I{} docker inspect {} --format '{{.Name}}: CPU={{.HostConfig.NanoCpus}} MEM={{.HostConfig.Memory}}'
# Containers with NanoCpus=0 or Memory=0 have no limits

# Host-level resource check
free -h
cat /proc/loadavg
uptime

#!/bin/bash
# Production networking configuration and debugging

# ── Docker Swarm overlay network ─────────────────────────────────────────────

# Create an overlay network with correct MTU
docker network create \
  --driver overlay \
  --opt com.docker.network.driver.mtu=8950 \
  --subnet 10.0.0.0/24 \
  --gateway 10.0.0.1 \
  app-overlay
# MTU calculation: VPC MTU (9001) - VXLAN overhead (50) = 8951, round to 8950

# Verify overlay network
docker network inspect app-overlay --format '{{.Driver}} {{.Options}}'
# overlay map[com.docker.network.driver.mtu:8950]

# ── DNS resolution verification ──────────────────────────────────────────────

# Check embedded DNS server
docker exec <container> cat /etc/resolv.conf
# nameserver 127.0.0.11
# options ndots:0

# Resolve a service name
docker exec <container> nslookup io-thecodeforge-api
# Server: 127.0.0.11
# Address: 10.0.0.5

# Check DNS query logs (Docker daemon)
sudo journalctl -u docker | grep 'DNS query' | tail -10

# ── Network health checks ───────────────────────────────────────────────────

# Check overlay network peer status
docker network inspect app-overlay --format '{{.Peers}}'
# Shows all nodes participating in the overlay

# Check IP fragment queue (critical for overlay networks)
cat /proc/net/snmp | grep -i frag
# Ip: FragCreates  FragOKs  FragFails
# If FragFails > 0, packets are being dropped due to fragment queue overflow

# Check MTU of container interface
docker exec <container> cat /sys/class/net/eth0/mtu
# Should match the overlay network MTU (8950)

# ── Traffic debugging with tcpdump ───────────────────────────────────────────

# Capture traffic on the overlay bridge
sudo tcpdump -i docker_gwbridge -n -c 20

# Capture traffic inside a container's namespace
CONTAINER_PID=$(docker inspect <container> --format '{{.State.Pid}}')
sudo nsenter --net --target $CONTAINER_PID tcpdump -i eth0 -n -c 20

# ── Kubernetes network policies (default-deny) ──────────────────────────────
cat <<'EOF' > /tmp/default-deny-policy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
EOF

cat <<'EOF' > /tmp/allow-api-to-db.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-api-to-db
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: io-thecodeforge-db
  policyTypes:
  - Ingress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: io-thecodeforge-api
    ports:
    - protocol: TCP
      port: 5432
EOF

kubectl apply -f /tmp/default-deny-policy.yaml
kubectl apply -f /tmp/allow-api-to-db.yaml

# ── Load balancer health verification ────────────────────────────────────────

# Check which containers are receiving traffic
curl -s http://localhost:80/health | jq .hostname
# Repeat 10 times — should show different hostnames (round-robin)
for i in $(seq 1 10); do
  curl -s http://localhost:80/health | jq -r .hostname
done

#!/bin/bash
# Production logging, monitoring, and observability setup

# ── Docker daemon log rotation ───────────────────────────────────────────────
cat <<'EOF' | sudo tee /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3",
    "compress": "true"
  }
}
EOF
sudo systemctl restart docker

# Verify log rotation is configured
docker info --format '{{.LoggingDriver}}'
# json-file

# ── Check container log size ────────────────────────────────────────────────

# Find large container logs
find /var/lib/docker/containers -name '*-json.log' -exec ls -lhS {} + | head -10
# If any log is > 100MB, rotation is not working

# Check total log disk usage
du -sh /var/lib/docker/containers/*

# ── Fluentd DaemonSet logging agent (Kubernetes) ────────────────────────────
cat <<'EOF' > /tmp/fluentd-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: fluentd
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1.16-debian-elasticsearch8-1
        resources:
          limits:
            memory: 512Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: dockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: dockercontainers
        hostPath:
          path: /var/lib/docker/containers
EOF
kubectl apply -f /tmp/fluentd-daemonset.yaml

# ── Prometheus alerting rules ────────────────────────────────────────────────
cat <<'EOF' > /tmp/container-alerts.yaml
groups:
- name: container-alerts
  rules:
  - alert: ContainerMemoryHigh
    expr: container_memory_usage_bytes / container_spec_memory_limit_bytes > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 'Container {{ $labels.name }} memory usage above 80%'

  - alert: ContainerCPUThrottled
    expr: rate(container_cpu_cfs_throttled_periods_total[5m]) / rate(container_cpu_cfs_periods_total[5m]) > 0.1
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 'Container {{ $labels.name }} CPU throttled > 10%'

  - alert: ContainerRestarting
    expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
    for: 0m
    labels:
      severity: critical
    annotations:
      summary: 'Container {{ $labels.container }} restarted {{ $value }} times in 1 hour'
EOF

# ── Structured logging example (Java) ────────────────────────────────────────
cat <<'EOF'
// io.thecodeforge.logging.StructuredLogger.java
package io.thecodeforge.logging;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.time.Instant;
import java.util.Map;

public class StructuredLogger {
    private static final ObjectMapper mapper = new ObjectMapper();
    private final String serviceName;

    public StructuredLogger(String serviceName) {
        this.serviceName = serviceName;
    }

    public void info(String message, String traceId, Map<String, Object> fields) {
        try {
            Map<String, Object> logEntry = Map.of(
                "timestamp", Instant.now().toString(),
                "level", "INFO",
                "message", message,
                "service", serviceName,
                "trace_id", traceId != null ? traceId : "",
                "fields", fields != null ? fields : Map.of()
            );
            System.out.println(mapper.writeValueAsString(logEntry));
        } catch (Exception e) {
            System.err.println("LOG_ERROR: " + e.getMessage());
        }
    }

    public void error(String message, String traceId, Throwable throwable) {
        try {
            Map<String, Object> logEntry = Map.of(
                "timestamp", Instant.now().toString(),
                "level", "ERROR",
                "message", message,
                "service", serviceName,
                "trace_id", traceId != null ? traceId : "",
                "error_class", throwable.getClass().getName(),
                "error_message", throwable.getMessage(),
                "stack_trace", throwable.getStackTrace()[0].toString()
            );
            System.err.println(mapper.writeValueAsString(logEntry));
        } catch (Exception e) {
            System.err.println("LOG_ERROR: " + e.getMessage());
        }
    }
}
EOF

#!/bin/bash
# Production CI/CD pipeline for Docker images

# ── Multi-stage Dockerfile ───────────────────────────────────────────────────
cat <<'EOF' > /tmp/Dockerfile
# Stage 1: Build
FROM node:20.11-alpine AS builder
WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production
COPY . .
RUN npm run build

# Stage 2: Runtime (minimal image)
FROM node:20.11-alpine AS runtime
RUN addgroup -g 1001 -S appgroup && \
    adduser -S appuser -u 1001 -G appgroup
WORKDIR /app
COPY --from=builder --chown=appuser:appgroup /app/dist ./dist
COPY --from=builder --chown=appuser:appgroup /app/node_modules ./node_modules
COPY --from=builder --chown=appuser:appgroup /app/package.json ./
USER appuser
EXPOSE 3000
HEALTHCHECK --interval=10s --timeout=5s --retries=3 \
  CMD wget -qO- http://localhost:3000/health || exit 1
CMD ["node", "dist/server.js"]
EOF

# ── Build with BuildKit ──────────────────────────────────────────────────────
GIT_SHA=$(git rev-parse --short HEAD)
IMAGE_TAG="registry.example.com/io-thecodeforge/api:${GIT_SHA}"

DOCKER_BUILDKIT=1 docker build \
  --tag ${IMAGE_TAG} \
  --label "io.thecodeforge.build.sha=${GIT_SHA}" \
  --label "io.thecodeforge.build.date=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
  --file /tmp/Dockerfile \
  .

docker push ${IMAGE_TAG}

# ── Image scanning with Trivy ───────────────────────────────────────────────
trivy image --severity HIGH,CRITICAL --exit-code 1 ${IMAGE_TAG}
# Exit code 1 = vulnerabilities found, block deployment
# Exit code 0 = no critical/high vulnerabilities

# ── Deployment: rolling update (Kubernetes) ──────────────────────────────────
kubectl -n production set image deployment/io-thecodeforge-api \
  api=${IMAGE_TAG}

# Monitor rollout
kubectl -n production rollout status deployment/io-thecodeforge-api --timeout=300s

# ── Rollback (one command) ──────────────────────────────────────────────────
kubectl -n production rollout undo deployment/io-thecodeforge-api

# ── Canary deployment (Kubernetes with Istio) ───────────────────────────────
cat <<'EOF' > /tmp/canary-virtualservice.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: io-thecodeforge-api
  namespace: production
spec:
  hosts:
  - api.example.com
  http:
  - route:
    - destination:
        host: io-thecodeforge-api
        subset: stable
      weight: 95
    - destination:
        host: io-thecodeforge-api
        subset: canary
      weight: 5
EOF
kubectl apply -f /tmp/canary-virtualservice.yaml

# Monitor canary error rate
# If error rate > 1%, rollback:
kubectl delete -f /tmp/canary-virtualservice.yaml

# ── Verify image immutability ────────────────────────────────────────────────
# Check that no two images share the same tag
docker images --format '{{.Repository}}:{{.Tag}} {{.ID}}' | sort | uniq -w 50 -d
# If output is non-empty, the same tag points to different image IDs (bad)

#!/bin/bash
# Production security hardening for Docker

# ── Non-root container ───────────────────────────────────────────────────────

# Dockerfile best practice
# RUN addgroup -g 1001 -S appgroup && adduser -S appuser -u 1001 -G appgroup
# USER appuser

# Runtime: run as non-root with dropped capabilities
docker run \
  --user 1001:1001 \
  --cap-drop=ALL \
  --cap-add=NET_BIND_SERVICE \
  --security-opt=no-new-privileges \
  --read-only \
  --tmpfs /tmp:size=64m \
  --name hardened-api \
  io-thecodeforge/api:1.0.0

# Verify non-root
docker exec hardened-api id
# uid=1001(appuser) gid=1001(appgroup)

# Verify capabilities
docker inspect hardened-api --format '{{.HostConfig.CapAdd}} {{.HostConfig.CapDrop}}'
# [NET_BIND_SERVICE] [ALL]

# ── Secrets management ──────────────────────────────────────────────────────

# Docker Swarm secrets
echo 'my-database-password' | docker secret create db-password -
docker service create --secret db-password --name io-thecodeforge-api \
  io-thecodeforge/api:1.0.0
# Secret is available at /run/secrets/db-password inside the container

# BuildKit: mount secrets during build (not in final image)
DOCKER_BUILDKIT=1 docker build --secret id=npmrc,src=$HOME/.npmrc -t api:1.0 .
# In Dockerfile: RUN --mount=type=secret,id=npmrc cp /run/secrets/npmrc $HOME/.npmrc && npm ci

# ── Image scanning and provenance ───────────────────────────────────────────

# Scan for vulnerabilities
trivy image --severity HIGH,CRITICAL io-thecodeforge/api:1.0.0

# Generate SBOM (Software Bill of Materials)
syft io-thecodeforge/api:1.0.0 -o spdx-json > sbom.json

# Verify image signature (Docker Content Trust)
export DOCKER_CONTENT_TRUST=1
docker pull io-thecodeforge/api:1.0.0
# Fails if the image is not signed

# ── Seccomp profile ─────────────────────────────────────────────────────────

# Use the default seccomp profile
docker run --security-opt seccomp=/etc/docker/seccomp/default.json \
  io-thecodeforge/api:1.0.0

# Create a custom seccomp profile (allow only required syscalls)
cat <<'EOF' > /tmp/seccomp-api.json
{
  "defaultAction": "SCMP_ACT_ERRNO",
  "syscalls": [
    {
      "names": ["read", "write", "open", "close", "stat", "fstat",
                "mmap", "mprotect", "munmap", "brk", "ioctl",
                "access", "socket", "connect", "sendto", "recvfrom",
                "clone", "execve", "exit", "exit_group", "futex",
                "epoll_create1", "epoll_ctl", "epoll_wait",
                "accept4", "listen", "bind", "setsockopt"],
      "action": "SCMP_ACT_ALLOW"
    }
  ]
}
EOF
docker run --security-opt seccomp=/tmp/seccomp-api.json \
  io-thecodeforge/api:1.0.0

# ── Daemon security ─────────────────────────────────────────────────────────

# Check if daemon socket is exposed
curl --unix-socket /var/run/docker.sock http://localhost/version
# If this works, the daemon is accessible — anyone with socket access has root

# Check if daemon is exposed over TCP
netstat -tlnp | grep 2375
# Port 2375 = unencrypted Docker API (NEVER expose this)
netstat -tlnp | grep 2376
# Port 2376 = TLS-encrypted Docker API (OK if TLS client auth is configured)

# Verify daemon configuration
cat /etc/docker/daemon.json
# Should NOT contain: "hosts": ["tcp://0.0.0.0:2375"]

#!/bin/bash
# Production scaling strategies and configuration

# ── Horizontal scaling (Docker Swarm) ────────────────────────────────────────

# Scale to 10 replicas
docker service scale io-thecodeforge-api=10

# Verify replicas are distributed across nodes
docker service ps io-thecodeforge-api --format '{{.Node}} {{.CurrentState}}'
# manager1  Running
# worker1   Running
# worker2   Running
# (distributed across 3 nodes)

# ── Horizontal scaling (Kubernetes) ──────────────────────────────────────────

# Manual scaling
kubectl -n production scale deployment/io-thecodeforge-api --replicas=10

# Auto-scaling based on CPU utilization
cat <<'EOF' > /tmp/hpa.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: io-thecodeforge-api-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: io-thecodeforge-api
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Pods
        value: 4
        periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Pods
        value: 2
        periodSeconds: 60
EOF
kubectl apply -f /tmp/hpa.yaml

# ── Graceful shutdown (preStop hook) ─────────────────────────────────────────
cat <<'EOF' > /tmp/graceful-shutdown-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: io-thecodeforge-api
  namespace: production
spec:
  template:
    spec:
      terminationGracePeriodSeconds: 30
      containers:
      - name: api
        image: registry.example.com/io-thecodeforge/api:1.0.0
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 5 && kill -SIGTERM 1"]
        # The preStop hook:
        # 1. Sleep 5s (allow load balancer to drain the pod)
        # 2. Send SIGTERM to PID 1 (the application)
        # 3. Application drains in-flight requests and exits
        # 4. Kubernetes waits up to terminationGracePeriodSeconds (30s)
EOF
kubectl apply -f /tmp/graceful-shutdown-deployment.yaml

# ── Pre-warming: pull images before scaling events ───────────────────────────

# Pre-pull images on all nodes (Kubernetes DaemonSet)
cat <<'EOF' > /tmp/prepull-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: image-prepull
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: image-prepull
  template:
    metadata:
      labels:
        app: image-prepull
    spec:
      initContainers:
      - name: prepull
        image: registry.example.com/io-thecodeforge/api:1.0.0
        command: ["true"]
      containers:
      - name: pause
        image: registry.k8s.io/pause:3.9
EOF
kubectl apply -f /tmp/prepull-daemonset.yaml
# This DaemonSet runs on every node and pulls the image into the node's cache

# ── Monitor scaling effectiveness ────────────────────────────────────────────

# Check current replica count and resource usage
kubectl -n production get deployment io-thecodeforge-api -o wide
# NAME                  READY   UP-TO-DATE   AVAILABLE   AGE
# io-thecodeforge-api   10/10   10           10          5d

# Check HPA status
kubectl -n production get hpa io-thecodeforge-api-hpa
# NAME                       REFERENCE                  TARGETS       MINPODS   MAXPODS   REPLICAS
# io-thecodeforge-api-hpa   Deployment/io-thecodeforge-api   45%/70%       3         20        5
# (45% CPU — below 70% threshold — HPA will scale down after stabilization window)

#!/bin/bash
# High availability and disaster recovery configuration

# ── Multi-host redundancy (Docker Swarm) ─────────────────────────────────────

# Create a service with replicas spread across nodes
docker service create \
  --name io-thecodeforge-api \
  --replicas 6 \
  --constraint 'node.role==worker' \
  --placement-pref 'spread=node.id' \
  --limit-cpu 1.0 \
  --limit-memory 512m \
  --update-parallelism 2 \
  --update-delay 10s \
  --update-failure-action rollback \
  --restart-condition on-failure \
  --restart-delay 5s \
  --restart-max-attempts 3 \
  registry.example.com/io-thecodeforge/api:1.0.0

# Verify distribution across nodes
docker service ps io-thecodeforge-api --format '{{.Node}} {{.CurrentState}}'
# worker1  Running
# worker2  Running
# worker3  Running
# worker1  Running
# worker2  Running
# worker3  Running

# ── Multi-AZ pod anti-affinity (Kubernetes) ──────────────────────────────────
cat <<'EOF' > /tmp/multi-az-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: io-thecodeforge-api
  namespace: production
spec:
  replicas: 6
  template:
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - io-thecodeforge-api
            topologyKey: topology.kubernetes.io/zone
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - io-thecodeforge-api
              topologyKey: kubernetes.io/hostname
EOF
kubectl apply -f /tmp/multi-az-deployment.yaml

# ── Volume backup (named volume to S3) ──────────────────────────────────────

# Create a backup container that mounts the volume and uploads to S3
docker run --rm \
  -v postgres-data:/data:ro \
  -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
  -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
  amazon/aws-cli s3 cp /data s3://my-backups/postgres-data/$(date +%Y-%m-%d)/ --recursive

# ── Kubernetes CronJob for automated backups ─────────────────────────────────
cat <<'EOF' > /tmp/backup-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: postgres-backup
  namespace: production
spec:
  schedule: "0 */6 * * *"  # Every 6 hours
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: postgres:16
            command:
            - /bin/sh
            - -c
            - |
              pg_dump -h db-host -U postgres -d mydb | \
              gzip | \
              aws s3 cp - s3://my-backups/db/mydb-$(date +%Y%m%d-%H%M%S).sql.gz
            env:
            - name: PGPASSWORD
              valueFrom:
                secretKeyRef:
                  name: db-credentials
                  key: password
          restartPolicy: OnFailure
EOF
kubectl apply -f /tmp/backup-cronjob.yaml

# ── Failover testing ────────────────────────────────────────────────────────

# Kill a random container and verify self-healing
docker service scale io-thecodeforge-api=3
sleep 5

# Kill a container
docker kill $(docker ps -q | head -1)

# Watch the orchestrator reschedule
watch docker service ps io-thecodeforge-api
# A new container should start within seconds

# Kubernetes: simulate node failure
cordon k8s-worker-2  # Mark node as unschedulable
drain k8s-worker-2   # Evict all pods from the node
# Pods are rescheduled to other nodes

# Verify all pods are running on remaining nodes
kubectl get pods -o wide | grep io-thecodeforge-api