Sidecar Pattern in Microservices: Internals, Trade-offs and Production Gotchas

#!/usr/bin/env bash
# ─────────────────────────────────────────────────────────────────────────────
# inspect_sidecar_iptables.sh
# Run this INSIDE the init container or as root inside a pod to see exactly
# how Istio redirects traffic through Envoy.
# This is the actual rule set Istio's istio-init container installs.
# ─────────────────────────────────────────────────────────────────────────────

# Show the ISTIO_OUTPUT chain — handles outbound traffic FROM the application
echo "=== OUTBOUND RULES (ISTIO_OUTPUT chain) ==="
iptables -t nat -L ISTIO_OUTPUT -n --line-numbers -v

# Expected output will show rules like:
#   REDIRECT  tcp  --  anywhere  anywhere  redir ports 15001
# meaning all outbound TCP from non-Envoy processes goes to port 15001 (Envoy)

echo ""
echo "=== INBOUND RULES (ISTIO_INBOUND chain) ==="
iptables -t nat -L ISTIO_INBOUND -n --line-numbers -v

# Expected output will show:
#   REDIRECT  tcp  --  anywhere  anywhere  tcp dpt:8080  redir ports 15006
# meaning inbound traffic to your app's port gets redirected to port 15006

echo ""
echo "=== Envoy's listening ports ==="
# Envoy listens on these ports inside the pod's shared network namespace
ss -tlnp | grep -E '15001|15006|15090|9901'
# 15001 = outbound listener
# 15006 = inbound listener (virtual inbound)
# 15090 = Prometheus metrics endpoint
# 9901  = Envoy admin API

// sidecar_proxy.go
// A minimal sidecar proxy in Go that:
//   1. Listens on localhost:7000 (the port your app talks to)
//   2. Injects a X-Request-ID header if one isn't present
//   3. Logs method, path, upstream status, and latency
//   4. Forwards the request to the real upstream (configured via env var)
//
// Run: UPSTREAM_URL=http://httpbin.org go run sidecar_proxy.go
// Then: curl http://localhost:7000/get

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/google/uuid" // go get github.com/google/uuid
)

// upstreamBaseURL is where we actually forward requests to.
// In a real sidecar this comes from service discovery / control plane config.
var upstreamBaseURL string

func main() {
	upstreamBaseURL = os.Getenv("UPSTREAM_URL")
	if upstreamBaseURL == "" {
		log.Fatal("UPSTREAM_URL environment variable is required")
	}

	// The sidecar listens on 7000. The main application is configured to
	// send ALL outbound HTTP through http://localhost:7000.
	// In a real deployment, iptables rules do this transparently.
	mux := http.NewServeMux()
	mux.HandleFunc("/", handleProxyRequest)

	listenAddr := "127.0.0.1:7000"
	log.Printf("[sidecar] proxy listening on %s → forwarding to %s", listenAddr, upstreamBaseURL)

	if err := http.ListenAndServe(listenAddr, mux); err != nil {
		log.Fatalf("[sidecar] failed to start: %v", err)
	}
}

func handleProxyRequest(responseWriter http.ResponseWriter, incomingRequest *http.Request) {
	startTime := time.Now()

	// ── Step 1: Ensure a trace ID exists ─────────────────────────────────────
	// If the application (or an upstream caller) didn't set X-Request-ID,
	// we generate one here. This is a classic sidecar responsibility:
	// the app never needs to know about tracing infrastructure.
	requestID := incomingRequest.Header.Get("X-Request-ID")
	if requestID == "" {
		requestID = uuid.NewString() // e.g. "3f2504e0-4f89-11d3-9a0c-0305e82c3301"
		incomingRequest.Header.Set("X-Request-ID", requestID)
	}

	// ── Step 2: Build the upstream request ───────────────────────────────────
	// We reconstruct the full upstream URL by prepending the configured base.
	// incomingRequest.RequestURI includes path + query string.
	upstreamURL := upstreamBaseURL + incomingRequest.RequestURI

	upstreamRequest, err := http.NewRequest(
		incomingRequest.Method,
		upstreamURL,
		incomingRequest.Body, // stream the body directly — don't buffer it in memory
	)
	if err != nil {
		log.Printf("[sidecar] ERROR building upstream request: %v", err)
		http.Error(responseWriter, "sidecar: failed to build upstream request", http.StatusBadGateway)
		return
	}

	// Copy all original headers to the upstream request (including our new X-Request-ID)
	for headerName, headerValues := range incomingRequest.Header {
		for _, value := range headerValues {
			upstreamRequest.Header.Add(headerName, value)
		}
	}

	// Identify ourselves in the Via header — helpful for debugging proxy chains
	upstreamRequest.Header.Set("Via", "1.1 sidecar-proxy")

	// ── Step 3: Execute the upstream call ────────────────────────────────────
	httpClient := &http.Client{Timeout: 10 * time.Second}
	upstreamResponse, err := httpClient.Do(upstreamRequest)
	if err != nil {
		log.Printf("[sidecar] ERROR calling upstream: %v", err)
		http.Error(responseWriter, "sidecar: upstream unreachable", http.StatusBadGateway)
		return
	}
	defer upstreamResponse.Body.Close()

	// ── Step 4: Stream the response back to the caller ───────────────────────
	// Copy upstream response headers back to our response
	for headerName, headerValues := range upstreamResponse.Header {
		for _, value := range headerValues {
			responseWriter.Header().Add(headerName, value)
		}
	}
	// Echo the request ID back so the caller can correlate logs
	responseWriter.Header().Set("X-Request-ID", requestID)
	responseWriter.WriteHeader(upstreamResponse.StatusCode)

	bytesWritten, _ := io.Copy(responseWriter, upstreamResponse.Body)

	// ── Step 5: Emit a structured access log ─────────────────────────────────
	// In production you'd encode this as JSON and ship to your log aggregator.
	// The app itself emits zero log lines for this request — the sidecar owns telemetry.
	latencyMs := time.Since(startTime).Milliseconds()
	fmt.Printf(
		`[sidecar] request_id=%s method=%s path=%s status=%d bytes=%d latency_ms=%d\n`,
		requestID,
		incomingRequest.Method,
		incomingRequest.URL.Path,
		upstreamResponse.StatusCode,
		bytesWritten,
		latencyMs,
	)
}

# sidecar_resource_limits.yaml
# Production-grade Kubernetes pod spec showing how to:
#   1. Co-locate a sidecar with your main application container
#   2. Set SEPARATE resource limits for app vs sidecar (critical — most teams forget this)
#   3. Control startup order so the sidecar is ready before the app starts taking traffic
#   4. Use Istio's Sidecar CR to scope which services the sidecar needs to know about

apiVersion: v1
kind: Pod
metadata:
  name: payments-service-pod
  annotations:
    # Tell Istio to inject Envoy automatically when this pod is created
    sidecar.istio.io/inject: "true"
    # Override default Envoy resource limits — don't let the sidecar starve your app
    sidecar.istio.io/proxyCPU: "200m"         # 0.2 vCPU — tune per observed usage
    sidecar.istio.io/proxyMemory: "128Mi"     # baseline Envoy footprint
    sidecar.istio.io/proxyCPULimit: "1000m"   # allow burst to 1 vCPU under load
    sidecar.istio.io/proxyMemoryLimit: "256Mi"
spec:
  # initContainers run before any regular containers.
  # Istio injects istio-init here automatically to install iptables rules.
  # We show it explicitly so you understand what's happening.
  initContainers:
    - name: istio-init
      image: docker.io/istio/proxyv2:1.20.0
      args: ["istio-iptables", "-p", "15001", "-z", "15006", "-u", "1337"]
      # 1337 is the UID Envoy runs as — traffic from UID 1337 is exempted from
      # iptables redirect to prevent infinite loops
      securityContext:
        capabilities:
          add: ["NET_ADMIN", "NET_RAW"] # required to modify iptables rules
        runAsNonRoot: false
        runAsUser: 0 # init container runs as root only to set iptables

  containers:
    # ── Main application container ────────────────────────────────────────────
    - name: payments-service
      image: myregistry/payments-service:2.4.1
      ports:
        - containerPort: 8080
      resources:
        requests:
          cpu: "500m"
          memory: "512Mi"
        limits:
          cpu: "2000m"
          memory: "1Gi"
      # Health check goes directly to the app — NOT through the sidecar
      # If you route health checks through Envoy and Envoy is slow to start,
      # your pod will be killed in a restart loop before the app is even ready.
      readinessProbe:
        httpGet:
          path: /healthz
          port: 8080
        initialDelaySeconds: 5
        periodSeconds: 10

    # ── Sidecar container (shown explicitly; normally injected automatically) ─
    # In production Istio injects this — we show it here for educational clarity.
    - name: istio-proxy
      image: docker.io/istio/proxyv2:1.20.0
      args:
        - proxy
        - sidecar
        - --serviceCluster
        - payments-service
        - --proxyLogLevel
        - warning  # Don't log at 'info' in prod — it's extremely verbose
      ports:
        - containerPort: 15090  # Prometheus scrape port for Envoy metrics
        - containerPort: 9901   # Envoy admin API — useful for debugging
      # Sidecar gets its OWN resource envelope, completely separate from the app.
      # This is the single most important production config most teams skip.
      resources:
        requests:
          cpu: "200m"
          memory: "128Mi"
        limits:
          cpu: "1000m"
          memory: "256Mi"
      # Lifecycle hook: drain connections gracefully before the pod terminates.
      # Without this, in-flight requests get hard-killed during rolling deploys.
      lifecycle:
        preStop:
          exec:
            command:
              - "/bin/sh"
              - "-c"
              - "sleep 5 && curl -sf -X POST http://127.0.0.1:9901/healthcheck/fail"
---
# Istio Sidecar CR: Scope what this sidecar needs to know about.
# By default, Envoy loads routing config for EVERY service in the mesh.
# This scopes it to only the services payments-service actually calls,
# reducing memory from ~150MB to ~30MB in large clusters.
apiVersion: networking.istio.io/v1beta1
kind: Sidecar
metadata:
  name: payments-service-sidecar-scope
  namespace: production
spec:
  workloadSelector:
    labels:
      app: payments-service
  egress:
    - hosts:
        - "production/user-service"    # only services we actually call
        - "production/fraud-service"
        - "istio-system/*"             # always include the control plane
  ingress:
    - port:
        number: 8080
        protocol: HTTP
        name: http-payments
      defaultEndpoint: 127.0.0.1:8080 # deliver to app on loopback

# three_container_patterns.yaml
# A single Kubernetes pod demonstrating all three container helper patterns:
#   - Sidecar:    Envoy proxy (traffic management, mTLS, observability)
#   - Adapter:    Fluent Bit (normalize app logs to structured JSON for Elasticsearch)
#   - Ambassador: Vault Agent (fetch secrets from HashiCorp Vault and expose on localhost)
#
# The main app (order-processor) does NONE of this itself. It:
#   - Writes plain-text logs to /var/log/app/orders.log
#   - Reads its DB password from /vault/secrets/db-password (written by Vault Agent)
#   - Makes HTTP calls to localhost:8200 when it needs additional secrets at runtime
#
# This is the sidecar pattern at full production maturity.

apiVersion: v1
kind: Pod
metadata:
  name: order-processor-pod
  labels:
    app: order-processor
  annotations:
    sidecar.istio.io/inject: "true"
spec:
  serviceAccountName: order-processor-sa  # needs Vault + Kubernetes auth

  volumes:
    # Shared volume between app and Fluent Bit adapter
    - name: app-log-volume
      emptyDir: {}
    # Shared volume where Vault Agent writes decrypted secrets
    - name: vault-secrets-volume
      emptyDir:
        medium: Memory  # NEVER write secrets to disk — use tmpfs (in-memory volume)

  containers:
    # ══════════════════════════════════════════════════════════════════════════
    # MAIN CONTAINER: The application itself. Blissfully ignorant of
    # infrastructure concerns. Reads secrets from files, writes plain logs.
    # ══════════════════════════════════════════════════════════════════════════
    - name: order-processor
      image: myregistry/order-processor:3.1.0
      env:
        # App reads DB password from a file. Vault Agent keeps this file fresh.
        - name: DB_PASSWORD_FILE
          value: /vault/secrets/db-password
        # App sends logs to this path. Fluent Bit tails this file.
        - name: LOG_FILE_PATH
          value: /var/log/app/orders.log
      volumeMounts:
        - name: app-log-volume
          mountPath: /var/log/app
        - name: vault-secrets-volume
          mountPath: /vault/secrets
          readOnly: true  # app can only READ secrets — cannot pollute the volume
      resources:
        requests: { cpu: "500m", memory: "256Mi" }
        limits:   { cpu: "2",    memory: "512Mi" }

    # ══════════════════════════════════════════════════════════════════════════
    # ADAPTER CONTAINER: Fluent Bit
    # Problem: app writes unstructured text logs like:
    #   "2024-01-15 14:32:01 INFO order_id=ORD-9921 status=FULFILLED"
    # Elasticsearch expects JSON with @timestamp and level fields.
    # Fluent Bit parses and re-emits as:
    #   {"@timestamp":"2024-01-15T14:32:01Z","level":"INFO","order_id":"ORD-9921",...}
    # The outside world (Elasticsearch) only sees normalized output.
    # ══════════════════════════════════════════════════════════════════════════
    - name: fluent-bit-adapter
      image: fluent/fluent-bit:3.0
      args:
        - /fluent-bit/bin/fluent-bit
        - --config=/fluent-bit/etc/fluent-bit.conf
      volumeMounts:
        - name: app-log-volume
          mountPath: /var/log/app
          readOnly: true  # adapter only READS logs — cannot write back to app's log dir
      resources:
        requests: { cpu: "50m",  memory: "32Mi" }
        limits:   { cpu: "200m", memory: "64Mi" }

    # ══════════════════════════════════════════════════════════════════════════
    # AMBASSADOR CONTAINER: HashiCorp Vault Agent
    # The app needs a DB password and a Stripe API key.
    # Without this ambassador, the app would need:
    #   - Vault SDK dependency
    #   - Token renewal logic
    #   - Secret lease management
    # With the ambassador, the app just reads a file. Vault Agent handles
    # auth, token refresh, secret rotation, and writes the fresh value.
    # The app calls localhost:8200 for dynamic secrets at runtime.
    # ══════════════════════════════════════════════════════════════════════════
    - name: vault-agent-ambassador
      image: hashicorp/vault:1.15
      args: ["agent", "-config=/vault/config/agent-config.hcl"]
      env:
        - name: VAULT_ADDR
          value: "https://vault.internal.mycompany.com:8200"
      ports:
        # Vault Agent exposes a local proxy on 8200 — app calls http://localhost:8200
        # Ambassador translates this into authenticated calls to the real Vault cluster
        - containerPort: 8200
          name: vault-proxy
      volumeMounts:
        - name: vault-secrets-volume
          mountPath: /vault/secrets  # writes decrypted secrets here
      resources:
        requests: { cpu: "50m",  memory: "64Mi" }
        limits:   { cpu: "200m", memory: "128Mi" }