Istio Service Mesh on Kubernetes: Internals, Traffic Control & Production Gotchas

#!/usr/bin/env bash
# PURPOSE: Inspect the iptables rules that Istio's init container installs
# inside a running pod. Run this to see exactly how traffic is intercepted.
# REQUIRES: kubectl and a pod with Istio injection enabled.

POD_NAME="payment-service-7d9f8b-xkp2q"
NAMESPACE="production"

# Step 1: Open a shell inside the istio-proxy sidecar (not your app container)
# We use nsenter to peek at the network namespace's iptables rules
kubectl exec -n "${NAMESPACE}" "${POD_NAME}" \
  -c istio-proxy \
  -- sh -c 'iptables-save' 2>/dev/null

# Step 2: Verify Envoy is listening on the expected interception ports
# 15001 = outbound traffic listener
# 15006 = inbound traffic listener  
# 15090 = Prometheus metrics scrape endpoint
kubectl exec -n "${NAMESPACE}" "${POD_NAME}" \
  -c istio-proxy \
  -- ss -tlnp | grep -E '15001|15006|15090|15021'

# Step 3: Check that Istiod has pushed config to this proxy
# SYNCED means Envoy has received and acknowledged the latest xDS config
istioctl proxy-status -n "${NAMESPACE}" "${POD_NAME}"

# Step 4: Dump the full Envoy config to understand exactly what Istio programmed
# WARNING: this is verbose — pipe to jq or save to file
istioctl proxy-config listeners "${POD_NAME}" -n "${NAMESPACE}" --output json | \
  jq '.[] | select(.address.socketAddress.portValue == 15006)'

# PURPOSE: Route 95% of payment-service traffic to stable v1,
# 5% to canary v2, with circuit breaking and connection pool limits.
# Apply with: kubectl apply -f payment-traffic-policy.yaml

---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: payment-service-destination
  namespace: production
spec:
  host: payment-service  # Matches the Kubernetes Service name
  
  # --- Connection pool limits applied to ALL subsets ---
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100          # Max TCP connections per Envoy instance to this host
      http:
        http2MaxRequests: 1000       # Max concurrent HTTP/2 requests
        pendingRequests: 50          # Requests queued when all connections are in use
        requestsPerConnection: 10    # Forces connection cycling; good for gRPC load balancing
    
    # --- Passive circuit breaker (outlier detection) ---
    outlierDetection:
      consecutiveGatewayErrors: 5   # Eject a host after 5 consecutive 5xx or connect failures
      interval: 30s                 # How often Envoy evaluates ejection criteria
      baseEjectionTime: 30s         # Minimum time a host stays ejected
      maxEjectionPercent: 50        # Never eject more than 50% of hosts (prevents cascade)
      minHealthPercent: 30          # Stop ejecting if fewer than 30% of hosts are healthy
  
  # --- Define traffic subsets by pod labels ---
  subsets:
    - name: stable
      labels:
        version: v1                  # Selects pods with label version=v1
      trafficPolicy:
        loadBalancer:
          simple: LEAST_CONN        # Override global policy: route to least-busy pod
    
    - name: canary
      labels:
        version: v2
      trafficPolicy:
        loadBalancer:
          simple: ROUND_ROBIN

---
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: payment-service-routing
  namespace: production
spec:
  # This VirtualService applies to requests going TO payment-service
  hosts:
    - payment-service
  
  http:
    # --- Rule 1: Beta users always go to canary ---
    - match:
        - headers:
            x-beta-user:
              exact: "true"         # Header must match exactly
      route:
        - destination:
            host: payment-service
            subset: canary          # Must match a subset name in DestinationRule
          weight: 100
      # Inject 50ms delay for beta users to test timeout handling
      fault:
        delay:
          percentage:
            value: 10.0             # Apply delay to 10% of beta user requests
          fixedDelay: 50ms
    
    # --- Rule 2: All other traffic — 95/5 weighted canary split ---
    - route:
        - destination:
            host: payment-service
            subset: stable
          weight: 95
        - destination:
            host: payment-service
            subset: canary
          weight: 5
      
      # Retry policy: retry on retriable errors, not on all failures
      retries:
        attempts: 3
        perTryTimeout: 2s           # Each individual attempt gets 2s, not the total budget
        retryOn: "gateway-error,connect-failure,retriable-4xx"

# PURPOSE: Lock down the payment-service to STRICT mTLS
# and only allow calls from the checkout-service ServiceAccount.
# This is what zero-trust networking looks like in Kubernetes.

---
# STEP 1: Enable STRICT mTLS for payment-service namespace
# No plain-text connections accepted — Envoy will return TLS handshake errors
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: payment-namespace-strict-mtls
  namespace: production
spec:
  # No 'selector' field = applies to ALL workloads in this namespace
  mtls:
    mode: STRICT
  # Per-port override: health check endpoints often need plain HTTP
  # (e.g., for kube-apiserver liveness probes that don't speak mTLS)
  portLevelMtls:
    15021:             # Istio health check port — exempt from mTLS
      mode: PERMISSIVE

---
# STEP 2: Require that ONLY checkout-service can call payment-service
# Identity is derived from ServiceAccount via SPIFFE URI, not IP address
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: payment-service-allow-checkout-only
  namespace: production
spec:
  selector:
    matchLabels:
      app: payment-service    # Applies to pods with this label
  
  action: ALLOW               # Default is DENY when any AuthorizationPolicy exists
  
  rules:
    - from:
        - source:
            # The SPIFFE principal for the checkout-service ServiceAccount
            principals:
              - "cluster.local/ns/production/sa/checkout-service-account"
      to:
        - operation:
            methods: ["POST"]          # Only POST calls
            paths: ["/api/v1/charge", "/api/v1/refund"]  # Only these paths
      when:
        # Extra condition: require a JWT claim (for external-to-mesh flows)
        - key: request.auth.claims[role]
          values: ["payment-processor", "admin"]

---
# STEP 3: Verify that the mTLS handshake is actually happening
# by inspecting the TLS certificate the proxy presents
# Run this from a pod inside the mesh:
apiVersion: v1
kind: Pod
metadata:
  name: mtls-debug-pod
  namespace: production
  annotations:
    # Exclude this debug pod from sidecar injection
    sidecar.istio.io/inject: "false"
spec:
  containers:
    - name: curl-debug
      image: curlimages/curl:8.5.0
      command: ["sleep", "3600"]

# PURPOSE: Configure Istio telemetry to balance observability with performance.
# Reducing trace sampling from 100% to 1% in production can cut Jaeger
# ingestion load by 100x while still giving statistically meaningful data.

---
# Telemetry API (Istio 1.12+) — replaces the old MeshConfig approach
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: mesh-default-telemetry
  namespace: istio-system   # istio-system = mesh-wide scope
spec:
  # --- Distributed tracing configuration ---
  tracing:
    - providers:
        - name: jaeger-collector   # Must match a provider defined in MeshConfig
      
      # 1% sampling in production is usually sufficient for latency analysis.
      # Use 100% only during active incident investigation.
      randomSamplingPercentage: 1.0
      
      # Propagate standard B3 headers so your app can forward them
      # Your app must still FORWARD these — Istio can't do that for you
      customTags:
        environment:
          literal:
            value: "production"
        git_sha:
          environment:
            name: GIT_COMMIT_SHA     # Read from pod env var set at deploy time
            defaultValue: "unknown"
  
  # --- Access log configuration ---
  accessLogging:
    - providers:
        - name: envoy              # Use Envoy's native access log format
      # Disable access logging for health check paths — these are noise
      # at scale (kubelet hits /health every 10s per pod = thousands of logs/min)
      filter:
        expression: "response.code != 200 || request.url_path != '/health'"

---
# Per-pod resource limits for the sidecar proxy
# Set these or Envoy will use whatever CPU is available during spikes
apiVersion: v1
kind: ConfigMap
metadata:
  name: istio-sidecar-injector
  namespace: istio-system
data:
  config: |
    policy: enabled
    defaultTemplates: [sidecar]
    template: |
      spec:
        containers:
        - name: istio-proxy
          resources:
            requests:
              cpu: 100m        # 0.1 vCPU — baseline for light traffic
              memory: 128Mi    # Enough for Envoy's config cache + runtime
            limits:
              cpu: 500m        # Cap at 0.5 vCPU to prevent noisy-neighbour issues
              memory: 256Mi    # OOM kill the proxy, not your app