Chaos Engineering — Why Probe Abort Missed Our Retry Storm

# Litmus Chaos ChaosEngine manifest with embedded steady-state validation
# This defines both the fault injection AND the hypothesis monitoring together
# Run with: kubectl apply -f steady_state_hypothesis.yaml

apiVersion: litmuschaos.io/v1alpha1
kind: ChaosEngine
metadata:
  name: checkout-service-resilience-test
  namespace: production-mirror          # NEVER run first experiments in live prod
spec:
  # The application under test — Litmus uses this to scope blast radius
  appinfo:
    appns: production-mirror
    applabel: "app=checkout-service"
    appkind: deployment

  # When the hypothesis is violated, Litmus can auto-stop the experiment
  jobCleanUpPolicy: retain             # Keep job logs for post-mortem analysis

  # Steady-state hypothesis: these probes ARE your monitoring assertions
  # Litmus evaluates them before injection (baseline), during, and after
  experiments:
    - name: pod-cpu-hog                # Fault: saturate CPU on checkout pods
      spec:
        probe:
          # Probe 1: HTTP probe — does the service still respond?
          - name: checkout-endpoint-alive
            type: httpProbe
            httpProbe/inputs:
              url: "http://checkout-service.production-mirror.svc.cluster.local/health"
              insecureSkipVerify: false
              method:
                get:
                  criteria: ==         # Exact HTTP status code match
                  responseCode: "200"
            mode: Continuous           # Keep checking THROUGHOUT the experiment
            runProperties:
              probeTimeout: 5          # Seconds before probe is marked failed
              interval: 10             # Check every 10 seconds
              retry: 2                 # Allow 2 transient failures before flagging
              probePollingInterval: 2

          # Probe 2: Prometheus probe — p99 latency is the REAL hypothesis
          # If CPU is pegged but latency stays under 300ms, the system is resilient
          - name: checkout-p99-latency-under-300ms
            type: promProbe
            promProbe/inputs:
              # PromQL query evaluating our SSH threshold
              # histogram_quantile computes p99 from Prometheus histogram buckets
              endpoint: "http://prometheus.monitoring.svc.cluster.local:9090"
              query: |
                histogram_quantile(
                  0.99,
                  rate(
                    http_request_duration_seconds_bucket{
                      service="checkout-service",
                      route="/api/checkout"
                    }[2m]  
                  )
                ) * 1000
              # The experiment FAILS if p99 latency exceeds 300ms at any Continuous check
              comparator:
                criteria: "<="
                type: float
                value: "300"           # Milliseconds — our SSH threshold
            mode: Continuous
            runProperties:
              probeTimeout: 10
              interval: 15
              retry: 1                 # Only 1 retry — latency spikes matter

        components:
          env:
            # Fault parameters — scope and duration of CPU stress
            - name: CPU_CORES
              value: "2"               # Hog 2 cores per pod
            - name: CPU_LOAD
              value: "90"              # 90% utilization on those cores
            - name: TOTAL_CHAOS_DURATION
              value: "120"             # Run fault for 120 seconds
            - name: PODS_AFFECTED_PERC
              value: "50"              # Blast radius: only 50% of pods affected
            - name: RAMP_TIME
              value: "10"              # 10s stabilization before fault starts

# Prometheus alerting rules that fire DURING a chaos experiment
# These rules correlate with the Litmus chaos event annotations
# Apply with: kubectl apply -f chaos_observability_stack.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: chaos-blast-radius-alerts
  namespace: monitoring
  labels:
    # This label tells the Prometheus Operator to load these rules
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
    - name: chaos-experiment-blast-radius
      # Evaluate every 10s during active experiments for fast feedback
      interval: 10s
      rules:

        # Rule 1: Detect upstream retry storm caused by fault injection
        # A retry storm means clients are hammering a degraded service
        - alert: ChaosInducedRetryStorm
          expr: |
            sum by (source_service, target_service) (
              rate(
                grpc_client_handled_total{
                  grpc_code=~"Unavailable|DeadlineExceeded|ResourceExhausted"
                }[1m]
              )
            ) > 50  # More than 50 retryable errors/sec between any two services
          for: 20s   # Must persist 20s — filters transient blips
          labels:
            severity: critical
            experiment_context: "blast-radius-propagation"
          annotations:
            summary: "Retry storm detected: {{ $labels.source_service }} → {{ $labels.target_service }}"
            description: |
              {{ $value | printf "%.1f" }} retryable errors/sec detected between services.
              This indicates the chaos fault has propagated beyond the intended blast radius.
              Check if circuit breaker on {{ $labels.source_service }} is open.
            runbook_url: "https://runbooks.internal/chaos/retry-storm"

        # Rule 2: Connection pool exhaustion — a common blast radius spillover
        # When a service slows down, connection pools fill up and starve other callers
        - alert: DatabaseConnectionPoolExhausted
          expr: |
            (
              db_pool_connections_in_use{pool="checkout-db-pool"}
              /
              db_pool_connections_max{pool="checkout-db-pool"}
            ) > 0.90  # Pool is more than 90% utilized
          for: 15s
          labels:
            severity: critical
            experiment_context: "blast-radius-database"
          annotations:
            summary: "DB connection pool near exhaustion during chaos experiment"
            description: |
              Pool utilization: {{ $value | humanizePercentage }}.
              New requests will queue or fail. Blast radius has reached the database tier.
              Experiment should be aborted if this persists beyond ramp-down window.

        # Rule 3: Business-layer impact — the metric that matters to leadership
        # Drop in order completion rate signals real user impact
        - alert: ChaosBusinessImpactDetected
          expr: |
            (
              rate(orders_completed_total[2m])
              /
              rate(orders_initiated_total[2m])
            ) < 0.95  # Completion rate dropped below 95%
          for: 30s
          labels:
            severity: page            # This one actually wakes someone up
            experiment_context: "blast-radius-business"
          annotations:
            summary: "Order completion rate below 95% — chaos experiment exceeding safe blast radius"
            description: |
              Current completion rate: {{ $value | humanizePercentage }}.
              Expected baseline: >99%. This represents real revenue impact.
              Abort the chaos experiment immediately via Litmus dashboard.

---
# Grafana annotation source — pushes chaos events as vertical lines on dashboards
# This makes every chart self-documenting during an experiment review
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-chaos-annotation-datasource
  namespace: monitoring
data:
  # Grafana reads this query to draw annotations on all dashboards
  # It queries the chaos event log stored as Prometheus labels
  annotation_query: |
    changes(
      litmuschaos_experiment_verdict{
        chaosresult_verdict=~"Pass|Fail|Stopped"
      }[1m]
    ) > 0
  # Annotation label shown in Grafana UI
  annotation_label: "Chaos Fault Window"

#!/usr/bin/env python3
"""
Chaos Experiment Watchdog Service

Listens for Alertmanager webhook callbacks and automatically aborts
a running Litmus Chaos experiment if a severity:page alert fires
during the experiment window.

Designed to run on a SEPARATE NODE POOL from the experiment blast radius.
Requirements: pip install fastapi uvicorn httpx pydantic
Run with: uvicorn chaos_watchdog_service:watchdog_app --host 0.0.0.0 --port 8090
"""

import asyncio
import logging
import os
from datetime import datetime, timezone
from typing import Optional

import httpx
from fastapi import FastAPI, HTTPException, Request
from pydantic import BaseModel, Field

# ── Configuration ─────────────────────────────────────────────────────────────
LITMUS_API_URL = os.environ["LITMUS_CHAOS_API_URL"]        # e.g. http://litmus.monitoring:9002
LITMUS_API_TOKEN = os.environ["LITMUS_API_TOKEN"]          # Service account token
GRAFANA_API_URL = os.environ["GRAFANA_API_URL"]            # For snapshot capture on abort
GRAFANA_API_TOKEN = os.environ["GRAFANA_API_TOKEN"]

# Dashboard UID that shows all chaos-related panels — captured at abort time
CHAOS_DASHBOARD_UID = os.environ.get("CHAOS_DASHBOARD_UID", "chaos-blast-radius-overview")

# Only abort experiments if alerts carry this label — prevents false triggers
REQUIRED_EXPERIMENT_LABEL = "experiment_context"

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s — %(message)s"
)
logger = logging.getLogger("chaos-watchdog")

# ── Pydantic models matching Alertmanager webhook payload ──────────────────────
class AlertmanagerLabel(BaseModel):
    alertname: str
    severity: Optional[str] = None
    experiment_context: Optional[str] = None   # Set in our PrometheusRule labels
    namespace: Optional[str] = None

class AlertmanagerAnnotation(BaseModel):
    summary: Optional[str] = None
    description: Optional[str] = None

class AlertmanagerAlert(BaseModel):
    status: str                                # 'firing' or 'resolved'
    labels: AlertmanagerLabel
    annotations: AlertmanagerAnnotation
    startsAt: str
    generatorURL: str

class AlertmanagerWebhookPayload(BaseModel):
    version: str = Field(default="4")
    status: str
    alerts: list[AlertmanagerAlert]

# ── FastAPI app ────────────────────────────────────────────────────────────────
watchdog_app = FastAPI(
    title="Chaos Experiment Watchdog",
    description="Auto-aborts chaos experiments when SSH is violated",
    version="1.0.0"
)

@watchdog_app.post("/alertmanager/webhook")
async def handle_alertmanager_webhook(payload: AlertmanagerWebhookPayload, request: Request):
    """
    Alertmanager calls this endpoint when an alert fires or resolves.
    We only act on 'firing' alerts that carry our experiment_context label.
    """
    logger.info(f"Received webhook — status={payload.status}, alert_count={len(payload.alerts)}")

    for alert in payload.alerts:
        # Only act on actively firing alerts
        if alert.status != "firing":
            logger.debug(f"Skipping resolved alert: {alert.labels.alertname}")
            continue

        # Only abort if alert is explicitly tagged as chaos-experiment-related
        experiment_context = alert.labels.experiment_context
        if not experiment_context:
            logger.debug(f"Alert {alert.labels.alertname} has no experiment_context — skipping")
            continue

        # severity:page means business impact — always abort
        if alert.labels.severity == "page":
            logger.warning(
                f"PAGE-SEVERITY alert during chaos experiment! "
                f"Alert={alert.labels.alertname} "
                f"Context={experiment_context}"
            )

            # Step 1: Capture Grafana dashboard snapshot BEFORE aborting
            # This preserves the system state at the moment of breach
            snapshot_url = await capture_grafana_snapshot(
                dashboard_uid=CHAOS_DASHBOARD_UID,
                annotation=f"AUTO-ABORT: {alert.labels.alertname} at {datetime.now(timezone.utc).isoformat()}"
            )

            # Step 2: Abort the running chaos experiment via Litmus API
            abort_result = await abort_litmus_experiment(
                experiment_context=experiment_context,
                abort_reason=alert.annotations.summary or alert.labels.alertname
            )

            logger.info(
                f"Experiment aborted successfully. "
                f"Snapshot: {snapshot_url} "
                f"Litmus response: {abort_result}"
            )

    return {"status": "processed", "timestamp": datetime.now(timezone.utc).isoformat()}


async def capture_grafana_snapshot(dashboard_uid: str, annotation: str) -> str:
    """
    Creates a Grafana snapshot of the chaos dashboard at the current moment.
    Returns the public snapshot URL for inclusion in post-mortem reports.
    """
    async with httpx.AsyncClient(timeout=10.0) as grafana_client:
        # First, get the dashboard JSON model
        dashboard_response = await grafana_client.get(
            f"{GRAFANA_API_URL}/api/dashboards/uid/{dashboard_uid}",
            headers={"Authorization": f"Bearer {GRAFANA_API_TOKEN}"}
        )
        dashboard_response.raise_for_status()
        dashboard_model = dashboard_response.json()["dashboard"]

        # Create a snapshot — Grafana stores it and returns a share URL
        snapshot_response = await grafana_client.post(
            f"{GRAFANA_API_URL}/api/snapshots",
            headers={"Authorization": f"Bearer {GRAFANA_API_TOKEN}"},
            json={
                "dashboard": dashboard_model,
                "name": f"Chaos Abort Snapshot — {annotation}",
                "expires": 604800,     # Snapshot expires in 7 days
            }
        )
        snapshot_response.raise_for_status()
        snapshot_url = snapshot_response.json()["url"]
        logger.info(f"Grafana snapshot captured: {snapshot_url}")
        return snapshot_url


async def abort_litmus_experiment(experiment_context: str, abort_reason: str) -> dict:
    """
    Calls the Litmus Chaos API to stop the running experiment.
    experiment_context maps to the ChaosEngine name via our labeling convention.
    """
    # Derive the ChaosEngine name from the experiment_context label
    # Convention: experiment_context label value matches ChaosEngine metadata.name
    chaos_engine_name = experiment_context.replace("blast-radius-", "")

    async with httpx.AsyncClient(timeout=15.0) as litmus_client:
        # Litmus REST API: PATCH the engine status to 'stop'
        stop_response = await litmus_client.patch(
            f"{LITMUS_API_URL}/api/chaosengine/{chaos_engine_name}",
            headers={
                "Authorization": f"Bearer {LITMUS_API_TOKEN}",
                "Content-Type": "application/json"
            },
            json={
                "spec": {
                    "engineState": "stop"    # Litmus graceful stop — rolls back faults
                },
                "metadata": {
                    "annotations": {
                        # Record WHY this was aborted — visible in Litmus dashboard
                        "chaos.abort.reason": abort_reason,
                        "chaos.abort.timestamp": datetime.now(timezone.utc).isoformat(),
                        "chaos.abort.source": "watchdog-service"
                    }
                }
            }
        )
        stop_response.raise_for_status()
        logger.info(f"Litmus experiment '{chaos_engine_name}' stopped via API")
        return stop_response.json()


@watchdog_app.get("/health")
async def health_check():
    """Liveness probe — the watchdog must be reachable during experiments."""
    return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}