Alerting and On-Call Best Practices That Actually Reduce Burnout

# prometheus_symptom_alerts.yml
# These rules live in your Prometheus alerting rules directory.
# Load them via the 'rule_files' block in prometheus.yml.

groups:
  - name: user_facing_symptoms
    # Evaluate every 60 seconds
    interval: 60s
    rules:

      # GOOD ALERT: Fires on what users experience — high error rate
      # This is a SYMPTOM. Users are getting 5xx errors. Action is clear.
      - alert: HighErrorRateShopping
        expr: |
          (
            sum(rate(http_requests_total{service="shopping-cart", status=~"5.."}[5m]))
            /
            sum(rate(http_requests_total{service="shopping-cart"}[5m]))
          ) > 0.05
        # Wait 2 minutes before firing — avoids alerting on a single bad scrape
        for: 2m
        labels:
          severity: critical
          team: checkout
        annotations:
          summary: "Shopping cart error rate above 5% for 2 minutes"
          # Link directly to the runbook — engineers shouldn't have to search
          runbook_url: "https://runbooks.internal/shopping-cart-errors"
          description: "Current error rate: {{ $value | humanizePercentage }}"

      # GOOD ALERT: Fires on latency degradation users feel
      - alert: CheckoutLatencyHigh
        expr: |
          histogram_quantile(
            0.99,
            sum(rate(http_request_duration_seconds_bucket{
              service="checkout", handler="/api/purchase"
            }[10m])) by (le)
          ) > 2.0
        for: 5m
        labels:
          severity: warning
          team: checkout
        annotations:
          summary: "p99 checkout latency exceeded 2 seconds"
          runbook_url: "https://runbooks.internal/checkout-latency"
          description: "p99 latency is {{ $value }}s. SLO threshold is 2.0s."

      # BAD ALERT (shown for contrast — commented out intentionally)
      # Do NOT do this. CPU spiking doesn't mean users are affected.
      # - alert: HighCPU
      #   expr: node_cpu_utilization > 0.80
      #   for: 1m
      #   annotations:
      #     summary: "CPU is high" # <-- What should the engineer DO with this?

# pagerduty_escalation_policy.yml
# This is a Terraform representation of a PagerDuty escalation policy.
# Apply with: terraform apply
# Requires: pagerduty Terraform provider configured with your API token.

resource "pagerduty_escalation_policy" "checkout_team" {
  name      = "Checkout Team - Production Escalation"
  num_loops = 2  # After 2 full loops with no acknowledgment, re-alert from top

  # LEVEL 1 — Primary on-call engineer gets paged first
  rule {
    escalation_delay_in_minutes = 10  # Wait 10 min before escalating to secondary

    target {
      type = "schedule_reference"
      # This schedule rotates weekly across your primary on-call engineers
      id   = pagerduty_schedule.checkout_primary_rotation.id
    }
  }

  # LEVEL 2 — Secondary on-call gets paged if primary doesn't acknowledge
  rule {
    escalation_delay_in_minutes = 10  # Another 10 min before going to manager

    target {
      type = "schedule_reference"
      # Secondary rotation — separate schedule, separate engineers
      id   = pagerduty_schedule.checkout_secondary_rotation.id
    }
  }

  # LEVEL 3 — Engineering manager is the last resort, not the first call
  rule {
    escalation_delay_in_minutes = 15

    target {
      type = "user_reference"
      # Direct page to the EM — this should be rare
      id   = data.pagerduty_user.checkout_engineering_manager.id
    }
  }
}

# Weekly rotating schedule for primary on-call
resource "pagerduty_schedule" "checkout_primary_rotation" {
  name      = "Checkout Primary On-Call"
  time_zone = "America/New_York"

  layer {
    name                         = "Weekly Rotation"
    start                        = "2024-01-08T09:00:00-05:00"
    # Rotate every 7 days — weekly on-call is the industry standard baseline
    rotation_turn_length_seconds = 604800
    rotation_virtual_start       = "2024-01-08T09:00:00-05:00"

    # Engineers in the rotation — minimum 4 for healthy coverage
    users = [
      data.pagerduty_user.alice.id,
      data.pagerduty_user.bob.id,
      data.pagerduty_user.carol.id,
      data.pagerduty_user.david.id,
    ]
  }
}

# slo_burn_rate_alert.yml
# Multi-window burn rate alerting — the approach recommended by Google SRE Workbook.
# This fires when you're consuming error budget fast enough to exhaust it prematurely.
# Assumes: 30-day SLO window, 99.9% availability target (0.1% error budget).

groups:
  - name: slo_burn_rates
    rules:

      # Fast burn — consuming budget 14x faster than sustainable
      # At this rate, 30 days of error budget is gone in ~52 hours
      # Use SHORT windows to catch it quickly. PAGE IMMEDIATELY.
      - alert: CheckoutSLOFastBurn
        expr: |
          (
            # 1-hour window burn rate
            (
              1 - (
                sum(rate(http_requests_total{service="checkout", status!~"5.."}[1h]))
                /
                sum(rate(http_requests_total{service="checkout"}[1h]))
              )
            ) / 0.001  # 0.001 = 1 - 0.999 SLO target
          ) > 14
          and
          (
            # 5-minute window must also be burning fast — avoids single-minute noise
            (
              1 - (
                sum(rate(http_requests_total{service="checkout", status!~"5.."}[5m]))
                /
                sum(rate(http_requests_total{service="checkout"}[5m]))
              )
            ) / 0.001
          ) > 14
        labels:
          severity: critical
          alert_type: slo_burn
        annotations:
          summary: "Checkout SLO: fast burn rate — error budget exhausted in ~52h"
          runbook_url: "https://runbooks.internal/checkout-slo-burn"
          description: "Burn rate is {{ $value }}x the sustainable rate. Immediate action required."

      # Slow burn — consuming budget 3x faster than sustainable
      # At this rate, budget is gone in ~10 days. Urgent but not drop-everything.
      # Use LONGER windows — this is a trend, not a spike.
      - alert: CheckoutSLOSlowBurn
        expr: |
          (
            (
              1 - (
                sum(rate(http_requests_total{service="checkout", status!~"5.."}[6h]))
                /
                sum(rate(http_requests_total{service="checkout"}[6h]))
              )
            ) / 0.001
          ) > 3
          and
          (
            (
              1 - (
                sum(rate(http_requests_total{service="checkout", status!~"5.."}[30m]))
                /
                sum(rate(http_requests_total{service="checkout"}[30m]))
              )
            ) / 0.001
          ) > 3
        labels:
          severity: warning
          alert_type: slo_burn
        annotations:
          summary: "Checkout SLO: slow burn — error budget exhausted in ~10 days"
          runbook_url: "https://runbooks.internal/checkout-slo-burn"
          description: "Burn rate is {{ $value }}x sustainable. Investigate during business hours."