Kubernetes Monitoring with Prometheus — Deep Dive for Production

# This is the core Prometheus scrape configuration for Kubernetes pod discovery.
# It lives inside your prometheus.yml (or a PrometheusRule CR if using the operator).

scrape_configs:
  - job_name: 'kubernetes-pods'
    # Prometheus will query the Kubernetes API server to find all pods.
    kubernetes_sd_configs:
      - role: pod
        # Restrict discovery to a specific namespace for security isolation.
        # Remove this line to discover pods cluster-wide.
        namespaces:
          names:
            - production
            - staging

    # relabel_configs runs BEFORE each scrape — it filters and transforms targets.
    relabel_configs:
      # STEP 1: Only scrape pods that explicitly opt in via annotation.
      # If prometheus.io/scrape is not 'true', drop this target entirely.
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: 'true'

      # STEP 2: Allow pods to declare a custom metrics path (default is /metrics).
      # e.g. annotation: prometheus.io/path: '/actuator/prometheus'
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)

      # STEP 3: Allow pods to declare a custom port for scraping.
      # e.g. annotation: prometheus.io/port: '8080'
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        # Regex captures the IP from __address__ and combines with the annotation port.
        regex: '([^:]+)(?::\d+)?;(\d+)'
        replacement: '$1:$2'
        target_label: __address__

      # STEP 4: Carry namespace as a label so dashboards can filter by it.
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace

      # STEP 5: Carry the pod name as a label for drill-down in Grafana.
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

      # STEP 6: Carry the app label from the pod so we can group by application.
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: replace
        target_label: app

      # STEP 7: Drop any targets running on hostNetwork — they often expose
      # system-level ports and should be handled by the node exporter job instead.
      - source_labels: [__meta_kubernetes_pod_host_ip, __address__]
        regex: '(\d+\.\d+\.\d+\.\d+);\1:\d+'
        action: drop

package main

import (
	"math/rand"
	"net/http"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

// --- Metric Declarations ---
// promauto.New* registers metrics automatically with the default registry.
// Always declare metrics at package level — not inside handlers — to avoid
// re-registering on every request (causes a panic in production).

var (
	// Counter: tracks total HTTP requests. Labels let us slice by method and status code.
	httpRequestsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "payment_service_http_requests_total",
			Help: "Total number of HTTP requests processed by the payment service.",
		},
		[]string{"method", "endpoint", "status_code"},
	)

	// Histogram: tracks request duration. Bucket boundaries are chosen to match
	// our SLO thresholds: 95th percentile < 200ms, 99th percentile < 500ms.
	// DefBuckets (default) go up to 10s which is often too coarse for APIs.
	httpRequestDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "payment_service_http_request_duration_seconds",
			Help: "HTTP request duration in seconds, bucketed by endpoint.",
			// Custom buckets aligned to SLO boundaries — critical in production.
			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1.0, 2.5},
		},
		[]string{"method", "endpoint"},
	)

	// Gauge: tracks the number of in-flight (currently processing) requests.
	// This helps detect thread pool saturation before latency spikes show up.
	inFlightRequests = promauto.NewGauge(
		prometheus.GaugeOpts{
			Name: "payment_service_in_flight_requests",
			Help: "Number of HTTP requests currently being processed.",
		},
	)

	// Gauge: tracks the depth of the async payment processing queue.
	paymentQueueDepth = promauto.NewGauge(
		prometheus.GaugeOpts{
			Name: "payment_service_queue_depth",
			Help: "Current number of payments waiting in the processing queue.",
		},
	)
)

// instrumentedHandler wraps an HTTP handler with Prometheus instrumentation.
// It measures latency, increments the request counter, and tracks in-flight requests.
func instrumentedHandler(endpoint string, handlerFunc http.HandlerFunc) http.HandlerFunc {
	return func(responseWriter http.ResponseWriter, request *http.Request) {
		// Record that one more request is in flight.
		inFlightRequests.Inc()
		// Use defer so this always runs even if the handler panics.
		defer inFlightRequests.Dec()

		// Start the latency timer.
		startTime := time.Now()

		// Wrap the ResponseWriter to capture the status code.
		wrappedWriter := &statusCapturingWriter{ResponseWriter: responseWriter, statusCode: http.StatusOK}

		// Call the actual business logic handler.
		handlerFunc(wrappedWriter, request)

		// Calculate how long the request took.
		durationSeconds := time.Since(startTime).Seconds()

		// Record latency in the histogram — this updates _bucket, _sum, and _count.
		httpRequestDuration.WithLabelValues(request.Method, endpoint).Observe(durationSeconds)

		// Increment the total requests counter with the final status code.
		httpRequestsTotal.WithLabelValues(
			request.Method,
			endpoint,
			strconv.Itoa(wrappedWriter.statusCode),
		).Inc()
	}
}

// statusCapturingWriter is a thin wrapper around http.ResponseWriter that
// intercepts WriteHeader so we can record the HTTP status code as a label.
type statusCapturingWriter struct {
	http.ResponseWriter
	statusCode int
}

func (scw *statusCapturingWriter) WriteHeader(code int) {
	scw.statusCode = code
	scw.ResponseWriter.WriteHeader(code)
}

// processPayment simulates payment processing with realistic latency variance.
func processPayment(responseWriter http.ResponseWriter, request *http.Request) {
	// Simulate variable processing time between 5ms and 300ms.
	processingTime := time.Duration(5+rand.Intn(295)) * time.Millisecond
	time.Sleep(processingTime)

	// Simulate a 2% error rate — realistic for downstream dependency failures.
	if rand.Float64() < 0.02 {
		http.Error(responseWriter, "upstream payment gateway timeout", http.StatusGatewayTimeout)
		return
	}

	responseWriter.WriteHeader(http.StatusOK)
	responseWriter.Write([]byte(`{"status":"processed"}`))
}

func main() {
	// Simulate queue depth fluctuations in a background goroutine.
	// In production this would read from your actual queue (Kafka, SQS, etc.).
	go func() {
		for {
			// Randomly vary queue depth between 0 and 500 items.
			paymentQueueDepth.Set(float64(rand.Intn(500)))
			time.Sleep(5 * time.Second)
		}
	}()

	// Register our instrumented business handler.
	http.HandleFunc("/api/v1/payments", instrumentedHandler("/api/v1/payments", processPayment))

	// Expose the /metrics endpoint — Prometheus will scrape this.
	// NEVER put /metrics behind authentication that Prometheus can't pass.
	http.Handle("/metrics", promhttp.Handler())

	// Listen on a separate port for metrics to allow network policies
	// to restrict /metrics access without blocking API traffic.
	go http.ListenAndServe(":9091", nil) // metrics port
	http.ListenAndServe(":8080", nil)    // API port (uses DefaultServeMux)
}

# This PrometheusRule custom resource is picked up automatically by the
# Prometheus Operator. No Prometheus restart needed — the operator reconciles it.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: payment-service-slo-rules
  namespace: production
  labels:
    # This label must match the ruleSelector in your Prometheus CR.
    # If this label is missing, the Prometheus Operator silently ignores the file.
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
    # ─────────────────────────────────────────────
    # GROUP 1: Recording Rules — pre-compute expensive queries
    # ─────────────────────────────────────────────
    - name: payment_service_recording_rules
      # Evaluate every 1 minute — must be >= scrape_interval.
      interval: 1m
      rules:
        # Pre-compute 5-minute request rate per job and endpoint.
        # Queries against this are ~100x cheaper than querying the raw counter.
        - record: job_endpoint:payment_service_http_requests_total:rate5m
          expr: |
            rate(payment_service_http_requests_total[5m])

        # Pre-compute the error ratio (4xx + 5xx / total) per endpoint.
        # This is the core signal for SLO tracking.
        - record: job_endpoint:payment_service_error_ratio:rate5m
          expr: |
            sum by (job, endpoint) (
              rate(payment_service_http_requests_total{
                status_code=~"5.."
              }[5m])
            )
            /
            sum by (job, endpoint) (
              rate(payment_service_http_requests_total[5m])
            )

        # Pre-compute p99 latency using the histogram.
        # histogram_quantile needs the _bucket series — this is why we use histograms.
        - record: job_endpoint:payment_service_latency_p99:rate5m
          expr: |
            histogram_quantile(
              0.99,
              sum by (job, endpoint, le) (
                rate(payment_service_http_request_duration_seconds_bucket[5m])
              )
            )

    # ─────────────────────────────────────────────
    # GROUP 2: Alerts — fire on SLO burn rate, not raw thresholds
    # ─────────────────────────────────────────────
    - name: payment_service_alerts
      rules:
        # ALERT 1: High-burn-rate alert (fast burn — page immediately).
        # This fires if the error rate is 14.4x the SLO budget for 1 hour.
        # At this rate, you'll exhaust a 30-day 99.9% SLO budget in 2 hours.
        # Using 'for: 2m' prevents single-scrape blips from firing pages.
        - alert: PaymentServiceHighErrorBurnRate
          expr: |
            (
              job_endpoint:payment_service_error_ratio:rate5m > (14.4 * 0.001)
            )
          for: 2m
          labels:
            severity: critical
            team: payments
            # runbook_url label is a convention — attach it to PagerDuty in Alertmanager.
            runbook_url: https://wiki.company.com/runbooks/payment-service-errors
          annotations:
            summary: "Payment service burning error budget at critical rate"
            description: |
              Endpoint {{ $labels.endpoint }} error ratio is {{ $value | humanizePercentage }},
              which is 14.4x above the 0.1% SLO target. At this rate, the monthly
              error budget will be exhausted within 2 hours. Check downstream
              payment gateway connectivity and database connection pool saturation.

        # ALERT 2: Latency SLO breach.
        # Fires if p99 latency exceeds 500ms for 5 consecutive minutes.
        - alert: PaymentServiceHighLatency
          expr: |
            job_endpoint:payment_service_latency_p99:rate5m > 0.5
          for: 5m
          labels:
            severity: warning
            team: payments
          annotations:
            summary: "Payment service p99 latency exceeds SLO threshold"
            description: |
              p99 latency for {{ $labels.endpoint }} is {{ $value | humanizeDuration }},
              exceeding the 500ms SLO threshold. Current queue depth:
              {{ with query "payment_service_queue_depth" }}{{ . | first | value | humanize }}{{ end }} items.

        # ALERT 3: Dead queue — queue depth at zero while requests are incoming.
        # This catches silent consumer crashes before users notice.
        - alert: PaymentQueueConsumerDead
          expr: |
            payment_service_queue_depth == 0
            and
            sum(job_endpoint:payment_service_http_requests_total:rate5m) > 10
          for: 3m
          labels:
            severity: critical
            team: payments
          annotations:
            summary: "Payment queue depth is zero but traffic is flowing — consumer likely crashed"