Kubernetes Monitoring with Prometheus — Deep Dive for Production

# This is the core Prometheus scrape configuration for Kubernetes pod discovery.
# It lives inside your prometheus.yml (or a PrometheusRule CR if using the operator).

scrape_configs:
  - job_name: 'kubernetes-pods'
    # Prometheus will query the Kubernetes API server to find all pods.
    kubernetes_sd_configs:
      - role: pod
        # Restrict discovery to a specific namespace for security isolation.
        namespaces:
          names:
            - production
            - staging

    # relabel_configs runs BEFORE each scrape — it filters and transforms targets.
    relabel_configs:
      # STEP 1: Only scrape pods that explicitly opt in via annotation.
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: 'true'

      # STEP 2: Allow pods to declare a custom metrics path.
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)

      # STEP 3: Allow pods to declare a custom port for scraping.
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: '([^:]+)(?::\d+)?;(\d+)'
        replacement: '$1:$2'
        target_label: __address__

      # STEP 4: Carry namespace as a label.
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace

      # STEP 5: Carry the pod name as a label.
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

      # STEP 6: Carry the app label from the pod.
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: replace
        target_label: app

      # STEP 7: Drop targets running on hostNetwork.
      - source_labels: [__meta_kubernetes_pod_host_ip, __address__]
        regex: '(\d+\.\d+\.\d+\.\d+);\1:\d+'
        action: drop

package main

import (
	"math/rand"
	"net/http"
	"strconv"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	httpRequestsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "payment_service_http_requests_total",
			Help: "Total number of HTTP requests processed by the payment service.",
		},
		[]string{"method", "endpoint", "status_code"},
	)

	httpRequestDuration = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name: "payment_service_http_request_duration_seconds",
			Help: "HTTP request duration in seconds, bucketed by endpoint.",
			Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1.0, 2.5},
		},
		[]string{"method", "endpoint"},
	)

	inFlightRequests = promauto.NewGauge(
		prometheus.GaugeOpts{
			Name: "payment_service_in_flight_requests",
			Help: "Number of HTTP requests currently being processed.",
		},
	)

	paymentQueueDepth = promauto.NewGauge(
		prometheus.GaugeOpts{
			Name: "payment_service_queue_depth",
			Help: "Current number of payments waiting in the processing queue.",
		},
	)
)

func instrumentedHandler(endpoint string, handlerFunc http.HandlerFunc) http.HandlerFunc {
	return func(responseWriter http.ResponseWriter, request *http.Request) {
		inFlightRequests.Inc()
		defer inFlightRequests.Dec()
		startTime := time.Now()
		wrappedWriter := &statusCapturingWriter{ResponseWriter: responseWriter, statusCode: http.StatusOK}
		handlerFunc(wrappedWriter, request)
		durationSeconds := time.Since(startTime).Seconds()
		httpRequestDuration.WithLabelValues(request.Method, endpoint).Observe(durationSeconds)
		httpRequestsTotal.WithLabelValues(
			request.Method,
			endpoint,
			strconv.Itoa(wrappedWriter.statusCode),
		).Inc()
	}
}

type statusCapturingWriter struct {
	http.ResponseWriter
	statusCode int
}

func (scw *statusCapturingWriter) WriteHeader(code int) {
	scw.statusCode = code
	scw.ResponseWriter.WriteHeader(code)
}

func processPayment(responseWriter http.ResponseWriter, request *http.Request) {
	processingTime := time.Duration(5+rand.Intn(295)) * time.Millisecond
	time.Sleep(processingTime)
	if rand.Float64() < 0.02 {
		http.Error(responseWriter, "upstream payment gateway timeout", http.StatusGatewayTimeout)
		return
	}
	responseWriter.WriteHeader(http.StatusOK)
	responseWriter.Write([]byte(`{"status":"processed"}`))
}

func main() {
	go func() {
		for {
			paymentQueueDepth.Set(float64(rand.Intn(500)))
			time.Sleep(5 * time.Second)
		}
	}()
	http.HandleFunc("/api/v1/payments", instrumentedHandler("/api/v1/payments", processPayment))
	http.Handle("/metrics", promhttp.Handler())
	go http.ListenAndServe(":9091", nil)
	http.ListenAndServe(":8080", nil)
}

# PrometheusRule custom resource — picked up automatically by the Prometheus Operator.
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: payment-service-slo-rules
  namespace: production
  labels:
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
    - name: payment_service_recording_rules
      interval: 1m
      rules:
        - record: job_endpoint:payment_service_http_requests_total:rate5m
          expr: |
            rate(payment_service_http_requests_total[5m])

        - record: job_endpoint:payment_service_error_ratio:rate5m
          expr: |
            sum by (job, endpoint) (
              rate(payment_service_http_requests_total{status_code=~"5.."}[5m])
            )
            /
            sum by (job, endpoint) (
              rate(payment_service_http_requests_total[5m])
            )

        - record: job_endpoint:payment_service_latency_p99:rate5m
          expr: |
            histogram_quantile(
              0.99,
              sum by (job, endpoint, le) (
                rate(payment_service_http_request_duration_seconds_bucket[5m])
              )
            )

    - name: payment_service_alerts
      rules:
        - alert: PaymentServiceHighErrorBurnRate
          expr: |
            job_endpoint:payment_service_error_ratio:rate5m > (14.4 * 0.001)
          for: 2m
          labels:
            severity: critical
            team: payments
            runbook_url: https://wiki.company.com/runbooks/payment-service-errors
          annotations:
            summary: "Payment service burning error budget at critical rate"
            description: |
              Endpoint {{ $labels.endpoint }} error ratio is {{ $value | humanizePercentage }}.

        - alert: PaymentServiceHighLatency
          expr: |
            job_endpoint:payment_service_latency_p99:rate5m > 0.5
          for: 5m
          labels:
            severity: warning
            team: payments
          annotations:
            summary: "Payment service p99 latency exceeds SLO threshold"

        - alert: PaymentQueueConsumerDead
          expr: |
            payment_service_queue_depth == 0
            and
            sum(job_endpoint:payment_service_http_requests_total:rate5m) > 10
          for: 3m
          labels:
            severity: critical
            team: payments
          annotations:
            summary: "Payment queue depth is zero but traffic is flowing"

# Alertmanager configuration — deployed as a Secret in the monitoring namespace.
# The Prometheus Operator picks this up from the alertmanager.yaml key.
global:
  resolve_timeout: 5m
  pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'
  slack_api_url: 'https://hooks.slack.com/services/T00/B00/xxxx'

route:
  # Default receiver for alerts that don't match any sub-route.
  receiver: 'slack-catchall'
  group_by: ['alertname', 'namespace', 'endpoint']
  group_wait: 30s         # Wait 30s to group similar alerts.
  group_interval: 5m      # Send grouped updates every 5m.
  repeat_interval: 4h     # Re-send unresolved alerts every 4h.

  routes:
    # Critical alerts -> PagerDuty (page the on-call engineer).
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      group_wait: 10s       # Page faster for critical alerts.
      repeat_interval: 1h   # Re-page every hour if unresolved.

    # Warning alerts -> Slack channel (no page, just notification).
    - match:
        severity: warning
      receiver: 'slack-warnings'
      group_wait: 1m
      repeat_interval: 4h

    # Team-specific routing.
    - match:
        team: payments
      receiver: 'slack-payments-team'

receivers:
  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: '<pagerduty-integration-key>'
        severity: 'critical'
        description: '{{ .CommonAnnotations.summary }}'
        details:
          firing: '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
          num_firing: '{{ .Alerts.Firing | len }}'
          runbook: '{{ .CommonAnnotations.runbook_url }}'

  - name: 'slack-warnings'
    slack_configs:
      - channel: '#alerts-warnings'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ "\n" }}{{ end }}'
        send_resolved: true

  - name: 'slack-payments-team'
    slack_configs:
      - channel: '#payments-alerts'
        title: '[Payments] {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ "\n" }}{{ end }}'

  - name: 'slack-catchall'
    slack_configs:
      - channel: '#alerts-catchall'
        title: 'Unrouted Alert: {{ .GroupLabels.alertname }}'

# Inhibition: suppress warning alerts when a critical alert is already firing
# for the same service. Prevents alert storms.
inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'namespace']

# Prometheus StatefulSet with retention and storage configuration.
# For the kube-prometheus-stack Helm chart, these go in prometheus.prometheusSpec.
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: kube-prometheus
  namespace: monitoring
spec:
  # Retention: how long to keep data locally.
  # 15d is typical. Longer retention = more disk and memory.
  retention: 15d

  # Retention by size: delete oldest blocks when storage exceeds this.
  # Use this as a safety net alongside time-based retention.
  retentionSize: 50GB

  # Storage: PVC for persistent TSDB blocks.
  storage:
    volumeClaimTemplate:
      spec:
        storageClassName: fast-ssd
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 100Gi

  # Resources: Prometheus is memory-hungry.
  # Rule of thumb: ~16KB per active time series.
  # 1M series = 16GB RAM. Plan accordingly.
  resources:
    requests:
      cpu: '1'
      memory: 16Gi
    limits:
      cpu: '4'
      memory: 32Gi

  # External labels: applied to all metrics when using Thanos/Cortex.
  # Identifies which Prometheus instance scraped the data.
  externalLabels:
    cluster: production-us-east-1
    environment: production

  # Thanos sidecar: uploads blocks to object storage for long-term retention.
  thanos:
    objectStorageConfig:
      name: thanos-objstore-config
      key: objstore.yml

  # Sample limit: max series per scrape target.
  # Safety net against high-cardinality targets.
  serviceMonitorSelectorNilUsesHelmValues: false
  podMonitorSelectorNilUsesHelmValues: false