Monitoring vs Observability — P99 Blind Spot Disaster

package io.thecodeforge.monitoring;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import io.opentelemetry.api.trace.Span;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

/**
 * io.thecodeforge — Three-Pillar Instrumentation Pattern
 *
 * Demonstrates all three observability pillars in a single operation:
 *   - Metric: forge.orders.success (counter) and forge.payment.latency (histogram timer)
 *   - Log:    Structured log line with traceId injected from active OpenTelemetry span
 *   - Trace:  Active span context is propagated by the OTel agent — no manual span creation needed
 *
 * Key design decisions:
 *   1. traceId comes from the active OTel Span — it is the same ID visible in Jaeger/Tempo
 *   2. MDC injection makes traceId appear in EVERY log line within this thread scope
 *   3. The Timer wraps the entire business operation — it measures wall clock time including I/O
 *   4. Counter uses .tag("status", "success") so failures can be counted separately:
 *      forge.orders.processed{status="failure"} without creating a separate metric
 */
public class OrderMonitor {

    private static final Logger log = LoggerFactory.getLogger(OrderMonitor.class);

    private final Counter orderSuccessCounter;
    private final Counter orderFailureCounter;
    private final Timer paymentTimer;

    public OrderMonitor(MeterRegistry registry) {
        // Use tagged counters so success and failure are queryable separately in PromQL
        // rate(forge.orders.processed_total{status="failure"}[5m]) gives failure rate
        this.orderSuccessCounter = registry.counter("forge.orders.processed", "status", "success");
        this.orderFailureCounter = registry.counter("forge.orders.processed", "status", "failure");

        // Timer automatically produces _count, _sum, and _bucket metrics
        // Enables histogram_quantile(0.99, rate(forge_payment_latency_seconds_bucket[5m]))
        this.paymentTimer = registry.timer("forge.payment.latency");
    }

    public void processOrder(String orderId) {
        // Inject the active OpenTelemetry trace ID into MDC before any log statements
        // This ensures EVERY log line within this method includes the traceId automatically
        String traceId = Span.current().getSpanContext().getTraceId();
        MDC.put("traceId", traceId);
        MDC.put("orderId", orderId);

        try {
            paymentTimer.record(() -> {
                // log.info output will include traceId and orderId from MDC automatically
                // Logback/Log4j2 pattern: %X{traceId} %X{orderId} %msg
                log.info("Order processing started");

                // ... business logic: validate, reserve inventory, charge payment ...

                orderSuccessCounter.increment();
                log.info("Order processing completed successfully");
            });
        } catch (Exception e) {
            orderFailureCounter.increment();
            // ERROR logs include traceId from MDC — jump directly to the trace in Jaeger
            log.error("Order processing failed — check trace for upstream dependency state",
                Map.of("error", e.getMessage(), "errorType", e.getClass().getSimpleName()));
            throw e;
        } finally {
            // Always clear MDC — thread pool reuse will leak context if you skip this
            MDC.clear();
        }
    }
}

/*
 * Example log output (JSON format via logstash-logback-encoder):
 * {
 *   "level":   "INFO",
 *   "logger":  "io.thecodeforge.monitoring.OrderMonitor",
 *   "message": "Order processing completed successfully",
 *   "traceId": "4bf92f3577b34da6a3ce929d0e0e4736",   <-- same ID in Jaeger/Tempo
 *   "orderId": "ORD-101"
 * }
 *
 * Metrics emitted (visible in Prometheus):
 *   forge_orders_processed_total{status="success"} 1.0
 *   forge_payment_latency_seconds_count 1.0
 *   forge_payment_latency_seconds_sum 0.043
 *   forge_payment_latency_seconds_bucket{le="0.05"} 1.0
 */

# io.thecodeforge — Production Prometheus Scrape Configuration
#
# Design decisions explained inline:
#   - scrape_interval: 15s is the standard for most services
#     Reduce to 5s for latency-sensitive paths, but watch Prometheus CPU/memory
#   - honor_labels: false (default) — Prometheus overwrites conflicting labels
#     from the target with its own. Set true only if you trust the target's labels.
#   - relabel_configs allow normalizing instance labels before storage
#     This prevents cardinality drift from inconsistent hostname formats

global:
  scrape_interval: 15s
  evaluation_interval: 15s  # How often alerting rules are evaluated
  # Global labels added to every time series scraped by this instance
  external_labels:
    datacenter: 'us-east-1'
    environment: 'production'

rule_files:
  - 'rules/forge_alerts.yml'   # Alert rules — evaluated every evaluation_interval
  - 'rules/forge_records.yml'  # Recording rules — pre-aggregate expensive PromQL queries

scrape_configs:
  - job_name: 'forge-api-service'
    # Spring Boot Actuator exposes Prometheus metrics at /actuator/prometheus
    # Not /metrics — this is a common misconfiguration that produces DOWN targets
    metrics_path: '/actuator/prometheus'
    scrape_interval: 5s      # Override global for latency-sensitive checkout service
    scrape_timeout: 4s       # Must be less than scrape_interval to avoid overlap
    static_configs:
      - targets:
          - 'forge-prod-app-01:8080'
          - 'forge-prod-app-02:8080'
        labels:
          service: 'order-service'  # Added to every time series from these targets
    relabel_configs:
      # Normalize the instance label to a stable name instead of host:port
      # Prevents cardinality drift if the port changes or hostname format varies
      - source_labels: [__address__]
        regex: '(forge-prod-app-\d+):.*'
        target_label: instance
        replacement: '$1'

  - job_name: 'forge-payment-service'
    metrics_path: '/actuator/prometheus'
    static_configs:
      - targets: ['forge-payment:8080']
        labels:
          service: 'payment-service'

---
# rules/forge_alerts.yml
# Alert on P99 latency — not average. Average latency alerts miss tail latency problems.
groups:
  - name: forge.latency
    rules:
      - alert: ForgeCheckoutP99LatencyHigh
        expr: |
          histogram_quantile(
            0.99,
            rate(http_server_requests_seconds_bucket{
              job="forge-api-service",
              uri="/api/checkout"
            }[5m])
          ) > 2.0
        for: 3m   # Must be elevated for 3 minutes before firing — reduces noise
        labels:
          severity: critical
          team: platform
        annotations:
          summary: 'Checkout P99 latency exceeds 2s'
          description: |
            P99 latency for /api/checkout is {{ $value | humanizeDuration }}.
            Check Jaeger for slow traces on this endpoint.
            Dashboard: https://grafana.thecodeforge.io/d/checkout-slo

---
# rules/forge_records.yml
# Recording rules pre-aggregate expensive queries so dashboards load instantly
# Without this, a 7-day P99 query runs at render time and takes 30+ seconds
groups:
  - name: forge.recordings
    rules:
      - record: job:http_request_duration_p99:rate5m
        expr: |
          histogram_quantile(
            0.99,
            sum by (job, le) (
              rate(http_server_requests_seconds_bucket[5m])
            )
          )