N+1 Queries Hide in Low CPU — APM Metrics That Expose Them

package io.thecodeforge.apm;

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Scope;
import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;

import java.util.concurrent.TimeUnit;

/**
 * Production OpenTelemetry instrumentation for a Java service.
 *
 * This adds distributed tracing so you can see exactly where latency
 * is hiding — database calls, HTTP requests, or your own code.
 */
public class OpenTelemetryInstrumentation {

    private final Tracer tracer;

    public OpenTelemetryInstrumentation(String serviceName, String otlpEndpoint) {
        // Configure OTLP exporter — sends traces to collector (Jaeger, Tempo, etc.)
        OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder()
                .setEndpoint(otlpEndpoint)  // e.g., "http://jaeger-collector:14250"
                .setTimeout(30, TimeUnit.SECONDS)
                .build();

        Resource serviceResource = Resource.getDefault().toBuilder()
                .put(ResourceAttributes.SERVICE_NAME, serviceName)
                .put(ResourceAttributes.SERVICE_VERSION, "1.2.3")
                .build();

        SdkTracerProvider tracerProvider = SdkTracerProvider.builder()
                .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build())
                .setResource(serviceResource)
                .build();

        OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
                .setTracerProvider(tracerProvider)
                .buildAndRegisterGlobal();

        this.tracer = openTelemetry.getTracer(serviceName, "1.0.0");
    }

    /**
     * Example: instrument a database query with custom span.
     *
     * This creates a child span under the current request trace.
     * In APM UI, you'll see exactly how long the database call took
     * and can correlate it with other spans in the same trace.
     */
    public void executeDatabaseQuery(String query) {
        Span dbSpan = tracer.spanBuilder("DB Query")
                .setAttribute("db.statement", query)
                .setAttribute("db.system", "postgresql")
                .startSpan();

        try (Scope scope = dbSpan.makeCurrent()) {
            // Execute actual database query here
            // connection.execute(query);
            System.out.println("Executing: " + query);
        } catch (Exception e) {
            dbSpan.recordException(e);
            dbSpan.setAttribute("error", true);
            throw e;
        } finally {
            dbSpan.end();  // Duration recorded here — visible in trace UI
        }
    }

    /**
     * Example: instrument an HTTP call to an external API.
     */
    public void callExternalApi(String url) {
        Span httpSpan = tracer.spanBuilder("HTTP " + url)
                .setAttribute("http.url", url)
                .setAttribute("http.method", "GET")
                .startSpan();

        try (Scope scope = httpSpan.makeCurrent()) {
            // Make the actual HTTP call
            // httpClient.get(url);
            System.out.println("Calling: " + url);
        } catch (Exception e) {
            httpSpan.recordException(e);
            httpSpan.setAttribute("error", true);
            throw e;
        } finally {
            httpSpan.end();
        }
    }
}

package io.thecodeforge.apm;

import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.Counter;
import io.prometheus.client.Histogram;
import io.prometheus.client.exporter.HTTPServer;

import java.io.IOException;

/**
 * Production RED metrics (Rate, Errors, Duration) using Prometheus.
 *
 * These three metrics are enough to know if a service is healthy
 * from the user's perspective — without looking at CPU or memory.
 */
public class REDMetrics {

    // ─── RATE: Total requests per endpoint ───────────────────────────────────
    // Counter represents requests total. Rate = increase over time.
    private static final Counter requestTotal = Counter.build()
            .name("http_requests_total")
            .labelNames("method", "endpoint", "status")
            .help("Total HTTP requests")
            .register();

    // ─── ERRORS: Error counter (subset of requestTotal) ──────────────────────
    // Track errors separately for easier alerting, but also derived from requestTotal
    private static final Counter errorTotal = Counter.build()
            .name("http_errors_total")
            .labelNames("method", "endpoint", "error_type")
            .help("Total HTTP errors (status >= 500 or exception)")
            .register();

    // ─── DURATION: Request latency histogram ─────────────────────────────────
    // Buckets chosen to capture p50 (5-10ms), p95 (50-100ms), p99 (250-500ms)
    // Adjust buckets based on your service's typical latency.
    private static final Histogram requestDuration = Histogram.build()
            .name("http_request_duration_seconds")
            .labelNames("method", "endpoint")
            .help("HTTP request latency in seconds")
            .buckets(0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
            .register();

    /**
     * Record metrics for a completed request.
     * Call this in your API framework's response filter/middleware.
     */
    public static void recordRequest(String method, String endpoint,
                                      int statusCode, long durationMs) {
        String status = String.valueOf(statusCode);
        requestTotal.labels(method, endpoint, status).inc();

        if (statusCode >= 500) {
            errorTotal.labels(method, endpoint, "http_5xx").inc();
        }

        requestDuration.labels(method, endpoint).observe(durationMs / 1000.0);
    }

    /**
     * Record an exception that wasn't caught by normal status code handling.
     */
    public static void recordException(String method, String endpoint, String exceptionType) {
        errorTotal.labels(method, endpoint, exceptionType).inc();
    }

    /**
     * Start Prometheus metrics endpoint on port 8081 (separate from app port).
     * Scraped by Prometheus every 15 seconds.
     */
    public static void startMetricsServer() throws IOException {
        HTTPServer server = new HTTPServer(8081);
        System.out.println("Prometheus metrics available at http://localhost:8081/metrics");
    }
}

package io.thecodeforge.apm;

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Context;
import io.opentelemetry.context.propagation.TextMapGetter;
import io.opentelemetry.context.propagation.TextMapSetter;

import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

/**
 * Distributed tracing with context propagation across service boundaries.
 *
 * The key challenge in distributed tracing is propagating the trace context
 * from caller to callee. OpenTelemetry's propagator handles automatically
 * when using instrumented clients. For custom protocols (message queues),
 * inject the context manually.
 */
public class DistributedTracing {

    private final Tracer tracer;
    private final OpenTelemetry openTelemetry;
    private final HttpClient httpClient;

    public DistributedTracing(OpenTelemetry openTelemetry) {
        this.openTelemetry = openTelemetry;
        this.tracer = openTelemetry.getTracer("api-service");
        this.httpClient = HttpClient.newHttpClient();
    }

    /**
     * Example: calling a downstream service with automatic trace propagation.
     *
     * When using OpenTelemetry-instrumented HTTP client, the trace context
     * is automatically injected into the `traceparent` header.
     * The downstream service extracts it and creates a child span.
     */
    public String callOrderService(String orderId) throws Exception {
        // Start a child span for this HTTP call
        Span httpSpan = tracer.spanBuilder("HTTP POST /orders")
                .setAttribute("order.id", orderId)
                .startSpan();

        try (var scope = httpSpan.makeCurrent()) {
            HttpRequest request = HttpRequest.newBuilder()
                    .uri(java.net.URI.create("http://order-service/api/orders"))
                    .header("Content-Type", "application/json")
                    .POST(HttpRequest.BodyPublishers.ofString("{\"id\":\"" + orderId + "\"}"))
                    .build();

            // If using automatic instrumentation, the `traceparent` header
            // is added automatically. If manual, inject via:
            // openTelemetry.getPropagators().getTextMapPropagator()
            //     .inject(Context.current(), request, (r, k, v) -> r.headers().put(k, v));

            HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());

            httpSpan.setAttribute("http.status_code", response.statusCode());
            return response.body();
        } catch (Exception e) {
            httpSpan.recordException(e);
            throw e;
        } finally {
            httpSpan.end();
        }
    }

    /**
     * Extract trace context from incoming request headers.
     * OpenTelemetry's server instrumentation does this automatically.
     *
     * This is how a service knows it's part of an existing trace
     * rather than starting a new one.
     */
    public void handleIncomingRequest(String traceparentHeader) {
        // Extract context from headers (auto-instrumented frameworks do this)
        TextMapGetter<MapHeaders> getter = new TextMapGetter<>() {
            @Override
            public String get(MapHeaders carrier, String key) {
                return carrier.get(key);
            }

            @Override
            public Iterable<String> keys(MapHeaders carrier) {
                return carrier.keys();
            }
        };

        Context extractedContext = openTelemetry.getPropagators().getTextMapPropagator()
                .extract(Context.current(), new MapHeaders(traceparentHeader), getter);

        // Start a span as child of extracted context
        Span span = tracer.spanBuilder("handle request")
                .setParent(extractedContext)
                .startSpan();

        try (var scope = span.makeCurrent()) {
            // Process the request here
            System.out.println("Processing request with trace ID: " + span.getSpanContext().getTraceId());
        } finally {
            span.end();
        }
    }

    // Helper class for header propagation example
    static class MapHeaders {
        private final java.util.Map<String, String> headers = new java.util.HashMap<>();
        MapHeaders(String traceparent) { headers.put("traceparent", traceparent); }
        String get(String key) { return headers.get(key); }
        Iterable<String> keys() { return headers.keySet(); }
    }
}

// io.thecodeforge — devops tutorial

# Minimal APM config for a payment service
# Track the customer-impacting metrics, not the server noise

apm:
  service_name: payment-gateway
  environment: production
  metrics:
    - response_time_p99      # Slowest 1% of requests
    - error_rate_percent     # 4xx/5xx vs total requests
    - apdex_score            # User satisfaction threshold
  alerting:
    # Fire when p99 crosses 2 seconds for 5 minutes
    - rule: p99_latency_high
      condition: "p99 > 2000ms for 5m"
      severity: critical
      oncall: sre-team
    - rule: error_rate_spike
      condition: "error_rate > 5% for 1m"
      severity: warning
      oncall: dev-oncall

// io.thecodeforge — devops tutorial

# APM stack configuration for a microservices deployment
# Each component runs in its own namespace

components:
  euem:
    tool: rum-sensor
    enabled: true
    capture_requests: true
    capture_errors: true
  runtime:
    instrumentations:
      - python: "0.12.0"  # Auto-instrument all Python services
      - nodejs: "0.9.2"
      - java: "0.15.1"
  infrastructure:
    agent: node-exporter
    enabled: true
    metrics:
      - cpu_usage
      - memory_rss
      - disk_iops
  tracing:
    sampler: probabilistic
    sampling_rate: 0.1  # Trace 10% of requests in prod
    export: otlp-http

// io.thecodeforge — devops tutorial

# Essential metrics config for an API gateway
# Only four metrics, but correlated across all services

metrics:
  latency:
    dashboard: p50: 120ms | p95: 350ms | p99: 800ms
    alarm: p99 > 1500ms for 3 minutes
  error_rate:
    dashboard: 0.3% of requests (24/hour)
    error_budget: 99.9% uptime = 0.1% errors/month
    alarm: rate > 1% for 1 minute
  saturation:
    dashboard: db_conn_pool: 67% (134/200 connections)
    alarm: saturation > 80% for 5 minutes
  throughput:
    dashboard: 420 req/s (rolling 1m average)
    alarm: throughput drops > 50% in 1 minute

// io.thecodeforge — devops tutorial

apiVersion: v1
kind: Pod
metadata:
  name: payment-service-v2
  labels:
    app: payment
    version: "2.1.0"
spec:
  containers:
  - name: payment
    image: payment:2.1.0
    env:
    - name: OTEL_RESOURCE_ATTRIBUTES
      value: "k8s.pod.name=$(POD_NAME),k8s.node.name=$(NODE_NAME)"
    - name: POD_NAME
      valueFrom:
        fieldRef:
          fieldPath: metadata.name
    - name: NODE_NAME
      valueFrom:
        fieldRef:
          fieldPath: spec.nodeName

// io.thecodeforge — devops tutorial

groups:
- name: apm-critical
  rules:
  - alert: HighErrorRate
    expr: |
      rate(http_requests_total{status=~"5.."}[5m]) /
      rate(http_requests_total[5m]) > 0.02
    for: 3m
    labels:
      severity: critical
      team: platform
    annotations:
      summary: "Service {{ $labels.service }} error rate > 2%"
      runbook: "https://runbooks.thecodeforge.io/apm/error-spike"
      trace_id: "{{ $labels.trace_id }}"

// io.thecodeforge — devops tutorial

log_schema:
  version: "1.0"
  fields:
  - name: timestamp
    type: rfc3339
    required: true
  - name: level
    type: enum
    values: [DEBUG, INFO, WARN, ERROR, FATAL]
  - name: service
    type: string
  - name: trace_id
    type: string
    required: true
  - name: span_id
    type: string
  - name: message
    type: string
    required: true
  - name: error
    type: object
    properties:
      type: string
      stack: string
  - name: metadata
    type: object
    description: "Arbitrary key-value pairs for context"
  retention: 30 days