N+1 Queries Hide in Low CPU — APM Metrics That Expose Them

package io.thecodeforge.apm;

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Scope;
import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
import io.opentelemetry.semconv.resource.attributes.ResourceAttributes;

import java.util.concurrent.TimeUnit;

/**
 * Production OpenTelemetry instrumentation for a Java service.
 *
 * This adds distributed tracing so you can see exactly where latency
 * is hiding — database calls, HTTP requests, or your own code.
 */
public class OpenTelemetryInstrumentation {

    private final Tracer tracer;

    public OpenTelemetryInstrumentation(String serviceName, String otlpEndpoint) {
        // Configure OTLP exporter — sends traces to collector (Jaeger, Tempo, etc.)
        OtlpGrpcSpanExporter spanExporter = OtlpGrpcSpanExporter.builder()
                .setEndpoint(otlpEndpoint)  // e.g., "http://jaeger-collector:14250"
                .setTimeout(30, TimeUnit.SECONDS)
                .build();

        Resource serviceResource = Resource.getDefault().toBuilder()
                .put(ResourceAttributes.SERVICE_NAME, serviceName)
                .put(ResourceAttributes.SERVICE_VERSION, "1.2.3")
                .build();

        SdkTracerProvider tracerProvider = SdkTracerProvider.builder()
                .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build())
                .setResource(serviceResource)
                .build();

        OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
                .setTracerProvider(tracerProvider)
                .buildAndRegisterGlobal();

        this.tracer = openTelemetry.getTracer(serviceName, "1.0.0");
    }

    /**
     * Example: instrument a database query with custom span.
     *
     * This creates a child span under the current request trace.
     * In APM UI, you'll see exactly how long the database call took
     * and can correlate it with other spans in the same trace.
     */
    public void executeDatabaseQuery(String query) {
        Span dbSpan = tracer.spanBuilder("DB Query")
                .setAttribute("db.statement", query)
                .setAttribute("db.system", "postgresql")
                .startSpan();

        try (Scope scope = dbSpan.makeCurrent()) {
            // Execute actual database query here
            // connection.execute(query);
            System.out.println("Executing: " + query);
        } catch (Exception e) {
            dbSpan.recordException(e);
            dbSpan.setAttribute("error", true);
            throw e;
        } finally {
            dbSpan.end();  // Duration recorded here — visible in trace UI
        }
    }

    /**
     * Example: instrument an HTTP call to an external API.
     */
    public void callExternalApi(String url) {
        Span httpSpan = tracer.spanBuilder("HTTP " + url)
                .setAttribute("http.url", url)
                .setAttribute("http.method", "GET")
                .startSpan();

        try (Scope scope = httpSpan.makeCurrent()) {
            // Make the actual HTTP call
            // httpClient.get(url);
            System.out.println("Calling: " + url);
        } catch (Exception e) {
            httpSpan.recordException(e);
            httpSpan.setAttribute("error", true);
            throw e;
        } finally {
            httpSpan.end();
        }
    }
}

package io.thecodeforge.apm;

import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.Counter;
import io.prometheus.client.Histogram;
import io.prometheus.client.exporter.HTTPServer;

import java.io.IOException;

/**
 * Production RED metrics (Rate, Errors, Duration) using Prometheus.
 *
 * These three metrics are enough to know if a service is healthy
 * from the user's perspective — without looking at CPU or memory.
 */
public class REDMetrics {

    // ─── RATE: Total requests per endpoint ───────────────────────────────────
    // Counter represents requests total. Rate = increase over time.
    private static final Counter requestTotal = Counter.build()
            .name("http_requests_total")
            .labelNames("method", "endpoint", "status")
            .help("Total HTTP requests")
            .register();

    // ─── ERRORS: Error counter (subset of requestTotal) ──────────────────────
    // Track errors separately for easier alerting, but also derived from requestTotal
    private static final Counter errorTotal = Counter.build()
            .name("http_errors_total")
            .labelNames("method", "endpoint", "error_type")
            .help("Total HTTP errors (status >= 500 or exception)")
            .register();

    // ─── DURATION: Request latency histogram ─────────────────────────────────
    // Buckets chosen to capture p50 (5-10ms), p95 (50-100ms), p99 (250-500ms)
    // Adjust buckets based on your service's typical latency.
    private static final Histogram requestDuration = Histogram.build()
            .name("http_request_duration_seconds")
            .labelNames("method", "endpoint")
            .help("HTTP request latency in seconds")
            .buckets(0.001, 0.005, 0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
            .register();

    /**
     * Record metrics for a completed request.
     * Call this in your API framework's response filter/middleware.
     */
    public static void recordRequest(String method, String endpoint,
                                      int statusCode, long durationMs) {
        String status = String.valueOf(statusCode);
        requestTotal.labels(method, endpoint, status).inc();

        if (statusCode >= 500) {
            errorTotal.labels(method, endpoint, "http_5xx").inc();
        }

        requestDuration.labels(method, endpoint).observe(durationMs / 1000.0);
    }

    /**
     * Record an exception that wasn't caught by normal status code handling.
     */
    public static void recordException(String method, String endpoint, String exceptionType) {
        errorTotal.labels(method, endpoint, exceptionType).inc();
    }

    /**
     * Start Prometheus metrics endpoint on port 8081 (separate from app port).
     * Scraped by Prometheus every 15 seconds.
     */
    public static void startMetricsServer() throws IOException {
        HTTPServer server = new HTTPServer(8081);
        System.out.println("Prometheus metrics available at http://localhost:8081/metrics");
    }
}

package io.thecodeforge.apm;

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Context;
import io.opentelemetry.context.propagation.TextMapGetter;
import io.opentelemetry.context.propagation.TextMapSetter;

import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

/**
 * Distributed tracing with context propagation across service boundaries.
 *
 * The key challenge in distributed tracing is propagating the trace context
 * from caller to callee. OpenTelemetry's propagator handles automatically
 * when using instrumented clients. For custom protocols (message queues),
 * inject the context manually.
 */
public class DistributedTracing {

    private final Tracer tracer;
    private final OpenTelemetry openTelemetry;
    private final HttpClient httpClient;

    public DistributedTracing(OpenTelemetry openTelemetry) {
        this.openTelemetry = openTelemetry;
        this.tracer = openTelemetry.getTracer("api-service");
        this.httpClient = HttpClient.newHttpClient();
    }

    /**
     * Example: calling a downstream service with automatic trace propagation.
     *
     * When using OpenTelemetry-instrumented HTTP client, the trace context
     * is automatically injected into the `traceparent` header.
     * The downstream service extracts it and creates a child span.
     */
    public String callOrderService(String orderId) throws Exception {
        // Start a child span for this HTTP call
        Span httpSpan = tracer.spanBuilder("HTTP POST /orders")
                .setAttribute("order.id", orderId)
                .startSpan();

        try (var scope = httpSpan.makeCurrent()) {
            HttpRequest request = HttpRequest.newBuilder()
                    .uri(java.net.URI.create("http://order-service/api/orders"))
                    .header("Content-Type", "application/json")
                    .POST(HttpRequest.BodyPublishers.ofString("{\"id\":\"" + orderId + "\"}"))
                    .build();

            // If using automatic instrumentation, the `traceparent` header
            // is added automatically. If manual, inject via:
            // openTelemetry.getPropagators().getTextMapPropagator()
            //     .inject(Context.current(), request, (r, k, v) -> r.headers().put(k, v));

            HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());

            httpSpan.setAttribute("http.status_code", response.statusCode());
            return response.body();
        } catch (Exception e) {
            httpSpan.recordException(e);
            throw e;
        } finally {
            httpSpan.end();
        }
    }

    /**
     * Extract trace context from incoming request headers.
     * OpenTelemetry's server instrumentation does this automatically.
     *
     * This is how a service knows it's part of an existing trace
     * rather than starting a new one.
     */
    public void handleIncomingRequest(String traceparentHeader) {
        // Extract context from headers (auto-instrumented frameworks do this)
        TextMapGetter<MapHeaders> getter = new TextMapGetter<>() {
            @Override
            public String get(MapHeaders carrier, String key) {
                return carrier.get(key);
            }

            @Override
            public Iterable<String> keys(MapHeaders carrier) {
                return carrier.keys();
            }
        };

        Context extractedContext = openTelemetry.getPropagators().getTextMapPropagator()
                .extract(Context.current(), new MapHeaders(traceparentHeader), getter);

        // Start a span as child of extracted context
        Span span = tracer.spanBuilder("handle request")
                .setParent(extractedContext)
                .startSpan();

        try (var scope = span.makeCurrent()) {
            // Process the request here
            System.out.println("Processing request with trace ID: " + span.getSpanContext().getTraceId());
        } finally {
            span.end();
        }
    }

    // Helper class for header propagation example
    static class MapHeaders {
        private final java.util.Map<String, String> headers = new java.util.HashMap<>();
        MapHeaders(String traceparent) { headers.put("traceparent", traceparent); }
        String get(String key) { return headers.get(key); }
        Iterable<String> keys() { return headers.keySet(); }
    }
}