Spring Boot Actuator and Monitoring: Production-Grade Observability

# io.thecodeforge: Production Observability Stack
#
#  ┌─────────────┐     ┌─────────────┐     ┌─────────────┐
#  │  Spring Boot │────▶│  Prometheus │────▶│   Grafana   │
#  │   Actuator   │     │  (Scraper)  │     │ (Dashboard) │
#  └─────────────┘     └─────────────┘     └─────────────┘
#         │                                        │
#         │              ┌─────────────┐           │
#         └─────────────▶│ Alertmanager│◀──────────┘
#                        │  (PagerDuty)│
#                        └─────────────┘
#
#  ┌─────────────┐     ┌─────────────┐
#  │ Structured  │────▶│  Loki / ELK │
#  │    Logs     │     │  (Log Agg.) │
#  └─────────────┘     └─────────────┘
#
#  ┌─────────────┐     ┌─────────────┐
#  │  Micrometer │────▶│  Tempo /    │
#  │   Tracing   │     │  Jaeger     │
#  └─────────────┘     └─────────────┘
#
# What each layer answers:
#   Metrics  → Is something wrong right now?
#   Logs     → What exactly happened and why?
#   Traces   → Which service in the chain caused it?

# io.thecodeforge: Canonical Actuator Configuration
# This is the base configuration every production Spring Boot service should start with.
# Copy this, adjust the exposure list to match your needs, and ship it.

management:
  endpoints:
    web:
      exposure:
        # Whitelist only what your monitoring stack actually needs.
        # Never use * in production.
        include: health,info,prometheus,loggers
        exclude: heapdump,env,threaddump,shutdown
      base-path: /actuator

  endpoint:
    health:
      # when_authorized: only show component details to authenticated users
      # never: hide component details entirely (use for public-facing services)
      # always: NEVER use in production — exposes DB versions, pool sizes, API keys
      show-details: when_authorized
      show-components: when_authorized
      probes:
        # Enables /actuator/health/liveness and /actuator/health/readiness
        # Required for Kubernetes probe integration
        enabled: true

  # Global tag applied to every metric this service emits.
  # Without this, metrics from different services collide in Prometheus.
  # This is the one line most teams forget. Do not skip it.
  metrics:
    tags:
      application: ${spring.application.name}
    export:
      prometheus:
        # Explicitly enable the Prometheus registry.
        # Redundant if micrometer-registry-prometheus is on the classpath,
        # but makes intent explicit and prevents confusion.
        enabled: true

  # Security: never expose stack traces via the error endpoint
  server:
    add-application-context-header: false

server:
  error:
    # Never expose stack traces to clients or monitoring scrapers
    include-stacktrace: never
    include-message: never

spring:
  application:
    # This name flows into management.metrics.tags.application above.
    # Set it explicitly — never rely on the default.
    name: order-service

package io.thecodeforge.monitoring;

import io.micrometer.core.annotation.Timed;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Timer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.*;

import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * io.thecodeforge: Custom Business Metrics for Order Processing.
 *
 * Two patterns shown here:
 * 1. @Timed on controller methods — zero-boilerplate method-level timing
 * 2. MeterRegistry injection — explicit control for business counters and gauges
 */
@Service
public class OrderMetricsService {

    private final Counter ordersPlaced;
    private final Counter ordersFailed;
    private final Timer paymentLatency;
    private final AtomicInteger activeCarts = new AtomicInteger(0);

    public OrderMetricsService(MeterRegistry registry) {
        // Counter: Total orders placed (monotonically increasing)
        // Query in Prometheus: rate(orders_placed_total[5m]) → orders per second
        this.ordersPlaced = Counter.builder("orders.placed.total")
            .description("Total number of orders successfully placed")
            .tag("service", "order-service")
            .register(registry);

        // Counter: Total orders failed
        // Alert when rate(orders_failed_total[5m]) / rate(orders_placed_total[5m]) > 0.01
        this.ordersFailed = Counter.builder("orders.failed.total")
            .description("Total number of orders that failed processing")
            .tag("service", "order-service")
            .register(registry);

        // Timer: Payment processing latency with percentiles
        // publishPercentiles exposes p50, p95, p99 as separate Prometheus labels
        // Alert on p99 exceeding your SLA threshold — the mean will hide tail latency
        this.paymentLatency = Timer.builder("payment.processing.duration")
            .description("Time taken to process payment end to end")
            .tag("service", "order-service")
            .publishPercentiles(0.5, 0.95, 0.99)
            .register(registry);

        // Gauge: Active shopping carts — value goes up and down
        // Prometheus samples this on every scrape — you report the current value
        Gauge.builder("carts.active.current", activeCarts, AtomicInteger::get)
            .description("Number of active shopping carts in session")
            .register(registry);
    }

    public void recordSuccessfulOrder(long paymentDurationNanos) {
        ordersPlaced.increment();
        paymentLatency.record(paymentDurationNanos, TimeUnit.NANOSECONDS);
    }

    public void recordFailedOrder() {
        ordersFailed.increment();
    }

    public void cartOpened() { activeCarts.incrementAndGet(); }
    public void cartClosed() { activeCarts.decrementAndGet(); }
}

// --- @Timed annotation pattern: zero-boilerplate controller instrumentation ---

package io.thecodeforge.controller;

import io.micrometer.core.annotation.Timed;
import io.thecodeforge.dto.OrderDto;
import org.springframework.web.bind.annotation.*;

@RestController
@RequestMapping("/api/orders")
public class OrderController {

    /**
     * @Timed auto-instruments this method with a Timer named 'order.create.duration'.
     * Tracks: request count, total time, and percentiles.
     * Requires TimedAspect bean — auto-configured when spring-boot-starter-actuator
     * and micrometer-registry-prometheus are on the classpath.
     * extraTags: adds fixed labels to the metric for filtering in Grafana.
     */
    @Timed(
        value = "order.create.duration",
        description = "Time taken to create an order",
        extraTags = {"endpoint", "/api/orders", "method", "POST"},
        percentiles = {0.5, 0.95, 0.99},
        histogram = true
    )
    @PostMapping
    public OrderDto createOrder(@RequestBody OrderDto dto) {
        // Method execution is automatically timed.
        // A Timer named order_create_duration_seconds is created in Prometheus.
        return dto;
    }
}

package io.thecodeforge.monitoring;

import org.springframework.boot.actuate.endpoint.annotation.*;
import org.springframework.stereotype.Component;
import java.time.Instant;
import java.util.Map;

/**
 * io.thecodeforge: Custom Actuator Endpoint for deployment metadata and cache control.
 *
 * Exposed at: GET  /actuator/deployment  → returns deployment info
 *             POST /actuator/deployment  → triggers cache invalidation (ADMIN only)
 *
 * Spring Boot automatically applies your SecurityFilterChain to this endpoint.
 * Secure @WriteOperation methods with ADMIN role — they modify production state.
 */
@Component
@Endpoint(id = "deployment")
public class DeploymentEndpoint {

    private final String gitCommit;
    private final String buildVersion;
    private volatile Instant lastCacheInvalidation = null;
    private volatile String lastInvalidationReason = "none";

    public DeploymentEndpoint(
            @org.springframework.beans.factory.annotation.Value("${git.commit.id.abbrev:unknown}") String gitCommit,
            @org.springframework.beans.factory.annotation.Value("${build.version:unknown}") String buildVersion) {
        this.gitCommit = gitCommit;
        this.buildVersion = buildVersion;
    }

    /**
     * @ReadOperation → HTTP GET /actuator/deployment
     * Returns deployment metadata and cache state.
     * Safe to expose to MONITORING role — no side effects.
     */
    @ReadOperation
    public Map<String, Object> deploymentInfo() {
        return Map.of(
            "gitCommit", gitCommit,
            "buildVersion", buildVersion,
            "startedAt", Instant.now(),
            "lastCacheInvalidation", lastCacheInvalidation != null ? lastCacheInvalidation.toString() : "never",
            "lastInvalidationReason", lastInvalidationReason
        );
    }

    /**
     * @WriteOperation → HTTP POST /actuator/deployment
     * Triggers a cache invalidation without a redeploy.
     * Secure this with ADMIN role in your SecurityFilterChain.
     * Body parameter: { "reason": "stale pricing data after DB migration" }
     */
    @WriteOperation
    public Map<String, String> invalidateCache(
            @org.springframework.boot.actuate.endpoint.annotation.Selector String reason) {
        this.lastCacheInvalidation = Instant.now();
        this.lastInvalidationReason = reason != null ? reason : "manual trigger";
        // In real code: inject CacheManager and call cache.invalidateAll()
        return Map.of(
            "status", "cache invalidated",
            "reason", lastInvalidationReason,
            "timestamp", lastCacheInvalidation.toString()
        );
    }
}

package io.thecodeforge.monitoring;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.annotation.Order;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.web.SecurityFilterChain;

/**
 * io.thecodeforge: Actuator Security Configuration.
 *
 * @Order(1) gives this chain higher priority than your application's main chain.
 * Without it, your application's SecurityFilterChain may apply different rules
 * to /actuator/** than you intend.
 *
 * Role mapping:
 *   No role   → health, info (load balancers and Kubernetes probes)
 *   MONITORING → prometheus (Prometheus scraper service account)
 *   ADMIN      → loggers, env, heapdump, threaddump (ops team only)
 *   DENY       → shutdown, everything else not whitelisted
 */
@Configuration
public class ActuatorSecurityConfig {

    @Bean
    @Order(1)
    public SecurityFilterChain actuatorSecurityFilterChain(HttpSecurity http) throws Exception {
        http
            .securityMatcher("/actuator/**")
            .authorizeHttpRequests(auth -> auth
                // Public: health checks for load balancers and Kubernetes probes
                // These must be accessible without authentication
                .requestMatchers("/actuator/health/**").permitAll()
                .requestMatchers("/actuator/info").permitAll()

                // MONITORING role: Prometheus scraper authenticates via HTTP Basic
                .requestMatchers("/actuator/prometheus").hasRole("MONITORING")

                // ADMIN role: endpoints that expose or modify sensitive state
                // Log level changes can generate gigabytes/min if abused
                .requestMatchers(
                    "/actuator/loggers/**",
                    "/actuator/env",
                    "/actuator/heapdump",
                    "/actuator/threaddump"
                ).hasRole("ADMIN")

                // Deny everything not explicitly permitted above.
                // This includes /actuator/shutdown — never expose it.
                .anyRequest().denyAll()
            )
            // HTTP Basic: Prometheus natively supports basic_auth in scrape config
            .httpBasic(basic -> {})
            // Disable CSRF: Actuator endpoints are called by automated systems, not browsers
            .csrf(csrf -> csrf.disable());

        return http.build();
    }
}

# io.thecodeforge: Prometheus Scrape Configuration
# This file tells Prometheus where to find your Spring Boot metrics.

global:
  scrape_interval: 15s      # How often Prometheus scrapes all targets
  evaluation_interval: 15s  # How often alerting rules are evaluated

scrape_configs:

  # Static targets: works for a fixed number of servers or local development.
  # In production Kubernetes, replace this with kubernetes_sd_configs below.
  - job_name: 'spring-boot-order-service'
    # IMPORTANT: Use /actuator/prometheus, NOT /actuator/health.
    # /actuator/prometheus: lightweight metric reads, sub-millisecond.
    # /actuator/health: hits the DB, external APIs, disk — thundering herd at scale.
    metrics_path: '/actuator/prometheus'
    scrape_interval: 10s  # Override global for this job
    basic_auth:
      username: 'prometheus'
      # Use password_file in production — never inline passwords in prometheus.yml
      password_file: '/etc/prometheus/secrets/prometheus_password'
    static_configs:
      - targets: ['order-service-01:8080', 'order-service-02:8080']
        labels:
          environment: 'production'
          team: 'platform'

  # Kubernetes service discovery: use this instead of static_configs in K8s.
  # Pods annotated with prometheus.io/scrape: 'true' are scraped automatically.
  # No manual target management — works with rolling deploys and autoscaling.
  - job_name: 'spring-boot-k8s'
    metrics_path: '/actuator/prometheus'
    kubernetes_sd_configs:
      - role: pod
    basic_auth:
      username: 'prometheus'
      password_file: '/etc/prometheus/secrets/prometheus_password'
    relabel_configs:
      # Only scrape pods with prometheus.io/scrape: 'true' annotation
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: 'true'
      # Use the pod's prometheus.io/port annotation as the scrape port
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)
        replacement: '${1}'
      # Add pod name and namespace as labels for Grafana filtering
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace

# io.thecodeforge: Kubernetes Deployment with Actuator Probes
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  replicas: 3
  template:
    metadata:
      annotations:
        # Enable Prometheus auto-discovery (works with kubernetes_sd_configs above)
        prometheus.io/scrape: 'true'
        prometheus.io/port: '8080'
        prometheus.io/path: '/actuator/prometheus'
    spec:
      containers:
        - name: order-service
          image: io.thecodeforge/order-service:latest
          ports:
            - containerPort: 8080

          # STARTUP PROBE: Prevents liveness from killing a slow-starting app.
          # Math: failureThreshold (30) × periodSeconds (10) = 300 seconds max startup.
          # Measure your actual startup time and set this to at least 2× that value.
          # If your app starts in 45 seconds, use failureThreshold: 12, periodSeconds: 10
          # for a 120-second window. Never guess — measure.
          startupProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            failureThreshold: 30      # 30 × 10s = 300 seconds max startup window
            periodSeconds: 10
            timeoutSeconds: 3

          # LIVENESS PROBE: JVM-only — no external dependency checks.
          # If this fails, Kubernetes RESTARTS the pod.
          # Never check DB or external APIs here — a transient blip restarts all pods.
          livenessProbe:
            httpGet:
              path: /actuator/health/liveness
              port: 8080
            initialDelaySeconds: 0    # startupProbe handles the startup window
            periodSeconds: 10
            timeoutSeconds: 3
            failureThreshold: 3       # Restart after 3 consecutive failures (30 seconds)

          # READINESS PROBE: Checks dependencies — DB, cache, broker.
          # If this fails, pod is REMOVED from load balancer but NOT restarted.
          # This is the correct behavior for a transient dependency failure.
          readinessProbe:
            httpGet:
              path: /actuator/health/readiness
              port: 8080
            initialDelaySeconds: 0
            periodSeconds: 5
            timeoutSeconds: 3
            failureThreshold: 3       # Remove from LB after 15 seconds of failures

<!-- io.thecodeforge: Maven plugins for /actuator/info enrichment -->
<!-- Add these inside your <build><plugins> block -->
<build>
  <plugins>

    <!-- Plugin 1: Generates build-info.properties at compile time.
         Embeds: artifact name, version, build timestamp.
         Appears under "build" key in /actuator/info response. -->
    <plugin>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-maven-plugin</artifactId>
      <executions>
        <execution>
          <goals>
            <goal>build-info</goal>
          </goals>
        </execution>
      </executions>
    </plugin>

    <!-- Plugin 2: Embeds Git metadata at build time.
         Embeds: commit hash, branch, commit time, tags.
         Appears under "git" key in /actuator/info response.
         failOnNoGitDirectory=false: prevents build failure in CI environments
         where the .git directory may not be present (Docker build layers). -->
    <plugin>
      <groupId>pl.project13.maven</groupId>
      <artifactId>git-commit-id-plugin</artifactId>
      <version>4.9.10</version>
      <executions>
        <execution>
          <goals>
            <goal>revision</goal>
          </goals>
        </execution>
      </executions>
      <configuration>
        <generateGitPropertiesFile>true</generateGitPropertiesFile>
        <!-- Prevents build failure when .git is not present (shallow clones, CI) -->
        <failOnNoGitDirectory>false</failOnNoGitDirectory>
        <!-- Only embed the abbreviated commit hash, not the full 40-char hash -->
        <abbrevLength>7</abbrevLength>
      </configuration>
    </plugin>

  </plugins>
</build>

#!/bin/bash
# io.thecodeforge: Dynamic Log Level Management
# Run these during incident response — no restart, no redeploy required.

# --- Read current log level for a package ---
curl -s -u admin:${ACTUATOR_PASSWORD} \
  http://localhost:8080/actuator/loggers/io.thecodeforge.order | jq .
# Response: { "configuredLevel": "INFO", "effectiveLevel": "INFO" }

# --- Enable DEBUG for order package (troubleshooting) ---
curl -s -u admin:${ACTUATOR_PASSWORD} \
  -X POST \
  -H 'Content-Type: application/json' \
  -d '{"configuredLevel": "DEBUG"}' \
  http://localhost:8080/actuator/loggers/io.thecodeforge.order

# --- Enable TRACE for Hibernate SQL (see exact SQL + bind parameters) ---
# WARNING: This generates enormous log volume. Reset immediately after capturing the query.
curl -s -u admin:${ACTUATOR_PASSWORD} \
  -X POST \
  -H 'Content-Type: application/json' \
  -d '{"configuredLevel": "TRACE"}' \
  http://localhost:8080/actuator/loggers/org.hibernate.SQL

curl -s -u admin:${ACTUATOR_PASSWORD} \
  -X POST \
  -H 'Content-Type: application/json' \
  -d '{"configuredLevel": "TRACE"}' \
  http://localhost:8080/actuator/loggers/org.hibernate.type.descriptor.sql

# --- ALWAYS reset after capturing logs (mandatory step in your runbook) ---
curl -s -u admin:${ACTUATOR_PASSWORD} \
  -X POST \
  -H 'Content-Type: application/json' \
  -d '{"configuredLevel": null}' \
  http://localhost:8080/actuator/loggers/io.thecodeforge.order

# null = inherit from parent logger (returns to default)
# Setting to null is different from setting to INFO explicitly —
# null respects future configuration changes, INFO overrides them.

# io.thecodeforge: Multi-stage Dockerfile with Actuator Health Integration

# Stage 1: Build
FROM eclipse-temurin:17-jdk-alpine AS build
WORKDIR /app
COPY . .
RUN ./mvnw clean package -DskipTests

# Stage 2: Runtime
# eclipse-temurin:17-jre-alpine includes wget — required for HEALTHCHECK
# Do NOT use distroless if you need Docker HEALTHCHECK
FROM eclipse-temurin:17-jre-alpine
WORKDIR /app
COPY --from=build /app/target/*.jar app.jar

# Security: run as non-root user
RUN addgroup -S forgegroup && adduser -S forgeuser -G forgegroup
USER forgeuser

# Docker HEALTHCHECK: uses liveness endpoint for lightweight JVM-only check.
# interval: how often Docker checks (30s is conservative — adjust for your SLA)
# timeout: how long Docker waits for a response before marking unhealthy
# retries: consecutive failures before marking UNHEALTHY
# Uses /actuator/health/liveness — NOT /actuator/health
# Full health check would hit DB on every HEALTHCHECK — unnecessary for Docker daemon
HEALTHCHECK \
  --interval=30s \
  --timeout=3s \
  --retries=3 \
  CMD wget --quiet --tries=1 --spider http://localhost:8080/actuator/health/liveness || exit 1

EXPOSE 8080

# JVM flags for container environments:
# -XX:+UseContainerSupport: read container memory limits, not host memory
# -XX:MaxRAMPercentage=75.0: allocate 75% of container memory to the heap
#   leaving 25% for Metaspace, thread stacks, GC overhead, and OS
ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-XX:MaxRAMPercentage=75.0", "-jar", "app.jar"]