ASP.NET Core Health Checks: Liveness Probe Timeout Restarts

// Program.cs — Minimal API style (NET 6+)
// This is the absolute foundation. Every health check setup starts here.

var builder = WebApplication.CreateBuilder(args);

// Step 1: Register the health check services with the DI container.
// AddHealthChecks() returns an IHealthChecksBuilder you can chain onto.
builder.Services.AddHealthChecks()
    // Register a named check. The name appears in the JSON response
    // so ops teams know exactly WHICH check failed.
    .AddCheck("self", () => HealthCheckResult.Healthy("App is running"))
    
    // Tags let you group checks for different audiences.
    // 'live' = Kubernetes liveness probe (is the process alive?)
    // 'ready' = Kubernetes readiness probe (can it serve traffic?)
    .AddCheck(
        name: "startup-warmup",
        check: () => HealthCheckResult.Healthy("Warm-up complete"),
        tags: new[] { "live" }
    );

var app = builder.Build();

// Step 2: Map the health check endpoints.
// /healthz/live — only runs checks tagged 'live'
// /healthz/ready — only runs checks tagged 'ready'
// /healthz       — runs ALL checks (useful for ops dashboards)
app.MapHealthChecks("/healthz", new HealthCheckOptions
{
    // ResponseWriter controls what gets written to the HTTP response body.
    // WriteResponse is a static helper we define in the next section.
    ResponseWriter = HealthCheckResponseWriter.WriteResponse
});

app.MapHealthChecks("/healthz/live", new HealthCheckOptions
{
    // Predicate filters which checks run on this endpoint.
    // Here we only run checks tagged 'live'.
    Predicate = check => check.Tags.Contains("live"),
    ResponseWriter = HealthCheckResponseWriter.WriteResponse
});

app.MapHealthChecks("/healthz/ready", new HealthCheckOptions
{
    Predicate = check => check.Tags.Contains("ready"),
    ResponseWriter = HealthCheckResponseWriter.WriteResponse
});

app.Run();

// SqlServerHealthCheck.cs
// A production-grade database health check that validates the connection
// AND confirms the database can execute a real query — not just ping.

using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Data.SqlClient;

public class SqlServerHealthCheck : IHealthCheck
{
    private readonly string _connectionString;
    
    // Inject the connection string via DI rather than hard-coding it.
    // In production this comes from IConfiguration / environment variables.
    public SqlServerHealthCheck(IConfiguration configuration)
    {
        _connectionString = configuration.GetConnectionString("DefaultConnection")
            ?? throw new InvalidOperationException("DefaultConnection string is not configured.");
    }

    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        try
        {
            // Use a short timeout — health checks should fail fast.
            // 5 seconds is a reasonable maximum for a DB ping.
            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
            cts.CancelAfter(TimeSpan.FromSeconds(5));

            await using var connection = new SqlConnection(_connectionString);
            await connection.OpenAsync(cts.Token);

            // Run a trivial query — SELECT 1 confirms the DB engine is
            // accepting queries, not just that the TCP port is open.
            await using var command = connection.CreateCommand();
            command.CommandText = "SELECT 1";
            await command.ExecuteScalarAsync(cts.Token);

            // Include useful diagnostic data in the result.
            // This appears in the JSON response and in health check UI.
            var data = new Dictionary<string, object>
            {
                { "database", connection.Database },
                { "server", connection.DataSource }
            };

            return HealthCheckResult.Healthy(
                description: "SQL Server is reachable and accepting queries.",
                data: data
            );
        }
        catch (OperationCanceledException)
        {
            // Distinguish a timeout from a general failure —
            // timeouts and connection errors need different ops responses.
            return HealthCheckResult.Unhealthy(
                description: "SQL Server health check timed out after 5 seconds."
            );
        }
        catch (Exception ex)
        {
            // The exception message goes into the description field
            // so it shows up directly in your monitoring dashboard.
            return HealthCheckResult.Unhealthy(
                description: $"SQL Server check failed: {ex.Message}",
                exception: ex
            );
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// ExternalPaymentApiHealthCheck.cs
// Checks that a critical downstream HTTP dependency is healthy.
// Uses a named HttpClient registered via IHttpClientFactory — the correct
// pattern for health checks, which must not create HttpClient instances
// directly (causes socket exhaustion).

public class ExternalPaymentApiHealthCheck : IHealthCheck
{
    private readonly IHttpClientFactory _httpClientFactory;

    public ExternalPaymentApiHealthCheck(IHttpClientFactory httpClientFactory)
    {
        _httpClientFactory = httpClientFactory;
    }

    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        try
        {
            // Use the named client configured in Program.cs.
            // Named clients have pre-configured BaseAddress, timeout, etc.
            var httpClient = _httpClientFactory.CreateClient("PaymentApiClient");

            // Hit the payment API's own health endpoint rather than a
            // business endpoint — avoids triggering real business logic.
            var response = await httpClient.GetAsync("/health", cancellationToken);

            if (response.IsSuccessStatusCode)
            {
                return HealthCheckResult.Healthy(
                    description: $"Payment API responded with {(int)response.StatusCode}."
                );
            }

            // Degraded = the service is reachable but not fully healthy.
            // This is useful when a dependency is slow or partially down.
            return HealthCheckResult.Degraded(
                description: $"Payment API returned unexpected status: {(int)response.StatusCode}."
            );
        }
        catch (HttpRequestException ex)
        {
            return HealthCheckResult.Unhealthy(
                description: $"Cannot reach Payment API: {ex.Message}",
                exception: ex
            );
        }
        catch (TaskCanceledException)
        {
            return HealthCheckResult.Unhealthy(
                description: "Payment API health check timed out."
            );
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Program.cs — registering both custom checks

builder.Services.AddHttpClient("PaymentApiClient", client =>
{
    client.BaseAddress = new Uri("https://api.paymentprovider.com");
    // Set a tight timeout — do not rely on the default 100s HttpClient timeout.
    client.Timeout = TimeSpan.FromSeconds(8);
});

builder.Services.AddHealthChecks()
    .AddCheck<SqlServerHealthCheck>(
        name: "sql-server",
        failureStatus: HealthStatus.Unhealthy,  // a DB failure = fully unhealthy
        tags: new[] { "ready", "db" }
    )
    .AddCheck<ExternalPaymentApiHealthCheck>(
        name: "payment-api",
        failureStatus: HealthStatus.Degraded,   // payment API down = degraded, not dead
        tags: new[] { "ready", "external" }
    );

// HealthCheckResponseWriter.cs
// A reusable JSON response writer that returns rich diagnostic output.
// Reference this from every MapHealthChecks call.

using System.Text.Json;
using Microsoft.Extensions.Diagnostics.HealthChecks;

public static class HealthCheckResponseWriter
{
    public static Task WriteResponse(HttpContext context, HealthReport report)
    {
        // Always return JSON — never let this endpoint return HTML.
        context.Response.ContentType = "application/json; charset=utf-8";

        // Map each health check entry to a serialisable anonymous object.
        var responseBody = new
        {
            status = report.Status.ToString(),
            totalDuration = report.TotalDuration.ToString(),
            entries = report.Entries.ToDictionary(
                entry => entry.Key,   // check name e.g. "sql-server"
                entry => new
                {
                    status = entry.Value.Status.ToString(),
                    description = entry.Value.Description,
                    duration = entry.Value.Duration.ToString(),
                    // Serialise the exception message if one was captured.
                    // This is invaluable for on-call debugging.
                    exception = entry.Value.Exception?.Message,
                    data = entry.Value.Data
                }
            )
        };

        // Use camelCase to match the convention of JSON APIs everywhere.
        var jsonOptions = new JsonSerializerOptions
        {
            WriteIndented = true,
            PropertyNamingPolicy = JsonNamingPolicy.CamelCase
        };

        return context.Response.WriteAsync(
            JsonSerializer.Serialize(responseBody, jsonOptions)
        );
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Program.cs additions for Health Checks UI
// Install: dotnet add package AspNetCore.HealthChecks.UI
//          dotnet add package AspNetCore.HealthChecks.UI.Client
//          dotnet add package AspNetCore.HealthChecks.UI.InMemory.Storage

using HealthChecks.UI.Client;   // provides UIResponseWriter

var builder = WebApplication.CreateBuilder(args);

builder.Services.AddHealthChecks()
    .AddCheck<SqlServerHealthCheck>("sql-server", tags: new[] { "ready", "db" })
    .AddCheck<ExternalPaymentApiHealthCheck>("payment-api", tags: new[] { "ready", "external" });

// Register the UI services and configure in-memory storage for check history.
builder.Services
    .AddHealthChecksUI(settings =>
    {
        // How often the UI polls the health endpoint (in seconds).
        settings.SetEvaluationTimeInSeconds(15);
        
        // Maximum number of history entries to retain per endpoint.
        settings.MaximumHistoryEntriesPerEndpoint(50);
        
        // Register the endpoint the UI will poll.
        // The name shows up as a label in the UI dashboard.
        settings.AddHealthCheckEndpoint(
            name: "Production App",
            uri: "/healthz"
        );
    })
    .AddInMemoryStorage();   // stores check history in-process (use SQL for multi-instance)

var app = builder.Build();

// The /healthz endpoint uses the UI client's response writer.
// UIResponseWriter.WriteHealthCheckUIResponse outputs the exact JSON format
// the UI dashboard expects — richer than our custom writer.
app.MapHealthChecks("/healthz", new HealthCheckOptions
{
    ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
});

// Serve the Health Checks UI dashboard at /healthchecks-ui
// Restrict this to internal networks in production!
app.MapHealthChecksUI(options =>
{
    options.UIPath = "/healthchecks-ui";
    options.ApiPath = "/healthchecks-api";
});

app.Run();

// ─────────────────────────────────────────────────────────────────────────────
// appsettings.json — required for multi-service UI monitoring
// (When using AddHealthCheckEndpoint() in code, this section is optional
// but useful for environment-specific overrides via environment variables.)
/*
{
  "HealthChecksUI": {
    "HealthChecks": [
      {
        "Name": "Production App",
        "Uri": "https://myapp.internal/healthz"
      },
      {
        "Name": "Background Worker",
        "Uri": "https://worker.internal/healthz"
      }
    ],
    "EvaluationTimeInSeconds": 15,
    "MaximumHistoryEntriesPerEndpoint": 50
  }
}
*/

// Program.cs — Demonstrating ResultStatusCodes and FailureStatus configuration
// This is the production-ready pattern for a Kubernetes-hosted service.

var builder = WebApplication.CreateBuilder(args);

builder.Services.AddHealthChecks()
    // Critical dependency: DB down = service is Unhealthy
    .AddCheck<SqlServerHealthCheck>(
        name: "sql-server",
        failureStatus: HealthStatus.Unhealthy,
        tags: new[] { "ready" }
    )
    // Important but non-critical: cache down = service is Degraded
    // The service can still serve traffic without Redis, just slower.
    .AddCheck<RedisCacheHealthCheck>(
        name: "redis-cache",
        failureStatus: HealthStatus.Degraded,  // downgrade the severity
        tags: new[] { "ready" }
    )
    // External dependency: payment API down = Degraded (we can queue transactions)
    .AddCheck<ExternalPaymentApiHealthCheck>(
        name: "payment-api",
        failureStatus: HealthStatus.Degraded,
        tags: new[] { "ready" }
    );

var app = builder.Build();

// Liveness endpoint — only the self-check.
// A liveness failure triggers a pod RESTART. Keep this minimal.
// Do NOT include DB or external checks here — a slow DB causes
// restart loops, which makes an outage dramatically worse.
app.MapHealthChecks("/healthz/live", new HealthCheckOptions
{
    Predicate = _ => false,   // run NO registered checks — just return 200
    ResponseWriter = HealthCheckResponseWriter.WriteResponse
});

// Readiness endpoint — all 'ready' tagged checks.
// A readiness failure removes the pod from the load balancer.
app.MapHealthChecks("/healthz/ready", new HealthCheckOptions
{
    Predicate = check => check.Tags.Contains("ready"),
    ResponseWriter = HealthCheckResponseWriter.WriteResponse,

    // THE KEY CHANGE: map Degraded to 503 so K8s stops sending traffic
    // when any dependency is struggling, even if not fully failed.
    ResultStatusCodes =
    {
        [HealthStatus.Healthy]   = StatusCodes.Status200OK,
        [HealthStatus.Degraded]  = StatusCodes.Status503ServiceUnavailable,
        [HealthStatus.Unhealthy] = StatusCodes.Status503ServiceUnavailable
    }
});

// Full check endpoint — for ops dashboards and manual inspection.
// Returns 200 even when Degraded so the dashboard doesn't show false alarms.
app.MapHealthChecks("/healthz", new HealthCheckOptions
{
    ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
    // No ResultStatusCodes override — uses default (Degraded = 200)
});

app.Run();

// ─────────────────────────────────────────────────────────────────────────────
// RedisCacheHealthCheck.cs — a lightweight example showing Degraded usage

using StackExchange.Redis;

public class RedisCacheHealthCheck : IHealthCheck
{
    private readonly IConnectionMultiplexer _redis;

    public RedisCacheHealthCheck(IConnectionMultiplexer redis)
    {
        _redis = redis;
    }

    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        // IsConnected is synchronous — no await needed for a connection check.
        if (_redis.IsConnected)
        {
            return Task.FromResult(
                HealthCheckResult.Healthy("Redis is connected.")
            );
        }

        // Return Degraded here — the service can operate without cache,
        // but performance will degrade. Ops should know.
        // The FailureStatus on the registration (Degraded) means even
        // if this throws an exception, it reports as Degraded not Unhealthy.
        return Task.FromResult(
            HealthCheckResult.Degraded("Redis is not connected. Operating without cache.")
        );
    }
}

// BackgroundQueueProcessor.cs
// A BackgroundService that updates a shared health status object.

public class BackgroundQueueProcessor : BackgroundService
{
    private readonly ILogger<BackgroundQueueProcessor> _logger;
    private readonly WorkerHealthStatus _status;

    public BackgroundQueueProcessor(
        ILogger<BackgroundQueueProcessor> logger,
        WorkerHealthStatus status)  // registered as singleton
    {
        _logger = logger;
        _status = status;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Queue processor started.");
        _status.SetHealthy("Worker running, processing queue");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                // Simulate processing a batch of messages
                await Task.Delay(1000, stoppingToken);

                // Update health status with last run time
                _status.SetHealthy(
                    $"Queue processed at {DateTime.UtcNow:O}",
                    new Dictionary<string, object>
                    {
                        ["lastRun"] = DateTime.UtcNow,
                        ["processedCount"] = Interlocked.Increment(ref _processedCount)
                    }
                );
            }
            catch (OperationCanceledException)
            {
                // Graceful shutdown
                _status.SetDegraded("Worker stopping");
                break;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Queue processing failed");
                _status.SetUnhealthy($"Queue processing failed: {ex.Message}");
                // Optionally wait before retrying to avoid tight failure loops
                await Task.Delay(5000, stoppingToken);
            }
        }

        _status.SetDegraded("Worker stopped");
    }

    private long _processedCount;
}

// WorkerHealthStatus.cs
// Thread-safe health status holder for background workers.

public class WorkerHealthStatus
{
    private HealthStatus _status = HealthStatus.Unhealthy;
    private string _description = "Not started";
    private Dictionary<string, object> _data = new();
    private readonly object _lock = new();

    public void SetHealthy(string description, Dictionary<string, object> data = null)
    {
        lock (_lock)
        {
            _status = HealthStatus.Healthy;
            _description = description;
            _data = data ?? new Dictionary<string, object>();
        }
    }

    public void SetDegraded(string description)
    {
        lock (_lock)
        {
            _status = HealthStatus.Degraded;
            _description = description;
        }
    }

    public void SetUnhealthy(string description)
    {
        lock (_lock)
        {
            _status = HealthStatus.Unhealthy;
            _description = description;
        }
    }

    public HealthCheckResult GetResult()
    {
        lock (_lock)
        {
            return new HealthCheckResult(_status, _description, data: _data);
        }
    }
}

// BackgroundWorkerHealthCheck.cs
// IHealthCheck that reads from the shared status object.

public class BackgroundWorkerHealthCheck : IHealthCheck
{
    private readonly WorkerHealthStatus _status;

    public BackgroundWorkerHealthCheck(WorkerHealthStatus status)
    {
        _status = status;
    }

    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken cancellationToken = default)
    {
        // Just delegate to the shared status object — no async work needed.
        return Task.FromResult(_status.GetResult());
    }
}

// Program.cs — registration for worker health check

builder.Services.AddSingleton<WorkerHealthStatus>();
builder.Services.AddHostedService<BackgroundQueueProcessor>();

builder.Services.AddHealthChecks()
    .AddCheck<BackgroundWorkerHealthCheck>(
        name: "queue-worker",
        failureStatus: HealthStatus.Degraded,
        tags: new[] { "ready", "background" }
    );