BackgroundService Silent Failures — Why Your Worker Stopped

using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Threading;
using System.Threading.Tasks;

/// <summary>
/// Warms a local in-memory cache before the HTTP server accepts any traffic.
/// Because StartAsync is awaited by the host before Kestrel starts, requests
/// will never see a cold cache state.
/// </summary>
public sealed class CacheWarmingService : IHostedService
{
    private readonly IProductCacheService _productCache;
    private readonly ILogger<CacheWarmingService> _logger;

    public CacheWarmingService(
        IProductCacheService productCache,
        ILogger<CacheWarmingService> logger)
    {
        _productCache = productCache;
        _logger = logger;
    }

    // Called by the host before the HTTP server starts.
    // We AWAIT the cache warm here intentionally — we want it complete before traffic arrives.
    public async Task StartAsync(CancellationToken cancellationToken)
    {
        _logger.LogInformation("[CacheWarmingService] Warming product cache before accepting traffic...");

        // Pass cancellationToken down so we can abort if the host is shutting down
        // before we even finish starting (e.g., rapid Ctrl+C during startup).
        await _productCache.WarmAsync(cancellationToken);

        _logger.LogInformation("[CacheWarmingService] Cache warm complete. Ready for traffic.");
    }

    // Called by the host when shutdown is signalled.
    // Nothing to clean up here — the cache service handles its own disposal.
    public Task StopAsync(CancellationToken cancellationToken)
    {
        _logger.LogInformation("[CacheWarmingService] Stopping — no cleanup required.");
        return Task.CompletedTask;
    }
}

// --- Registration in Program.cs ---
// builder.Services.AddHostedService<CacheWarmingService>();
// builder.Services.AddSingleton<IProductCacheService, ProductCacheService>();

using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Threading;
using System.Threading.Tasks;

/// <summary>
/// Polls an order queue every 5 seconds and processes any pending orders.
/// Demonstrates the correct BackgroundService pattern including:
///   - Scoped service resolution inside a singleton worker
///   - Cancellation token propagation
///   - Exception handling that keeps the loop alive
/// </summary>
public sealed class OrderProcessingWorker : BackgroundService
{
    // We inject IServiceScopeFactory — NOT IOrderRepository directly.
    // BackgroundService is registered as a singleton, but IOrderRepository
    // is likely scoped. Injecting a scoped service into a singleton causes
    // the 'captured dependency' bug. IServiceScopeFactory is always safe.
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly ILogger<OrderProcessingWorker> _logger;
    private static readonly TimeSpan PollingInterval = TimeSpan.FromSeconds(5);

    public OrderProcessingWorker(
        IServiceScopeFactory scopeFactory,
        ILogger<OrderProcessingWorker> logger)
    {
        _scopeFactory = scopeFactory;
        _logger = logger;
    }

    // ExecuteAsync is called once by BackgroundService.StartAsync on a background Task.
    // It runs until stoppingToken is cancelled (host shutdown) or an exception escapes.
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("[OrderProcessingWorker] Worker started.");

        // Loop runs until the host signals shutdown via stoppingToken.
        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                await ProcessPendingOrdersAsync(stoppingToken);
            }
            catch (OperationCanceledException)
            {
                // This is normal — stoppingToken was cancelled during an await.
                // Break the loop cleanly rather than logging a spurious error.
                _logger.LogInformation("[OrderProcessingWorker] Shutdown requested during processing.");
                break;
            }
            catch (Exception ex)
            {
                // Log the error but DON'T rethrow — rethrowing kills the hosted service.
                // Instead, we pause briefly and retry on the next iteration.
                // In production you'd also want alerting here (Sentry, Application Insights, etc.).
                _logger.LogError(ex, "[OrderProcessingWorker] Unhandled exception in processing loop. Retrying in {Interval}s.", PollingInterval.TotalSeconds);
            }

            // Task.Delay observes the cancellation token — if shutdown happens during
            // the delay, it throws OperationCanceledException immediately rather
            // than waiting out the full interval. This is what makes shutdown fast.
            await Task.Delay(PollingInterval, stoppingToken);
        }

        _logger.LogInformation("[OrderProcessingWorker] Worker stopped cleanly.");
    }

    private async Task ProcessPendingOrdersAsync(CancellationToken cancellationToken)
    {
        // Create a fresh DI scope per iteration — this gives us a fresh DbContext,
        // fresh unit-of-work, etc. Scope is disposed at end of using block.
        await using var scope = _scopeFactory.CreateAsyncScope();

        var orderRepository = scope.ServiceProvider.GetRequiredService<IOrderRepository>();
        var orderNotifier = scope.ServiceProvider.GetRequiredService<IOrderNotifier>();

        var pendingOrders = await orderRepository.GetPendingOrdersAsync(cancellationToken);

        if (pendingOrders.Count == 0)
        {
            _logger.LogDebug("[OrderProcessingWorker] No pending orders found.");
            return;
        }

        _logger.LogInformation("[OrderProcessingWorker] Processing {Count} pending orders.", pendingOrders.Count);

        foreach (var order in pendingOrders)
        {
            // Pass cancellationToken to every async call so we can abort mid-batch on shutdown.
            await orderRepository.MarkAsProcessingAsync(order.Id, cancellationToken);
            await orderNotifier.SendConfirmationAsync(order, cancellationToken);
            await orderRepository.MarkAsCompleteAsync(order.Id, cancellationToken);

            _logger.LogInformation("[OrderProcessingWorker] Order {OrderId} processed successfully.", order.Id);
        }
    }
}

// --- Registration in Program.cs ---
// builder.Services.AddHostedService<OrderProcessingWorker>();

using System.Threading.Channels;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.DependencyInjection;
using System;

/// <summary>
/// Represents a single email dispatch request written by HTTP handlers
/// and consumed by the background worker.
/// </summary>
public sealed record EmailDispatchRequest(
    string RecipientAddress,
    string Subject,
    string HtmlBody,
    DateTimeOffset EnqueuedAt);

/// <summary>
/// Singleton channel that acts as the in-process message bus between
/// HTTP request handlers (producers) and the email worker (consumer).
/// Registered as a singleton so both producers and the worker share the same instance.
/// </summary>
public sealed class EmailDispatchChannel
{
    // BoundedCapacity of 500 means the channel holds at most 500 pending emails.
    // If the worker falls behind, WriteAsync on producers will apply backpressure
    // (await until space is available) rather than silently dropping messages.
    private readonly Channel<EmailDispatchRequest> _channel = Channel.CreateBounded<EmailDispatchRequest>(
        new BoundedChannelOptions(capacity: 500)
        {
            FullMode = BoundedChannelFullMode.Wait,       // Block producers rather than drop
            SingleReader = true,                           // Only the worker reads — allows optimizations
            SingleWriter = false                           // Many HTTP request threads can write
        });

    public ChannelWriter<EmailDispatchRequest> Writer => _channel.Writer;
    public ChannelReader<EmailDispatchRequest> Reader => _channel.Reader;
}

/// <summary>
/// Background worker that consumes email dispatch requests from the channel.
/// Uses ChannelReader.ReadAllAsync which is the cleanest cancellation-aware
/// consume pattern — it stops iteration automatically when the channel is
/// completed OR the cancellation token is fired.
/// </summary>
public sealed class EmailDispatchWorker : BackgroundService
{
    private readonly EmailDispatchChannel _emailChannel;
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly IHostApplicationLifetime _appLifetime;
    private readonly ILogger<EmailDispatchWorker> _logger;

    public EmailDispatchWorker(
        EmailDispatchChannel emailChannel,
        IServiceScopeFactory scopeFactory,
        IHostApplicationLifetime appLifetime,
        ILogger<EmailDispatchWorker> logger)
    {
        _emailChannel = emailChannel;
        _scopeFactory = scopeFactory;
        _appLifetime = appLifetime;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("[EmailDispatchWorker] Starting — listening for dispatch requests.");

        // ReadAllAsync yields each item as it arrives, blocking asynchronously
        // when the channel is empty. When stoppingToken fires, the IAsyncEnumerable
        // stops yielding and the loop exits cleanly.
        await foreach (var request in _emailChannel.Reader.ReadAllAsync(stoppingToken))
        {
            var lag = DateTimeOffset.UtcNow - request.EnqueuedAt;

            // Alert if emails are sitting in the queue for more than 30 seconds.
            if (lag > TimeSpan.FromSeconds(30))
            {
                _logger.LogWarning(
                    "[EmailDispatchWorker] High lag detected: {Lag:F1}s for email to {Recipient}.",
                    lag.TotalSeconds, request.RecipientAddress);
            }

            try
            {
                await DispatchEmailAsync(request, stoppingToken);

                _logger.LogInformation(
                    "[EmailDispatchWorker] Email dispatched to {Recipient} (lag: {Lag:F1}s).",
                    request.RecipientAddress, lag.TotalSeconds);
            }
            catch (OperationCanceledException)
            {
                // Shutdown during dispatch — requeue or accept the loss depending on your SLA.
                _logger.LogWarning(
                    "[EmailDispatchWorker] Cancelled mid-dispatch for {Recipient}. Item may be lost.",
                    request.RecipientAddress);
                break;
            }
            catch (Exception ex)
            {
                _logger.LogError(ex,
                    "[EmailDispatchWorker] Failed to dispatch email to {Recipient}. Continuing with next item.",
                    request.RecipientAddress);

                // For truly critical workers: call _appLifetime.StopApplication() here
                // to crash the pod intentionally so Kubernetes restarts it.
                // Safer than silently skipping and accumulating failures.
            }
        }

        _logger.LogInformation("[EmailDispatchWorker] Stopped.");
    }

    private async Task DispatchEmailAsync(EmailDispatchRequest request, CancellationToken cancellationToken)
    {
        await using var scope = _scopeFactory.CreateAsyncScope();
        var emailSender = scope.ServiceProvider.GetRequiredService<IEmailSender>();
        await emailSender.SendAsync(request.RecipientAddress, request.Subject, request.HtmlBody, cancellationToken);
    }
}

// --- Registration in Program.cs ---
// builder.Services.AddSingleton<EmailDispatchChannel>();
// builder.Services.AddHostedService<EmailDispatchWorker>();
//
// --- Usage in a Controller or Minimal API endpoint ---
// await emailChannel.Writer.WriteAsync(new EmailDispatchRequest(
//     RecipientAddress: "user@example.com",
//     Subject: "Your order is confirmed",
//     HtmlBody: "<h1>Thanks for your order!</h1>",
//     EnqueuedAt: DateTimeOffset.UtcNow), cancellationToken);

using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.DependencyInjection;
using System;

var builder = WebApplication.CreateBuilder(args);

// --- Configure shutdown timeout to 30 seconds ---
// Default is 5 seconds — almost always too short for real workers.
// Match this value to your Kubernetes terminationGracePeriodSeconds minus a 5s buffer.
builder.Services.Configure<HostOptions>(options =>
{
    options.ShutdownTimeout = TimeSpan.FromSeconds(30);
});

// --- Register infrastructure services ---
builder.Services.AddSingleton<IProductCacheService, ProductCacheService>();
builder.Services.AddScoped<IOrderRepository, OrderRepository>();
builder.Services.AddScoped<IOrderNotifier, OrderNotifier>();
builder.Services.AddScoped<IEmailSender, SmtpEmailSender>();
builder.Services.AddSingleton<EmailDispatchChannel>();

// --- Register hosted services in dependency order ---
// CacheWarmingService runs first (StartAsync blocks until cache is warm)
// before any worker that might query the cache.
builder.Services.AddHostedService<CacheWarmingService>();
builder.Services.AddHostedService<OrderProcessingWorker>();
builder.Services.AddHostedService<EmailDispatchWorker>();

// --- Add a hosted service that monitors other services and restarts them ---
// Pattern: inject IHostApplicationLifetime to self-heal or escalate crashes.
builder.Services.AddHostedService<WorkerHealthMonitor>();

builder.Services.AddControllers();

var app = builder.Build();

// Register a callback on ApplicationStopping to flush any in-flight telemetry
// before the process exits. This runs before StopAsync on any hosted service.
var lifetime = app.Services.GetRequiredService<IHostApplicationLifetime>();
lifetime.ApplicationStopping.Register(() =>
    app.Logger.LogInformation("[Host] Shutdown initiated — flushing telemetry..."));

app.MapControllers();

app.Run();

// --- WorkerHealthMonitor: a self-healing pattern ---
// Demonstrates using IHostApplicationLifetime to crash intentionally on unrecoverable failure.
public sealed class WorkerHealthMonitor : BackgroundService
{
    private readonly IHostApplicationLifetime _appLifetime;
    private readonly ILogger<WorkerHealthMonitor> _logger;

    public WorkerHealthMonitor(
        IHostApplicationLifetime appLifetime,
        ILogger<WorkerHealthMonitor> logger)
    {
        _appLifetime = appLifetime;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        // Listen for the application started event before beginning health checks.
        await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);

        while (!stoppingToken.IsCancellationRequested)
        {
            // In a real implementation, check metrics endpoints, event counters,
            // or a shared health flag set by other workers.
            bool systemHealthy = await CheckSystemHealthAsync(stoppingToken);

            if (!systemHealthy)
            {
                _logger.LogCritical(
                    "[WorkerHealthMonitor] Critical subsystem failure detected. Initiating controlled shutdown.");

                // StopApplication triggers graceful shutdown — all StopAsync methods run,
                // ShutdownTimeout is respected, and the process exits cleanly.
                // In Kubernetes this causes a pod restart — which is what we want.
                _appLifetime.StopApplication();
                return;
            }

            await Task.Delay(TimeSpan.FromSeconds(15), stoppingToken);
        }
    }

    private Task<bool> CheckSystemHealthAsync(CancellationToken cancellationToken)
    {
        // Placeholder — in production, query a health check endpoint or
        // check a shared failure counter from other workers.
        return Task.FromResult(true);
    }
}

using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Threading;
using System.Threading.Tasks;

// This is the default Worker.cs from the .NET Worker Service template.
// It's a pure BackgroundService — identical to what you'd write inside a web app.
// The only difference is no Kestrel, no HTTP middleware.
public sealed class Worker : BackgroundService
{
    private readonly ILogger<Worker> _logger;

    public Worker(ILogger<Worker> logger)
    {
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Worker running at: {Time}", DateTimeOffset.Now);

        while (!stoppingToken.IsCancellationRequested)
        {
            _logger.LogInformation("Worker doing work at: {Time}", DateTimeOffset.Now);
            await Task.Delay(1000, stoppingToken);
        }

        _logger.LogInformation("Worker stopped at: {Time}", DateTimeOffset.Now);
    }
}

// --- Program.cs for Worker Service ---
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;

var builder = Host.CreateDefaultBuilder(args);

builder.ConfigureServices((hostContext, services) =>
{
    // Register dependencies exactly as you would in a web app
    services.AddSingleton<IEmailSender, SmtpEmailSender>();
    services.AddHostedService<Worker>();
});

var host = builder.Build();
await host.RunAsync();