Multi-Level Caching: Stop Thundering Herds and Slash Latency in Production

// io.thecodeforge — System Design tutorial

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

public class L1CacheWithMutex {
    // L1 cache: fast, small, short TTL
    private final Cache<String, String> l1 = Caffeine.newBuilder()
            .maximumSize(10_000)
            .expireAfterWrite(5, TimeUnit.SECONDS)
            .build();

    // L2 client (e.g., Redis)
    private final RedisClient l2 = new RedisClient();

    // Mutex per key to prevent thundering herd
    private final ReentrantLock lock = new ReentrantLock();

    public String get(String key) {
        String value = l1.getIfPresent(key);
        if (value != null) return value;

        // Only one thread rebuilds per key
        lock.lock();
        try {
            value = l1.getIfPresent(key); // double-check
            if (value != null) return value;

            value = l2.get(key);
            if (value == null) {
                value = loadFromDatabase(key);
                l2.set(key, value, 60, TimeUnit.SECONDS);
            }
            l1.put(key, value);
            return value;
        } finally {
            lock.unlock();
        }
    }

    private String loadFromDatabase(String key) {
        // Simulate DB call
        return "value_for_" + key;
    }
}

// io.thecodeforge — System Design tutorial

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.concurrent.TimeUnit;

public class InProcessL1 {
    // Per-instance cache: fast but not shared
    private final Cache<String, String> l1 = Caffeine.newBuilder()
            .maximumSize(5_000)
            .expireAfterWrite(2, TimeUnit.SECONDS)
            .recordStats()
            .build();

    public String get(String key) {
        String val = l1.getIfPresent(key);
        if (val != null) return val;
        // fallback to L2
        val = l2Get(key);
        if (val != null) l1.put(key, val);
        return val;
    }

    // Called by write path to invalidate L1 across all instances
    public void invalidate(String key) {
        l1.invalidate(key);
        // Also publish invalidation event to message bus for other instances
    }

    private String l2Get(String key) {
        // Redis get
        return null;
    }
}

// io.thecodeforge — System Design tutorial

import redis.clients.jedis.JedisPool;
import redis.clients.jedis.Jedis;

public class RedisL2Client {
    private final JedisPool pool = new JedisPool("localhost", 6379);

    public String get(String key) {
        try (Jedis jedis = pool.getResource()) {
            return jedis.get(key);
        }
    }

    public void set(String key, String value, int ttlSeconds) {
        try (Jedis jedis = pool.getResource()) {
            jedis.setex(key, ttlSeconds, value);
        }
    }

    // Use for write-through: set in L2 and invalidate L1
    public void setAndInvalidateL1(String key, String value, int ttlSeconds) {
        set(key, value, ttlSeconds);
        // Publish invalidation to L1 instances via Redis Pub/Sub
        try (Jedis jedis = pool.getResource()) {
            jedis.publish("cache-invalidation", key);
        }
    }
}

// io.thecodeforge — System Design tutorial

public class CacheAsidePattern {
    private final L1Cache l1 = new L1Cache();
    private final RedisL2Client l2 = new RedisL2Client();
    private final Database db = new Database();

    public String read(String key) {
        // Check L1 first
        String val = l1.get(key);
        if (val != null) return val;

        // Check L2
        val = l2.get(key);
        if (val != null) {
            l1.put(key, val);
            return val;
        }

        // Load from DB
        val = db.get(key);
        if (val != null) {
            l2.set(key, val, 60);
            l1.put(key, val);
        }
        return val;
    }

    public void write(String key, String value) {
        // Write to DB first
        db.put(key, value);
        // Invalidate both caches
        l1.invalidate(key);
        l2.delete(key);
    }
}

// io.thecodeforge — System Design tutorial

import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
import java.util.concurrent.TimeUnit;

public class RefreshAheadCache {
    private final LoadingCache<String, String> cache = Caffeine.newBuilder()
            .maximumSize(10_000)
            .expireAfterWrite(10, TimeUnit.SECONDS)
            .refreshAfterWrite(8, TimeUnit.SECONDS) // refresh before expiry
            .build(key -> loadFromL2OrDB(key));

    public String get(String key) {
        return cache.get(key);
    }

    private String loadFromL2OrDB(String key) {
        // Check L2, then DB
        return "value";
    }
}