Distributed Locking Service: Build a Production-Grade Lock That Won't Fail at 3 AM

// io.thecodeforge — System Design tutorial

// Scenario: GC pause causes lock expiry
lock = redis.setnx('resource_lock', instance_id, ttl=30)
// ... 45 seconds of GC ...
// Lock expired, another instance acquired it
// Now two instances think they hold the lock
// Data corruption follows

// Fix: Use a watchdog thread
watchdog = new Thread(() -> {
    while (holdingLock) {
        redis.expire('resource_lock', 30);
        Thread.sleep(10_000);
    }
});
watchdog.start();

// io.thecodeforge — System Design tutorial

// Redlock implementation (simplified)
public class Redlock {
    private static final int QUORUM = 3; // out of 5 nodes
    private static final int TTL = 1000; // ms

    public boolean acquire(String resource, String instanceId) {
        int acquired = 0;
        long start = System.currentTimeMillis();
        for (RedisNode node : nodes) {
            if (node.setnx(resource, instanceId, TTL)) {
                acquired++;
            }
        }
        // Check if we got majority AND didn't take too long
        long elapsed = System.currentTimeMillis() - start;
        return acquired >= QUORUM && elapsed < TTL;
    }

    public void release(String resource, String instanceId) {
        // Lua script to ensure we only delete our own lock
        String script = "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end";
        for (RedisNode node : nodes) {
            node.eval(script, 1, resource, instanceId);
        }
    }
}

// io.thecodeforge — System Design tutorial

// ZooKeeper distributed lock using ephemeral sequential nodes
public class ZkLock {
    private ZooKeeper zk;
    private String lockPath = "/locks/resource";
    private String currentNode;

    public boolean acquire() throws Exception {
        // Create an ephemeral sequential node
        currentNode = zk.create(lockPath + "/lock-", new byte[0],
                ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
        
        // Get all children and check if we are the smallest
        List<String> children = zk.getChildren(lockPath, false);
        Collections.sort(children);
        String smallest = children.get(0);
        if (currentNode.endsWith(smallest)) {
            return true; // We hold the lock
        }
        
        // Watch the node just before us
        int myIndex = children.indexOf(currentNode.substring(lockPath.length()+1));
        String predecessor = lockPath + "/" + children.get(myIndex - 1);
        CountDownLatch latch = new CountDownLatch(1);
        zk.exists(predecessor, event -> latch.countDown());
        latch.await(); // Wait for predecessor to disappear
        return true;
    }

    public void release() throws Exception {
        zk.delete(currentNode, -1);
    }
}

// io.thecodeforge — System Design tutorial

// Fencing token generation in lock service
public class FencingTokenLock {
    private AtomicLong counter = new AtomicLong(0);
    
    public LockAcquireResult acquire(String resource, String clientId) {
        // ... acquire lock logic ...
        long token = counter.incrementAndGet();
        return new LockAcquireResult(true, token);
    }
}

// Client uses token in write
public void writeData(Data data, long fencingToken) {
    // Database checks: token must be > last seen token
    String sql = "UPDATE resource SET data = ?, token = ? WHERE token < ?";
    int rows = jdbc.update(sql, data, fencingToken, fencingToken);
    if (rows == 0) {
        throw new StaleLockException("Fencing token rejected");
    }
}

// io.thecodeforge — System Design tutorial

// Reentrant distributed lock using Redis
public class ReentrantRedisLock {
    private ThreadLocal<Integer> reentrantCount = ThreadLocal.withInitial(() -> 0);
    private String lockKey;
    private String instanceId;

    public boolean acquire() {
        if (reentrantCount.get() > 0) {
            reentrantCount.set(reentrantCount.get() + 1);
            return true;
        }
        boolean acquired = redis.setnx(lockKey, instanceId, 30);
        if (acquired) {
            reentrantCount.set(1);
        }
        return acquired;
    }

    public void release() {
        int count = reentrantCount.get();
        if (count == 0) throw new IllegalStateException("Not locked");
        if (count > 1) {
            reentrantCount.set(count - 1);
            return;
        }
        redis.eval("if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
                Collections.singletonList(lockKey), Collections.singletonList(instanceId));
        reentrantCount.set(0);
    }
}

// io.thecodeforge — System Design tutorial

// Health check endpoint for lock service
@GET
@Path("/health")
public Response health() {
    // Check each backend node
    for (LockBackend backend : backends) {
        if (!backend.isHealthy()) {
            return Response.status(503)
                .entity("Backend " + backend.getName() + " is down")
                .build();
        }
    }
    // Check latency
    long start = System.nanoTime();
    boolean acquired = tryAcquire("health-check-lock", "health-check", 100);
    long latency = (System.nanoTime() - start) / 1_000_000;
    release("health-check-lock", "health-check");
    if (latency > 500) {
        return Response.status(503)
            .entity("Lock latency too high: " + latency + "ms")
            .build();
    }
    return Response.ok().build();
}

// io.thecodeforge — System Design tutorial

// Idempotency key pattern (alternative to locks)
public class PaymentService {
    public PaymentResult processPayment(String idempotencyKey, PaymentRequest request) {
        // Check if already processed
        PaymentResult existing = cache.get(idempotencyKey);
        if (existing != null) return existing;
        
        // Use database unique constraint to prevent duplicates
        try {
            PaymentResult result = paymentGateway.charge(request);
            cache.set(idempotencyKey, result);
            return result;
        } catch (DuplicateKeyException e) {
            // Another request already processed this key
            return cache.get(idempotencyKey);
        }
    }
}