Docker Daemon Socket Errors — Why Disk Full Crashes CI/CD

import os
import subprocess
import json
from dataclasses import dataclass
from typing import Optional, Dict, List
from enum import Enum


class DaemonStatus(Enum):
    RUNNING = "running"
    STOPPED = "stopped"
    FAILED = "failed"
    UNKNOWN = "unknown"
    NOT_INSTALLED = "not_installed"


class SocketStatus(Enum):
    EXISTS = "exists"
    MISSING = "missing"
    PERMISSION_DENIED = "permission_denied"
    WRONG_PATH = "wrong_path"


@dataclass
class DiagnosticResult:
    component: str
    status: str
    detail: str
    fix_command: Optional[str] = None
    severity: str = "info"


class DockerDaemonDiagnostics:
    """
    Diagnoses Docker daemon connection issues.
    """

    SOCKET_PATH = "/var/run/docker.sock"
    DAEMON_SERVICE = "docker"

    @staticmethod
    def check_daemon_installed() -> DiagnosticResult:
        """
        Check if Docker is installed.
        """
        try:
            result = subprocess.run(
                ["docker", "--version"],
                capture_output=True, text=True, timeout=5
            )
            if result.returncode == 0:
                return DiagnosticResult(
                    component="docker_installation",
                    status="installed",
                    detail=result.stdout.strip(),
                    severity="info",
                )
            return DiagnosticResult(
                component="docker_installation",
                status="error",
                detail="Docker command exists but returned an error",
                severity="warning",
            )
        except FileNotFoundError:
            return DiagnosticResult(
                component="docker_installation",
                status="not_installed",
                detail="Docker CLI not found in PATH",
                fix_command="Install Docker: https://docs.docker.com/engine/install/",
                severity="critical",
            )

    @staticmethod
    def check_daemon_running() -> DiagnosticResult:
        """
        Check if the Docker daemon process is running.
        """
        try:
            result = subprocess.run(
                ["systemctl", "is-active", "docker"],
                capture_output=True, text=True, timeout=5
            )
            status = result.stdout.strip()

            if status == "active":
                return DiagnosticResult(
                    component="daemon_process",
                    status="running",
                    detail="Docker daemon is active and running",
                    severity="info",
                )
            elif status == "inactive":
                return DiagnosticResult(
                    component="daemon_process",
                    status="stopped",
                    detail="Docker daemon is installed but not running",
                    fix_command="sudo systemctl start docker && sudo systemctl enable docker",
                    severity="critical",
                )
            elif status == "failed":
                return DiagnosticResult(
                    component="daemon_process",
                    status="failed",
                    detail="Docker daemon entered a failed state",
                    fix_command="sudo journalctl -u docker.service -n 50 --no-pager",
                    severity="critical",
                )
            else:
                return DiagnosticResult(
                    component="daemon_process",
                    status="unknown",
                    detail=f"Unexpected daemon status: {status}",
                    severity="warning",
                )
        except FileNotFoundError:
            return DiagnosticResult(
                component="daemon_process",
                status="unknown",
                detail="systemctl not found — possibly macOS or non-systemd Linux",
                severity="info",
            )

    @staticmethod
    def check_socket() -> DiagnosticResult:
        """
        Check if the Docker socket exists and is accessible.
        """
        socket_path = DockerDaemonDiagnostics.SOCKET_PATH

        if not os.path.exists(socket_path):
            return DiagnosticResult(
                component="socket_file",
                status="missing",
                detail=f"Socket file does not exist at {socket_path}",
                fix_command="sudo systemctl restart docker",
                severity="critical",
            )

        if not os.access(socket_path, os.R_OK | os.W_OK):
            stat_info = os.stat(socket_path)
            import grp
            try:
                group_name = grp.getgrgid(stat_info.st_gid).gr_name
            except KeyError:
                group_name = str(stat_info.st_gid)

            return DiagnosticResult(
                component="socket_permissions",
                status="permission_denied",
                detail=f"Socket exists but current user lacks read/write permissions. Socket group: {group_name}",
                fix_command=f"sudo usermod -aG {group_name} $USER && newgrp {group_name}",
                severity="critical",
            )

        return DiagnosticResult(
            component="socket_file",
            status="exists",
            detail=f"Socket file exists and is accessible at {socket_path}",
            severity="info",
        )

    @staticmethod
    def check_disk_space() -> DiagnosticResult:
        """
        Check disk space on the Docker data directory.
        """
        docker_root = "/var/lib/docker"

        if not os.path.exists(docker_root):
            return DiagnosticResult(
                component="disk_space",
                status="unknown",
                detail=f"Docker data directory not found at {docker_root}",
                severity="warning",
            )

        stat = os.statvfs(docker_root)
        free_gb = (stat.f_bavail * stat.f_frsize) / (1024 ** 3)
        total_gb = (stat.f_blocks * stat.f_frsize) / (1024 ** 3)
        used_pct = ((total_gb - free_gb) / total_gb) * 100

        if used_pct > 95:
            return DiagnosticResultcritical",
                detail=f"Disk usage at {used_pct:.1f}% — {free_gb:.1f}GB free of {total_gb:.1f}GB",
                fix_command="docker system prune -af --volumes",
                severity="critical",
            )
        elif used_pct > 80:
            return DiagnosticResult(
                component="disk_space",
                status="warning",
                detail=f"Disk usage at {used_pct:.1f}% — {free_gb:.1f}GB free of {total_gb:.1f}GB",
                fix_command="docker system prune -af --filter 'until=168h'",
                severity="warning",
            )

        return DiagnosticResult(
            component="disk_space",
            status="healthy",
            detail=f"Disk usage at {used_pct:.1f}% — {free_gb:.1f}GB free of {total_gb:.1f}GB",
            severity="info",
        )

    @staticmethod
    def check_user_group() -> DiagnosticResult:
        """
        Check if the current user is in the docker group.
        """
        try:
            result = subprocess.run(
                ["groups"],
                capture_output=True, text=True, timeout=5
            )
            groups = result.stdout.strip().split()

            if "docker" in groups:
                return DiagnosticResult(
                    component="user_group",
                    status="member",
                    detail="Current user is in the docker group",
                    severity="info",
                )
            else:
                return DiagnosticResult(
                    component="user_group",
                    status="not_member",
                    detail=f"Current user ({os.getlogin()}) is not in the docker group. Groups: {', '.join(groups)}",
                    fix_command="sudo usermod -aG docker $USER && newgrp docker",
                    severity(
                component="disk_space",
                status="="critical",
                )
        except Exception as e:
            return DiagnosticResult(
                component="user_group",
                status="unknown",
                detail=f"Could not check group membership: {e}",
                severity="warning",
            )

    @staticmethod
    def run_full_diagnosis() -> List[DiagnosticResult]:
        """
        Run all diagnostic checks and return results.
        """
        checks = [
            DockerDaemonDiagnostics.check_daemon_installed(),
            DockerDaemonDiagnostics.check_daemon_running(),
            DockerDaemonDiagnostics.check_socket(),
            DockerDaemonDiagnostics.check_disk_space(),
            DockerDaemonDiagnostics.check_user_group(),
        ]
        return checks

    @staticmethod
    def print_report(results: List[DiagnosticResult]) -> None:
        """
        Print a formatted diagnostic report.
        """
        severity_icons = {
            "critical": "[FAIL]",
            "warning": "[WARN]",
            "info": "[ OK ]",
        }

        print("\nDocker Daemon Diagnostic Report")
        print("=" * 60)

        for r in results:
            icon = severity_icons.get(r.severity, "[????]")
            print(f"\n{icon} {r.component}: {r.status}")
            print(f"    {r.detail}")
            if r.fix_command:
                print(f"    Fix: {r.fix_command}")

        critical = [r for r in results if r.severity == "critical"]
        if critical:
            print(f"\n{'=' * 60}")
            print(f"ACTION REQUIRED: {len(critical)} critical issue(s) found.")
            print("Run the fix commands above to resolve.")


# Example usage
if __name__ == "__main__":
    results = DockerDaemonDiagnostics.run_full_diagnosis()
    DockerDaemonDiagnostics.print_report(results)

#!/bin/bash
# ============================================
# Docker Daemon Connection Fix for Linux
# Run these commands in order until the error is resolved
# ============================================

set -e

# Step 1: Check if Docker is installed
if ! command -v docker &> /dev/null; then
    echo "Docker is not installed."
    echo "Install with: curl -fsSL https://get.docker.com | sh"
    exit 1
fi

echo "Docker version: $(docker --version)"

# Step 2: Check daemon status
echo "\nChecking daemon status..."
sudo systemctl status docker --no-pager

# Step 3: Start daemon if not running
if ! systemctl is-active --quiet docker; then
    echo "Daemon is not running. Starting..."
    sudo systemctl start docker
    sudo systemctl enable docker
    echo "Daemon started and enabled on boot."
fi

# Step 4: Check socket file
if [ ! -S /var/run/docker.sock ]; then
    echo "Socket file missing. Restarting daemon..."
    sudo# Step 5: Check user group membership
if ! groups | grep -q docker; then
    echo "Current user is not in the docker group systemctl restart docker
    sleep 2
fi

."
    echo "Adding user to docker group..."
    sudo usermod -aG docker $USER
    echo "Run 'newgrp docker' or log out and back in for changes to take effect."
    newgrp docker
fi

# Step 6: Check disk space
DOCKER_ROOT="/var/lib/docker"
if [ -d "$DOCKER_ROOT" ]; then
    USAGE=$(df "$DOCKER_ROOT" | tail -1 | awk '{print $5}' | tr -d '%')
    echo "\nDisk usage on $DOCKER_ROOT: ${USAGE}%"
    if [ "$USAGE" -gt 90 ]; then
        echo "WARNING: Disk usage above 90%. Consider running:"
        echo "  docker system prune -af --volumes"
    fi
fi

# Step 7: Verify connection
echo "\nVerifying Docker connection..."
if docker info &> /dev/null; then
    echo "SUCCESS: Docker daemon is accessible."
    docker info --format 'Server Version: {{.ServerVersion}}'
else
    echo "FAILED: Still cannot connect to Docker daemon."
    echo "Check logs: sudo journalctl -u docker.service -n 50"
    exit 1
fi

#!/bin/bash
# ============================================
# Docker Daemon Connection Fix for macOS and WSL2
# ============================================

# ---- macOS Fix ----
fix_macos() {
    echo "Checking Docker Desktop on macOS..."

    # Check if Docker Desktop is running
    if ! pgrep -x "Docker Desktop" > /dev/null; then
        echo "Docker Desktop is not running. Starting..."
        open -a "Docker Desktop"
        echo "Waiting for Docker Desktop to start (up to 60 seconds)..."

        for i in $(seq 1 60); do
            if docker info &> /dev/null; then
                echo "Docker Desktop is ready."
                return 0
            fi
            sleep 1
        done

        echo "Docker Desktop failed to start within 60 seconds."
        echo "Try: killall Docker Desktop && open -a 'Docker Desktop'"
        return 1
    fi

    # Check socket symlink
    if [ ! -S /var/run/docker.sock ]; then
        echo "Socket symlink missing. Restarting Docker Desktop..."
        osascript -e 'quit app "Docker Desktop"'
        sleep 5
        open -a "Docker Desktop"
        sleep 30
    fi

    # Verify
    if docker info &> /dev/null; then
        echo "Docker connection OK."
    else
        echo "Still cannot connect. Try:"
        echo "  1. Docker Desktop > Troubleshoot > Clean / Purge data"
        echo "  2. Docker Desktop > Restart"
        echo "  3. rm -rf ~/.docker/run/docker.sock && restart Docker Desktop"
    fi
}

# ---- WSL2 Fix ----
fix_wsl2() {
    echo "Checking Docker in WSL2..."

    # Check if we are in WSL
    if ! grep -qi microsoft /proc/version 2>/dev/null; then
        echo "Not running in WSL. Use macOS fix or native Linux fix."
        return 1
    fi

    # Check if Docker Desktop is running on Windows
    echo "Ensure Docker Desktop is running on Windows."
    echo "Check: Docker Desktop > Settings > Resources > WSL Integration"
    echo "Enable integration for your WSL2 distribution."

    # Restart WSL
    echo "\nRestarting WSL to refresh connection..."
    echo "Run from Windows PowerShell: wsl --shutdown"
    echo "Then reopen your WSL terminal."

    # Check socket
    if [ -S /var/run/docker.sock ]; then
        echo "Socket exists."
    else
        echo "Socket missing. Docker Desktop WSL integration may be disabled."
        echo "Open Docker Desktop > Settings > Resources > WSL Integration"
        echo "Toggle your distro off and on again."
    fi
}

# Main
case "$(uname -s)" in
    Darwin*) fix_macos ;;
    Linux*)
        if grep -qi microsoft /proc/version 2>/dev/null; then
            fix_wsl2
        else
            echo "Use the Linux fix script instead."
        fi
        ;;
    *) echo "Unsupported OS: $(uname -s)" ;;
esac

#!/bin/bash
# ============================================
# Docker Daemon Prevention Strategies
# Run once on every production Docker host
# ============================================

set -e

# ---- Strategy 1: Systemd Watchdog ----
# Automatically restart Docker daemon if it becomes unresponsive

setup_watchdog() {
    echo "Setting up systemd watchdog for Docker daemon..."

    # Create override directory
    sudo mkdir -p /etc/systemd/system/docker.service.d

    # Create watchdog override
    sudo tee /etc/systemd/system/docker.service.d/watchdog.conf > /dev/null << 'EOF'
[Service]
WatchdogSec=60
Restart=always
RestartSec=5
EOF

    # Create health check script
    sudo tee /usr/local/bin/docker-healthcheck.sh > /dev/null << 'EOF'
#!/bin/bash
if docker info &> /dev/null; then
    exit 0
else
    exit 1
fi
EOF

    sudo chmod +x /usr/local/bin/docker-healthcheck.sh

    # Reload systemd
    sudo systemctl daemon-reload
    sudo systemctl restart docker

    echo "Watchdog configured. Daemon will restart if unresponsive for 60 seconds."
}

# ---- Strategy 2: Automatic Disk Pruning ----
# Prevent disk exhaustion that crashes the daemon

setup_auto_prune() {
    echo "Setting up automatic Docker disk pruning..."

    # Create pruning script
    sudo tee /usr/local/bin/docker-prune.sh > /dev/null << 'EOF'
#!/bin/bash
# Remove images older than 7 days
# Keep running containers and their images

LOGFILE="/var/log/docker-prune.log"

{
    echo "=== Docker Prune: $(date) ==="
    echo "Before:"
    docker system df

    # Prune stopped containers, unused networks, dangling images
    docker system prune -f --filter "until=168h"

    # Prune unused images (not just dangling)
    docker image prune -af --filter "until=168h"

    # Prune build cache older than 7 days
    docker builder prune -f --filter "until=168h"

    echo "After:"
    docker system df
    echo "=== Done ==="
} >> "$LOGFILE" 2>&1
EOF

    sudo chmod +x /usr/local/bin/docker-prune.sh

    # Add cron job: run every Sunday at 3 AM
    CRON_JOB="0 3 * * 0 /usr/local/bin/docker-prune.sh"
    (crontab -l 2>/dev/null | grep -v docker-prune; echo "$CRON_JOB") | crontab -

    echo "Auto-prune configured. Runs every Sunday at 3 AM."
}

# ---- Strategy 3: Disk Usage Alerting ----
# Alert before disk reaches critical levels

setup_disk_alerts() {
    echo "Setting up disk usage alerts..."

    sudo tee /usr/local/bin/docker-disk-alert.sh > /dev/null << 'EOF'
#!/bin/bash
DOCKER_ROOT="/var/lib/docker"
THRESHOLD=85

USAGE=$(df "$DOCKER_ROOT" | tail -1 | awk '{print $5}' | tr -d '%')

if [ "$USAGE" -ge "$THRESHOLD" ]; then
    echo "WARNING: Docker disk usage at ${USAGE}% (threshold: ${THRESHOLD}%)"
    echo "Running containers:"
    docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Size}}'
    echo "\nDisk breakdown:"
    docker system df

    # Send alert (customize for your alerting system)
    # curl -X POST https://alerts.example.com/webhook -d "{\"text\": \"Docker disk at ${USAGE}%\"}"
fi
EOF

    sudo chmod +x /usr/local/bin/docker-disk-alert.sh

    # Add cron job: check every 15 minutes
    CRON_JOB="*/15 * * * * /usr/local/bin/docker-disk-alert.sh"
    (crontab -l 2>/dev/null | grep -v docker-disk-alert; echo "$CRON_JOB") | crontab -

    echo "Disk alerting configured. Checks every 15 minutes at 85% threshold."
}

# ---- Strategy 4: Socket Permission Enforcement ----
# Ensure socket permissions survive Docker upgrades

setup_socket_permissions() {
    echo "Enforcing socket permissions..."

    # Ensure docker group exists
    sudo groupadd -f docker

    # Set socket ownership
    sudo tee /etc/docker/daemon.json > /dev/null << 'EOF'
{
  "group": "docker",
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  },
  "storage-driver": "overlay2"
}
EOF

    sudo systemctl restart docker

    echo "Socket permissions enforced via daemon.json."
}

# Main
setup_watchdog
setup_auto_prune
setup_disk_alerts
setup_socket_permissions

echo "\nAll prevention strategies configured."
echo "Docker daemon will now:"
echo "  - Auto-restart if unresponsive (60s watchdog)"
echo "  - Auto-prune old resources (weekly)"
echo "  - Alert at 85% disk usage (every 15 min)"
echo "  - Maintain socket permissions across restarts"

// io.thecodeforge — devops tutorial

# Verify socket exists and permissions are correct
- name: Check Docker socket permissions
  shell: ls -la /var/run/docker.sock
  register: socket_info

- name: Fail if socket missing or wrong ownership
  fail:
    msg: "Docker socket is absent or owned by wrong group."
  when: socket_info.stdout.find('srw-rw---- 1 root docker') == -1

# Expected output:
# srw-rw---- 1 root docker 0 Jan 15 09:42 /var/run/docker.sock

// io.thecodeforge — devops tutorial

- name: Ensure Docker socket unit is active and enabled
  systemd:
    name: docker.socket
    state: started
    enabled: yes

- name: Restart Docker service cleanly
  systemd:
    name: docker.service
    state: restarted
    daemon_reload: yes

- name: Test connection
  command: docker version
  changed_when: false

# Expected output:
# Client:
#  Version:           24.0.7
# Server:
#  Engine:
#   Version:          24.0.7

// io.thecodeforge — devops tutorial

# Check which socket is being used
- name: Inspect Docker socket in WSL2
  shell: ls -la /var/run/docker.sock
  register: socket_wsl

- name: Verify Docker Desktop integration
  shell: docker context ls
  register: context_list

- name: Set context to default if using Docker Desktop
  command: docker context use default
  when: context_list.stdout.find('desktop-linux') != -1

# Expected output of docker context ls:
# NAME                TYPE                DESCRIPTION
# default             moby                Current DOCKER_HOST based configuration
# desktop-linux *     moby                Docker Desktop