Docker Internals: PID Namespace Bug Kills Containers

#!/bin/bash
# Inspect every layer of the Docker stack

# ── Docker CLI -> Daemon communication ───────────────────────────────────────
# The CLI sends HTTP requests to the daemon. You can do this manually:
curl --unix-socket /var/run/docker.sock http://localhost/version | python3 -m json.tool
# Shows: Docker version, API version, Go version, OS, architecture

curl --unix-socket /var/run/docker.sock http://localhost/containers/json | python3 -m json.tool
# Shows: all running containers (same as docker ps)

# ── Check if containerd is running ───────────────────────────────────────────
systemctl status containerd
# containerd is the container runtime daemon
# It manages container lifecycle independently of dockerd

# ── Find the runc binary ─────────────────────────────────────────────────────
which runc
# Typically: /usr/bin/runc or /usr/local/bin/runc

runc --version
# Shows: runc version, commit, spec version (OCI 1.0.2)

# ── Inspect the OCI runtime spec for a running container ─────────────────────
# containerd stores the OCI spec for each container
CONTAINER_ID=$(docker ps -q | head -1)

# Find the container's bundle directory (contains config.json)
find /run/containerd/io.containerd.runtime.v2.task/default/ -name config.json 2>/dev/null | head -1
# This file is the OCI runtime spec — it defines namespaces, cgroups, mounts

# ── Trace the container creation flow ────────────────────────────────────────
# Start a container and watch the kernel calls
strace -f -e trace=clone,unshare,pivot_root,chroot,execve \
  -o /tmp/container-trace.log \
  runc run test-container &

# The trace shows:
# clone(CLONE_NEWPID|CLONE_NEWNET|CLONE_NEWNS|...) = <child-pid>
# pivot_root(".", "/old-root") = 0
# execve("/app/server", ["server"], ...) = 0

# ── Check the daemon socket ─────────────────────────────────────────────────
ls -la /var/run/docker.sock
# srw-rw---- 1 root docker /var/run/docker.sock
# The socket is owned by root:docker group
# Any process in the docker group can control ALL containers

# ── Check the daemon process tree ────────────────────────────────────────────
pstree -p $(pidof dockerd)
# dockerd ─┬─ containerd ─┬─ containerd-shim-runc-v2 ─┬─ <app-pid>
#          │              │                            └─ pause
#          │              └─ containerd-shim-runc-v2 ─┬─ <app-pid>
#          │                                          └─ pause
#          └─ docker-proxy (for published ports)

#!/bin/bash
# Inspect and compare namespaces for containers and the host

# ── Get a container's host PID ───────────────────────────────────────────────
CONTAINER_NAME="my-api"
CONTAINER_PID=$(docker inspect --format '{{.State.Pid}}' $CONTAINER_NAME)
echo "Container $CONTAINER_NAME is PID $CONTAINER_PID on the host"

# ── List all namespaces for the container process ────────────────────────────
ls -la /proc/$CONTAINER_PID/ns/
# Output:
# lrwxrwxrwx ... ipc -> 'ipc:[4026532XXX]'
# lrwxrwxrwx ... mnt -> 'mnt:[4026532XXX]'
# lrwxrwxrwx ... net -> 'net:[4026532XXX]'
# lrwxrwxrwx ... pid -> 'pid:[4026532XXX]'
# lrwxrwxrwx ... user -> 'user:[4026531XXX]'
# lrwxrwxrwx ... uts -> 'uts:[4026532XXX]'

# ── Compare with host namespaces ─────────────────────────────────────────────
ls -la /proc/1/ns/
# The host PID 1 (systemd) has different namespace IDs than the container
# If namespace IDs match, the container shares that namespace with the host

# ── PID namespace: container sees its own PID tree ────────────────────────────
docker exec $CONTAINER_NAME ps aux
# PID 1 is the container's entrypoint process
# The container cannot see host processes

# On the host, the same process has a different PID:
ps aux | grep $(docker exec $CONTAINER_NAME cat /proc/1/cmdline | tr '\0' ' ')
# The host sees the real PID, the container sees PID 1

# ── Network namespace: container has its own network stack ───────────────────
docker exec $CONTAINER_NAME ip addr show
# Shows: lo (loopback) and eth0 (veth pair inside the container)

# On the host, inspect the veth pair:
ip link show | grep veth
# vethXXXX@if4: <BROADCAST,MULTICAST,UP> ... master docker0
# One end is in the container's net namespace, one end is on the docker0 bridge

# ── Enter a container's network namespace from the host ──────────────────────
sudo nsenter --net --target $CONTAINER_PID ip addr show
# Shows the same network config as docker exec, but from the host
# Useful for debugging without a shell inside the container

# ── Mount namespace: inspect the overlay2 filesystem ─────────────────────────
docker inspect --format '{{.GraphDriver.Data}}' $CONTAINER_NAME
# Shows: MergedDir, UpperDir, LowerDir, WorkDir
# MergedDir is what the container sees as /
# UpperDir is the writable layer (container-specific changes)
# LowerDir is the read-only image layers (colon-separated)

# ── User namespace: check if remapping is enabled ────────────────────────────
cat /etc/subuid
# If userns-remap is enabled: dockremap:100000:65536
# This maps container UID 0 to host UID 100000

# ── UTS namespace: container has its own hostname ─────────────────────────────
docker exec $CONTAINER_NAME hostname
# Shows the container's hostname (usually the container ID)

hostname
# Shows the host's hostname — different from the container

# ── IPC namespace: container has its own IPC resources ───────────────────────
docker exec $CONTAINER_NAME ipcs
# Shows only IPC resources created inside the container

ipcs
# Shows host IPC resources — not visible inside the container

#!/bin/bash
# Inspect and configure cgroup resource limits for containers

# ── Get a container's cgroup path ────────────────────────────────────────────
CONTAINER_ID=$(docker ps -q | head -1)

# cgroup v1 path:
ls /sys/fs/cgroup/cpu/docker/$CONTAINER_ID/
# cpu.shares, cpu.cfs_quota_us, cpu.cfs_period_us

# cgroup v2 path:
ls /sys/fs/cgroup/system.slice/docker-${CONTAINER_ID}.scope/
# cpu.max, memory.max, pids.max

# ── Check CPU limits ─────────────────────────────────────────────────────────

# cgroup v1:
cat /sys/fs/cgroup/cpu/docker/$CONTAINER_ID/cpu.shares
# Default: 1024 (1 CPU share). Set with --cpu-shares
cat /sys/fs/cgroup/cpu/docker/$CONTAINER_ID/cpu.cfs_quota_us
# -1 means no limit. Set with --cpus=1.0 (becomes 100000)
cat /sys/fs/cgroup/cpu/docker/$CONTAINER_ID/cpu.cfs_period_us
# Default: 100000 (100ms)

# cgroup v2:
cat /sys/fs/cgroup/system.slice/docker-${CONTAINER_ID}.scope/cpu.max
# Format: quota period. Example: 100000 100000 (1 CPU limit)

# ── Check memory limits ─────────────────────────────────────────────────────

# cgroup v1:
cat /sys/fs/cgroup/memory/docker/$CONTAINER_ID/memory.limit_in_bytes
# 9223372036854771712 means no limit (max int64)
# Set with --memory=512m (becomes 536870912)
cat /sys/fs/cgroup/memory/docker/$CONTAINER_ID/memory.usage_in_bytes
# Current memory usage in bytes

# cgroup v2:
cat /sys/fs/cgroup/system.slice/docker-${CONTAINER_ID}.scope/memory.max
# max means no limit. Set with --memory=512m
cat /sys/fs/cgroup/system.slice/docker-${CONTAINER_ID}.scope/memory.current
# Current memory usage

# ── Check OOM events ────────────────────────────────────────────────────────
cat /sys/fs/cgroup/memory/docker/$CONTAINER_ID/memory.oom_control
# oom_kill_disable: 0 (OOM killer enabled)
# under_oom: 0 (not currently under OOM pressure)

# Check kernel OOM log:
dmesg | grep -i 'oom\|killed process' | tail -10

# ── Check PID limits ────────────────────────────────────────────────────────

# cgroup v1:
cat /sys/fs/cgroup/pids/docker/$CONTAINER_ID/pids.max
# max means no limit. Set with --pids-limit=256
cat /sys/fs/cgroup/pids/docker/$CONTAINER_ID/pids.current
# Current number of processes

# ── Run a container with resource limits ─────────────────────────────────────
docker run -d \
  --name resource-limited \
  --cpus=1.0 \
  --memory=512m \
  --pids-limit=256 \
  --memory-swap=512m \
  # --memory-swap=512m disables swap (swap = memory limit)
  alpine:3.19 sleep 3600

# Verify the limits:
docker inspect resource-limited --format '{{.HostConfig.NanoCpus}}'
# 1000000000 = 1 CPU (in nanoseconds)

docker inspect resource-limited --format '{{.HostConfig.Memory}}'
# 536870912 = 512MB (in bytes)

docker stats resource-limited --no-stream
# Shows: MEM USAGE / LIMIT — 512MiB / 512MiB

#!/bin/bash
# Inspect the overlay2 filesystem for a running container

# ── Get the overlay2 paths for a container ───────────────────────────────────
CONTAINER_ID=$(docker ps -q | head -1)

GRAPH_DATA=$(docker inspect --format '{{json .GraphDriver.Data}}' $CONTAINER_ID)
echo $GRAPH_DATA | python3 -m json.tool

# Extract individual paths:
MERGED_DIR=$(echo $GRAPH_DATA | python3 -c "import sys,json; print(json.load(sys.stdin)['MergedDir'])")
UPPER_DIR=$(echo $GRAPH_DATA | python3 -c "import sys,json; print(json.load(sys.stdin)['UpperDir'])")
LOWER_DIR=$(echo $GRAPH_DATA | python3 -c "import sys,json; print(json.load(sys.stdin)['LowerDir'])")
WORK_DIR=$(echo $GRAPH_DATA | python3 -c "import sys,json; print(json.load(sys.stdin)['WorkDir'])")

echo "Merged (container sees this as /): $MERGED_DIR"
echo "Upper (writable layer): $UPPER_DIR"
echo "Lower (read-only layers): $LOWER_DIR"
echo "Work (overlay2 internal): $WORK_DIR"

# ── Inspect the writable layer (upperdir) ────────────────────────────────────
ls -la $UPPER_DIR/
# Shows files the container has created or modified
# Files prefixed with .wh. are whiteout files (deleted from lower layers)

# ── Inspect the merged view ──────────────────────────────────────────────────
ls -la $MERGED_DIR/
# This is what the container sees as its root filesystem
# It is the combination of all lower layers + the upper layer

# ── Demonstrate the copy-up behavior ─────────────────────────────────────────
# Create a file in the container
docker exec $CONTAINER_ID sh -c 'echo "hello" > /tmp/test-file'

# The file appears in the writable layer (upperdir):
ls -la $UPPER_DIR/tmp/test-file
# The file is in the upper layer, not in any lower layer

# ── Demonstrate the whiteout behavior ────────────────────────────────────────
# Delete a file that exists in a lower layer
docker exec $CONTAINER_ID rm /etc/hostname

# A whiteout file appears in the upper layer:
ls -la $UPPER_DIR/etc/.wh.hostname
# This character device (0/0) tells overlay2 to hide the lower layer's file

# ── Check the number of layers in an image ───────────────────────────────────
docker inspect <image> --format '{{len .RootFS.Layers}} layers'
# Each layer is a directory under /var/lib/docker/overlay2/

# ── Check disk usage per layer ───────────────────────────────────────────────
du -sh /var/lib/docker/overlay2/* | sort -hr | head -10
# Shows disk usage for each layer (shared layers are counted once)

# ── Compare overlay2 with native filesystem performance ─────────────────────
# Write performance test:
time docker exec $CONTAINER_ID dd if=/dev/zero of=/tmp/test bs=1M count=100
# Overlay2 write: ~0.3s (writes to upperdir on host filesystem)

# Read performance test:
time docker exec $CONTAINER_ID dd if=/tmp/test of=/dev/null bs=1M
# Overlay2 read: ~0.1s (slightly slower than native due to layer lookup)

#!/bin/bash
# Trace the complete container lifecycle from clone to exec

# ── Step 1: Pull and inspect image layers ────────────────────────────────────
docker pull alpine:3.19

# Inspect the image layers:
docker inspect alpine:3.19 --format '{{json .RootFS.Layers}}' | python3 -m json.tool
# Each entry is a layer (SHA256 digest)

# Find the layers on disk:
ls /var/lib/docker/overlay2/ | head -5
# Each directory is a layer. Shared layers are hard-linked or reflinked.

# ── Step 2: Create a container and inspect the OCI spec ──────────────────────
# Create a container without starting it:
docker create --name lifecycle-demo alpine:3.19 echo 'hello'

# Find the OCI runtime spec:
find /run/containerd -name config.json -path '*lifecycle-demo*' 2>/dev/null
# This file is the OCI runtime spec that runc reads

# Inspect the spec (if found):
cat /run/containerd/io.containerd.runtime.v2.task/default/lifecycle-demo/config.json | python3 -m json.tool | head -50
# Shows: namespaces, mounts, cgroups, process config, root filesystem

# ── Step 3: Trace runc's kernel calls ────────────────────────────────────────
# Start a container with strace to see the kernel calls:
sudo strace -f -e trace=clone,clone3,unshare,sethostname,mount,pivot_root,setuid,setgid,execve \
  -o /tmp/runc-trace.log \
  runc run --bundle /path/to/bundle test-trace

# The trace shows:
# clone3({flags=CLONE_NEWPID|CLONE_NEWNET|CLONE_NEWNS|..., ...}) = 12345
# sethostname("container-id", 12) = 0
# mount("proc", "/proc", "proc", ...) = 0
# mount("sysfs", "/sys", "sysfs", ...) = 0
# pivot_root(".", "/old-root") = 0
# setuid(1000) = 0
# setgid(1000) = 0
# execve("/bin/sh", ["sh"], ...) = 0

# ── Step 4: Find the container process and pause process on the host ─────────
docker start lifecycle-demo

# Find the container's host PID:
CONTAINER_PID=$(docker inspect --format '{{.State.Pid}}' lifecycle-demo)
echo "Container process PID: $CONTAINER_PID"

# Find the pause process (holds namespaces open):
ps aux | grep pause | grep -v grep
# root  5678  0.0  0.0  1024  4  ?  Ss  10:23  0:00 /pause

# The pause process and the container process share the same namespaces:
ls -la /proc/$CONTAINER_PID/ns/net
ls -la /proc/$(pgrep -f '/pause' | head -1)/ns/net
# Both point to the same namespace inode

# ── Step 5: Watch the container process on the host ──────────────────────────
ps aux | grep $CONTAINER_PID
# root  5679  0.0  0.1  ...  echo hello
# This is the REAL process on the host, running inside namespaces

# ── Cleanup ──────────────────────────────────────────────────────────────────
docker rm -f lifecycle-demo