Advanced Kubernetes Interview Questions — Internals, Edge Cases & Production Gotchas

apiVersion: v1
kind: Pod
metadata:
  name: forge-app
  namespace: production
  labels:
    app: forge-api
    tier: backend
spec:
  containers:
  - name: forge-container
    image: io.thecodeforge/api:v1.2.0
    resources:
      requests:
        memory: "256Mi"
        cpu: "500m"
      limits:
        memory: "512Mi"
        cpu: "1"
    livenessProbe:
      httpGet:
        path: /healthz
        port: 8080
      initialDelaySeconds: 15
    readinessProbe:
      httpGet:
        path: /ready
        port: 8080
  topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: kubernetes.io/hostname
      whenUnsatisfiable: DoNotSchedule
      labelSelector:
        matchLabels:
          app: forge-api

# TheCodeForge Network Debugging Toolkit
# Package: io.thecodeforge.k8s

# 1. Check if the Service IP is active in iptables
iptables -L -t nat | grep FORGE-SERVICE-NAME

# 2. Inspect the CNI logs on the specific node
journalctl -u kubelet | grep cni

# 3. Test Pod-to-Pod connectivity bypassing the Service VIP
kubectl exec -it debug-pod -- curl <target-pod-ip>:8080/healthz

# 4. Check kube-proxy mode and health
kubectl get configmap kube-proxy -n kube-system -o yaml | grep mode
kubectl logs -n kube-system -l k8s-app=kube-proxy | tail -50

# 5. Verify NetworkPolicy is not blocking traffic
kubectl get networkpolicy -n <namespace> -o yaml
# If policies exist, check ingress/egress rules against the Pod's labels

# etcd Diagnostic Commands
# Package: io.thecodeforge.k8s

# 1. Check cluster member health
etcdctl endpoint health --cluster --write-out=table

# 2. Check member status (leader, DB size, Raft index)
etcdctl endpoint status --write-out=table

# 3. Check for alarm conditions (e.g., NOSPACE)
etcdctl alarm list

# 4. Defragment a member (reclaims space after compaction)
etcdctl defrag --endpoints=<endpoint>

# 5. Compact old revisions (prevents unbounded DB growth)
etcdctl compact $(etcdctl endpoint status --write-out=json | jq '.[0].Status.header.revision')

# 6. Snapshot backup
etcdctl snapshot save /backup/etcd-$(date +%Y%m%d-%H%M%S).db

# QoS Class: Guaranteed (requests == limits for all containers)
# Highest priority during eviction. Never OOMKilled unless node is under extreme pressure.
apiVersion: v1
kind: Pod
metadata:
  name: critical-service
  namespace: production
spec:
  containers:
    - name: app
      image: io.thecodeforge/api:stable
      resources:
        requests:
          cpu: "500m"
          memory: "512Mi"
        limits:
          cpu: "500m"      # Equal to request = Guaranteed QoS
          memory: "512Mi"   # Equal to request = Guaranteed QoS
---
# QoS Class: Burstable (requests < limits)
# Medium priority. Can burst above request but may be evicted under pressure.
apiVersion: v1
kind: Pod
metadata:
  name: web-frontend
  namespace: production
spec:
  containers:
    - name: app
      image: io.thecodeforge/frontend:latest
      resources:
        requests:
          cpu: "200m"
          memory: "256Mi"
        limits:
          cpu: "1"          # Can use up to 1 CPU core
          memory: "1Gi"     # Can use up to 1Gi RAM
---
# QoS Class: BestEffort (no requests or limits set)
# Lowest priority. First to be evicted. Not recommended for production.
apiVersion: v1
kind: Pod
metadata:
  name: debug-tool
  namespace: development
spec:
  containers:
    - name: debug
      image: io.thecodeforge/debug:latest
      # No resources defined = BestEffort QoS

# Least-privilege RBAC for a microservice
# Package: io.thecodeforge.k8s

# 1. Dedicated ServiceAccount (not default)
apiVersion: v1
kind: ServiceAccount
metadata:
  name: order-service
  namespace: production
automountServiceAccountToken: false  # Disable unless API access needed
---
# 2. Namespace-scoped Role with minimal permissions
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: order-service-role
  namespace: production
rules:
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["order-service-config"]  # Only specific configmap
    verbs: ["get", "watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["db-credentials"]  # Only specific secret
    verbs: ["get"]
---
# 3. Bind Role to ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: order-service-binding
  namespace: production
subjects:
  - kind: ServiceAccount
    name: order-service
    namespace: production
roleRef:
  kind: Role
  name: order-service-role
  apiGroup: rbac.authorization.k8s.io

// Simplified scheduler decision model
// Package: io.thecodeforge.k8s.scheduling
package io.thecodeforge.k8s.scheduling;

import java.util.List;
import java.util.Map;

public class SchedulerDecisionModel {

    /**
     * Phase 1: Filtering — Eliminate nodes that cannot run the Pod.
     * Filters are applied in order. If no nodes pass, the Pod stays Pending.
     */
    public List<String> filterNodes(List<String> allNodes, Pod pod) {
        return allNodes.stream()
            .filter(node -> hasEnoughResources(node, pod))     // NodeResourcesFit
            .filter(node -> matchesNodeAffinity(node, pod))    // NodeAffinity
            .filter(node -> toleratesTaints(node, pod))        // TaintToleration
            .filter(node -> matchesPodTopology(node, pod))     // PodTopologySpread
            .filter(node -> hasVolumeCapacity(node, pod))      // VolumeBinding
            .toList();
    }

    /**
     * Phase 2: Scoring — Rank feasible nodes by desirability.
     * Each scoring plugin assigns 0-100 points. Scores are summed.
     * The node with the highest total score wins.
     */
    public Map<String, Integer> scoreNodes(List<String> feasibleNodes, Pod pod) {
        // Simplified: In reality, each plugin scores independently
        return feasibleNodes.stream()
            .collect(java.util.stream.Collectors.toMap(
                node -> node,
                node -> scoreResourceBalancing(node, pod)    // NodeResourcesBalancedAllocation
                      + scorePodSpread(node, pod)             // PodTopologySpread
                      + scoreInterPodAffinity(node, pod)      // InterPodAffinity
                      + scoreImageLocality(node, pod)         // ImageLocality
            ));
    }

    private boolean hasEnoughResources(String node, Pod pod) { return true; }
    private boolean matchesNodeAffinity(String node, Pod pod) { return true; }
    private boolean toleratesTaints(String node, Pod pod) { return true; }
    private boolean matchesPodTopology(String node, Pod pod) { return true; }
    private boolean hasVolumeCapacity(String node, Pod pod) { return true; }
    private int scoreResourceBalancing(String node, Pod pod) { return 50; }
    private int scorePodSpread(String node, Pod pod) { return 50; }
    private int scoreInterPodAffinity(String node, Pod pod) { return 50; }
    private int scoreImageLocality(String node, Pod pod) { return 50; }
}

# Production-grade probe configuration
# Package: io.thecodeforge.k8s
apiVersion: v1
kind: Pod
metadata:
  name: api-server
  namespace: production
spec:
  containers:
    - name: api
      image: io.thecodeforge/api:3.0.0
      # Startup probe: Gates liveness/readiness until app boots
      # Critical for apps with slow startup (>30s)
      startupProbe:
        httpGet:
          path: /healthz
          port: 8080
        initialDelaySeconds: 5
        periodSeconds: 5
        failureThreshold: 30    # 30 * 5s = 150s max startup time
        successThreshold: 1
      # Liveness probe: Detects deadlocks and hung processes
      # Only active after startup probe succeeds
      livenessProbe:
        httpGet:
          path: /healthz
          port: 8080
        periodSeconds: 10
        failureThreshold: 3     # 3 failures = restart after 30s
        successThreshold: 1
        timeoutSeconds: 5
      # Readiness probe: Controls traffic routing
      # Failing = removed from Service endpoints
      readinessProbe:
        httpGet:
          path: /ready
          port: 8080
        periodSeconds: 5
        failureThreshold: 2     # 2 failures = remove from endpoints after 10s
        successThreshold: 1     # 1 success = add back to endpoints
        timeoutSeconds: 3
      resources:
        requests:
          cpu: "500m"
          memory: "512Mi"