Kubernetes Secret Env Var Snapshot Causes Connection Storm

# ConfigMap: Non-sensitive configuration
# Package: io.thecodeforge.kubernetes
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
  namespace: production
data:
  DATABASE_HOST: "postgres.internal.svc.cluster.local"
  LOG_LEVEL: "info"
  FEATURE_FLAG_NEW_UI: "true"
binaryData:
  logo.png: "iVBORw0KGgoAAAANSUhEUg..."  # binary data supported
---
# Secret: Sensitive credentials
apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
  namespace: production
type: Opaque  # default type
data:
  username: cG9zdGdyZXM=    # base64-encoded 'postgres'
  password: c3VwZXJzZWNyZXQ=  # base64-encoded 'supersecret'
stringData:
  connection-string: "postgresql://postgres:supersecret@postgres:5432/appdb"  # plain text, auto-encoded

# Envelope Encryption Configuration for Secrets at Rest
# This file goes on the API Server as --encryption-provider-config
# Package: io.thecodeforge.kubernetes

apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets       # Encrypt Secrets at rest
      - configmaps    # Optionally encrypt ConfigMaps too
    providers:
      - aescbc:       # Local encryption (AES-CBC)
          keys:
            - name: key1
              secret: <base64-encoded-32-byte-key>
      - identity: {}  # Fallback: no encryption (reads old unencrypted data)

# Pattern 1: Environment Variable Injection (Static)
# Values are set at pod creation time. Do NOT update on Secret change.
apiVersion: v1
kind: Pod
metadata:
  name: app-env-injection
spec:
  containers:
    - name: app
      image: io.thecodeforge/app:1.0
      envFrom:
        - configMapRef:
            name: app-config
        - secretRef:
            name: db-credentials
      env:
        - name: SPECIFIC_KEY
          valueFrom:
            secretKeyRef:
              name: db-credentials
              key: password
---
# Pattern 2: Volume Mount (Dynamic Rotation Support)
# Files update when source Secret/ConfigMap changes (kubelet sync interval).
apiVersion: v1
kind: Pod
metadata:
  name: app-volume-mount
spec:
  containers:
    - name: app
      image: io.thecodeforge/app:1.0
      volumeMounts:
        - name: config-volume
          mountPath: /etc/config
          readOnly: true
        - name: secret-volume
          mountPath: /etc/secrets
          readOnly: true
          # subPath mounts do NOT auto-update!
      # subPath example (static, no rotation):
      # - name: secret-volume
      #   mountPath: /etc/secrets/password.txt
      #   subPath: password
  volumes:
    - name: config-volume
      configMap:
        name: app-config
    - name: secret-volume
      secret:
        secretName: db-credentials
        defaultMode: 0440  # read-only for owner and group

# Service Account: Dedicated per workload
apiVersion: v1
kind: ServiceAccount
metadata:
  name: payment-service-account
  namespace: payments
automountServiceAccountToken: false  # Disable unless the app needs API access
---
# Role: Read only specific Secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: secret-reader
  namespace: payments
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["payment-gateway-key", "db-credentials"]  # Only specific secrets
    verbs: ["get"]  # NOT 'list' — prevents enumeration
---
# RoleBinding: Bind SA to Role
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: payment-secret-access
  namespace: payments
subjects:
  - kind: ServiceAccount
    name: payment-service-account
    namespace: payments
roleRef:
  kind: Role
  name: secret-reader
  apiGroup: rbac.authorization.k8s.io

# Secrets Store CSI Driver: Mount external secrets as volumes
# Requires: secrets-store-csi-driver installed in cluster
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
  name: vault-db-creds
  namespace: production
spec:
  provider: vault  # or aws, gcp, azure
  parameters:
    vaultAddress: "https://vault.internal:8200"
    roleName: "payment-service"
    objects: |
      - objectName: "db-password"
        secretPath: "secret/data/production/db"
        secretKey: "password"
---
# Pod consuming the external secret
apiVersion: v1
kind: Pod
metadata:
  name: payment-service
spec:
  containers:
    - name: app
      image: io.thecodeforge/payment-service:2.1
      volumeMounts:
        - name: secrets-store
          mountPath: /mnt/secrets
          readOnly: true
  volumes:
    - name: secrets-store
      csi:
        driver: secrets-store.csi.k8s.io
        readOnly: true
        volumeAttributes:
          secretProviderClass: "vault-db-creds"

apiVersion: v1
kind: Pod
metadata:
  name: config-consumer
spec:
  containers:
  - name: app
    image: myapp:1.0
    volumeMounts:
    - name: config
      mountPath: /etc/config
      readOnly: true
  volumes:
  - name: config
    configMap:
      name: app-config
      items:
      - key: PORT
        path: port.txt

# After updating ConfigMap, read the file:
# $ kubectl exec config-consumer -- cat /etc/config/port.txt
# 3000 -> 8080 (after sync, typically 30-60s delay)

apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: $(openssl rand -base64 32)
      - identity: {}

# Verify encryption is active:
# $ kubectl get secret my-secret -o yaml | grep -A1 'data:'
# data:
#   password: <base64><-- still base64 in API, but encrypted at rest in etcd

# Check etcd directly:
# $ ETCDCTL_API=3 etcdctl get /registry/secrets/default/my-secret | hexdump
# First bytes show "k8s:enc:aescbc:v1:key1" if encrypted

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
spec:
  refreshInterval: "1h"
  secretStoreRef:
    name: aws-secretsmanager
    kind: SecretStore
  target:
    name: db-secret
    creationPolicy: Owner
  data:
  - secretKey: password
    remoteRef:
      key: prod/database/primary
      property: password

# After rotation in AWS Secrets Manager:
# $ kubectl get secret db-secret -o jsonpath='{.data.password}' | base64 -d
# new_password_123 (updated within 1 hour, no pod restart)