Release Management Best Practices: Ship Faster Without Breaking Things

# GitHub Actions workflow that automatically determines the next
# semantic version, tags the release, and publishes release notes.
# Triggered only on pushes to the main branch (i.e., after a PR merges).

name: Automated Semantic Release

on:
  push:
    branches:
      - main  # Only run on merged PRs — never on feature branches

jobs:
  release:
    name: Determine Version and Tag Release
    runs-on: ubuntu-latest
    permissions:
      contents: write       # Needed to push the git tag
      issues: write         # Needed to comment on resolved issues
      pull-requests: write  # Needed to comment on merged PRs

    steps:
      - name: Checkout full git history
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # CRITICAL: semantic-release needs full history to
                          # calculate the correct version bump. Shallow clones
                          # (the default) will cause it to fail silently.

      - name: Set up Node.js for semantic-release tooling
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install semantic-release and changelog plugin
        run: |
          npm install --save-dev \
            semantic-release \
            @semantic-release/changelog \
            @semantic-release/git
          # @semantic-release/changelog writes a CHANGELOG.md automatically
          # @semantic-release/git commits the changelog back to main

      - name: Run semantic-release
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          # semantic-release reads commit messages to decide the bump:
          # 'fix: ...'           -> PATCH bump  (1.4.1 -> 1.4.2)
          # 'feat: ...'          -> MINOR bump  (1.4.2 -> 1.5.0)
          # 'feat!: ...' or
          # 'BREAKING CHANGE:'   -> MAJOR bump  (1.5.0 -> 2.0.0)
        run: npx semantic-release

  build-and-push:
    name: Build Docker Image with Version Tag
    runs-on: ubuntu-latest
    needs: release  # Only runs AFTER the version tag exists in git

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Extract the new version tag created by semantic-release
        id: get_version
        run: |
          # Pull the latest tag that semantic-release just created
          RELEASE_VERSION=$(git describe --tags --abbrev=0)
          echo "version=${RELEASE_VERSION}" >> $GITHUB_OUTPUT
          echo "Detected release version: ${RELEASE_VERSION}"

      - name: Build and tag Docker image with immutable version
        run: |
          docker build \
            --tag myapp:${{ steps.get_version.outputs.version }} \
            --tag myapp:latest \
            --label "org.opencontainers.image.version=${{ steps.get_version.outputs.version }}" \
            --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            .
          # Tagging with BOTH the exact version AND 'latest' is intentional:
          # - Exact version tag is immutable — you can always roll back to it
          # - 'latest' tag is for convenience in non-production environments
          # NEVER deploy 'latest' to production. Always use the exact version.

      - name: Push to container registry
        run: |
          docker push myapp:${{ steps.get_version.outputs.version }}
          docker push myapp:latest

# GitLab CI/CD pipeline demonstrating environment promotion with quality gates.
# The SAME Docker image (identified by its SHA) moves through each environment.
# No rebuilds between environments — what passed testing IS what gets deployed.

stages:
  - build         # Compile and package the artifact once
  - test          # Run all automated quality gates
  - deploy-staging    # Automatic on every main branch push
  - verify-staging    # Automated smoke tests against staging
  - deploy-production # Manual trigger — human makes the final call

variables:
  REGISTRY: registry.example.com
  IMAGE_NAME: $REGISTRY/payments-service
  # IMAGE_TAG is derived from the git commit SHA — this guarantees
  # we always know exactly which code is running in any environment.
  IMAGE_TAG: $CI_COMMIT_SHORT_SHA

# ─── STAGE 1: Build ───────────────────────────────────────────────────────────
build-docker-image:
  stage: build
  image: docker:24
  services:
    - docker:24-dind  # Docker-in-Docker so we can build images in CI
  script:
    - docker build --tag $IMAGE_NAME:$IMAGE_TAG .
    - docker push $IMAGE_NAME:$IMAGE_TAG
    # We push immediately so subsequent stages can pull the same image.
    # No stage ever calls 'docker build' again — this is the single source of truth.
  only:
    - main

# ─── STAGE 2: Test (Quality Gates) ───────────────────────────────────────────
run-unit-and-integration-tests:
  stage: test
  image: $IMAGE_NAME:$IMAGE_TAG  # Run tests INSIDE the built image
  script:
    - pytest tests/ --cov=src --cov-fail-under=80
    # --cov-fail-under=80 is a hard gate: if coverage drops below 80%,
    # this job returns exit code 1, the pipeline stops, no deployment happens.
  coverage: '/TOTAL.*\s+(\d+%)$/'

scan-for-vulnerabilities:
  stage: test
  image: aquasec/trivy:latest
  script:
    - trivy image --exit-code 1 --severity CRITICAL $IMAGE_NAME:$IMAGE_TAG
    # exit-code 1 on CRITICAL CVEs means a known critical vulnerability
    # in the image will BLOCK deployment. This is non-negotiable.
    # You can set --severity HIGH,CRITICAL once your baseline is clean.

check-performance-regression:
  stage: test
  script:
    - |
      # Compare p95 response time against the last production benchmark.
      # If we're more than 5% slower, we don't ship.
      CURRENT_P95=$(run-load-test --output p95)
      BASELINE_P95=$(fetch-baseline --metric p95)
      REGRESSION=$(( (CURRENT_P95 - BASELINE_P95) * 100 / BASELINE_P95 ))
      if [ "$REGRESSION" -gt 5 ]; then
        echo "GATE FAILED: p95 latency regressed by ${REGRESSION}% (threshold: 5%)"
        exit 1
      fi
      echo "Performance gate passed. Regression: ${REGRESSION}%"

# ─── STAGE 3: Deploy to Staging ───────────────────────────────────────────────
deploy-to-staging:
  stage: deploy-staging
  environment:
    name: staging
    url: https://staging.example.com
  script:
    - kubectl set image deployment/payments-service \
        payments-service=$IMAGE_NAME:$IMAGE_TAG \
        --namespace=staging
    # We're deploying the EXACT same $IMAGE_TAG that was built and tested.
    # kubectl set image updates the running deployment without recreating it.
    - kubectl rollout status deployment/payments-service --namespace=staging
    # rollout status blocks until the deployment is healthy or times out.
    # This ensures the next stage only runs if staging is actually up.
  only:
    - main

# ─── STAGE 4: Verify Staging ──────────────────────────────────────────────────
run-staging-smoke-tests:
  stage: verify-staging
  script:
    - |
      # Smoke tests hit the real staging URL and check critical user journeys:
      # login, create payment, view dashboard. Fast checks — not a full suite.
      newman run smoke-tests/payments-collection.json \
        --environment smoke-tests/staging-env.json \
        --reporters cli,junit \
        --reporter-junit-export smoke-test-results.xml
  artifacts:
    reports:
      junit: smoke-test-results.xml  # GitLab parses this to show pass/fail inline
  only:
    - main

# ─── STAGE 5: Deploy to Production (Manual Gate) ──────────────────────────────
deploy-to-production:
  stage: deploy-production
  environment:
    name: production
    url: https://app.example.com
  when: manual          # A human must click 'play' in the GitLab UI
  allow_failure: false  # If this fails, mark the whole pipeline as failed
  script:
    - kubectl set image deployment/payments-service \
        payments-service=$IMAGE_NAME:$IMAGE_TAG \
        --namespace=production
    - kubectl rollout status deployment/payments-service --namespace=production
    # Tag the production-deployed image with 'stable' so we always know
    # what the last known-good production image was.
    - docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:stable
    - docker push $IMAGE_NAME:stable
  only:
    - main

# Real-world feature flag pattern for a payment checkout flow.
# We're rolling out a new 'express checkout' experience gradually.
# The flag system is Unleash (open-source, self-hostable).

import logging
from unleash_client import UnleashClient
from unleash_client.strategies import Strategy

logger = logging.getLogger(__name__)

# ── Initialise the Unleash client once at application startup ──────────────────
# In production this points to your Unleash server.
# In tests, you can use FakeUnleash to avoid network calls.
unleash_client = UnleashClient(
    url="https://unleash.internal.example.com/api",
    app_name="checkout-service",
    custom_headers={"Authorization": "*:production.your-secret-token"}
)
unleash_client.initialize_client()
# initialize_client() fetches all flag states and caches them.
# The client polls for updates in the background — no per-request network calls.


def process_checkout(user_id: str, cart_items: list, user_tier: str) -> dict:
    """
    Routes the user to either the new express checkout or the classic flow
    depending on the feature flag state for this specific user.

    The flag can be configured in Unleash to:
      - Be ON for specific user IDs (early adopters / beta testers)
      - Be ON for a % of users (gradual rollout)
      - Be ON only for users with user_tier == 'premium' (targeted release)
      - Be completely OFF (emergency kill-switch)
    """
    # Context tells Unleash WHO is asking, so it can apply targeting rules.
    # This is what makes flags smarter than a simple boolean.
    flag_context = {
        "userId": user_id,
        "properties": {
            "userTier": user_tier  # Custom property for tier-based targeting
        }
    }

    # is_enabled() is the key call — it checks the local cache, NOT the server,
    # so it adds microseconds of latency, not milliseconds.
    use_express_checkout = unleash_client.is_enabled(
        "express-checkout-v2",   # Flag name as defined in Unleash UI
        context=flag_context,
        fallback_function=lambda feature_name, ctx: False
        # fallback_function returns False if Unleash is unreachable.
        # NEVER let a flag evaluation crash your application — always define a fallback.
    )

    if use_express_checkout:
        logger.info(
            "express_checkout_used",
            extra={"user_id": user_id, "flag": "express-checkout-v2", "variant": "on"}
            # Structured logging lets you correlate flag state with error rates
            # in your observability platform (Datadog, Grafana, etc.)
        )
        return _run_express_checkout(user_id, cart_items)
    else:
        logger.info(
            "classic_checkout_used",
            extra={"user_id": user_id, "flag": "express-checkout-v2", "variant": "off"}
        )
        return _run_classic_checkout(user_id, cart_items)


def _run_express_checkout(user_id: str, cart_items: list) -> dict:
    """New express checkout — single-page, saved payment methods, faster UX."""
    # New implementation here. This runs in production for flagged users
    # while _run_classic_checkout handles everyone else.
    return {
        "status": "success",
        "flow": "express",
        "steps_completed": 1,
        "order_id": f"EXP-{user_id}-001"
    }


def _run_classic_checkout(user_id: str, cart_items: list) -> dict:
    """Existing checkout flow — kept alive until express is 100% rolled out."""
    return {
        "status": "success",
        "flow": "classic",
        "steps_completed": 3,
        "order_id": f"CLX-{user_id}-001"
    }


# ── Example usage ──────────────────────────────────────────────────────────────
if __name__ == "__main__":
    # Premium user — assume flag is configured to target 'premium' tier
    premium_result = process_checkout(
        user_id="user-99201",
        cart_items=[{"sku": "SHOE-42", "qty": 1}],
        user_tier="premium"
    )
    print(f"Premium user checkout: {premium_result}")

    # Free tier user — flag is OFF for this tier
    free_result = process_checkout(
        user_id="user-10042",
        cart_items=[{"sku": "HAT-L", "qty": 2}],
        user_tier="free"
    )
    print(f"Free user checkout: {free_result}")

#!/usr/bin/env bash
# ─────────────────────────────────────────────────────────────────────────────
# Rollback Runbook — payments-service
# Run this when a production deployment causes errors above the SLO threshold.
# Prerequisites: kubectl configured, Docker registry access, Slack webhook set.
# ─────────────────────────────────────────────────────────────────────────────

set -euo pipefail
# -e: exit immediately on any error
# -u: treat unset variables as errors (prevents silent config bugs)
# -o pipefail: a pipe fails if ANY command in it fails, not just the last one

# ── Configuration ─────────────────────────────────────────────────────────────
NAMESPACE="production"
DEPLOYMENT_NAME="payments-service"
CONTAINER_NAME="payments-service"
REGISTRY="registry.example.com"
SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL}"  # Injected from CI/CD secrets

# ── Step 1: Confirm current broken state before touching anything ──────────────
echo "──────────────────────────────────────────────────"
echo "ROLLBACK INITIATED — $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
echo "Deployment: ${DEPLOYMENT_NAME} in namespace: ${NAMESPACE}"
echo "──────────────────────────────────────────────────"

CURRENT_IMAGE=$(kubectl get deployment "${DEPLOYMENT_NAME}" \
  --namespace="${NAMESPACE}" \
  --output=jsonpath='{.spec.template.spec.containers[0].image}')
echo "Current (broken) image: ${CURRENT_IMAGE}"

# ── Step 2: Find the last known-good image (tagged as 'stable') ────────────────
# The 'stable' tag was set during the last SUCCESSFUL production deploy.
# See deploy-to-production stage in the pipeline config above.
STABLE_IMAGE="${REGISTRY}/${DEPLOYMENT_NAME}:stable"
echo "Rolling back to stable image: ${STABLE_IMAGE}"

# ── Step 3: Execute the rollback ──────────────────────────────────────────────
kubectl set image deployment/"${DEPLOYMENT_NAME}" \
  "${CONTAINER_NAME}=${STABLE_IMAGE}" \
  --namespace="${NAMESPACE}" \
  --record  # --record writes this change to the deployment's change history

# Block until all pods are running the stable image.
# Timeout of 3 minutes — if it takes longer, something is seriously wrong.
kubectl rollout status deployment/"${DEPLOYMENT_NAME}" \
  --namespace="${NAMESPACE}" \
  --timeout=3m

echo "Rollback complete. Verifying pod health..."

# ── Step 4: Quick sanity check — are all pods Ready? ──────────────────────────
READY_PODS=$(kubectl get deployment "${DEPLOYMENT_NAME}" \
  --namespace="${NAMESPACE}" \
  --output=jsonpath='{.status.readyReplicas}')
DESIRED_PODS=$(kubectl get deployment "${DEPLOYMENT_NAME}" \
  --namespace="${NAMESPACE}" \
  --output=jsonpath='{.spec.replicas}')

if [ "${READY_PODS}" -ne "${DESIRED_PODS}" ]; then
  echo "WARNING: Only ${READY_PODS}/${DESIRED_PODS} pods are ready after rollback."
  echo "Check pod logs: kubectl logs -l app=${DEPLOYMENT_NAME} -n ${NAMESPACE}"
  exit 1
fi

echo "✓ All ${READY_PODS}/${DESIRED_PODS} pods are healthy."

# ── Step 5: Notify the team — a silent rollback is a sneaky rollback ───────────
curl --silent --fail --show-error \
  --request POST \
  --header 'Content-type: application/json' \
  --data "{
    \"text\": \":rotating_light: *ROLLBACK EXECUTED* :rotating_light:\",
    \"attachments\": [{
      \"color\": \"danger\",
      \"fields\": [
        {\"title\": \"Service\",         \"value\": \"${DEPLOYMENT_NAME}\",  \"short\": true},
        {\"title\": \"Environment\",      \"value\": \"${NAMESPACE}\",       \"short\": true},
        {\"title\": \"Rolled back FROM\", \"value\": \"${CURRENT_IMAGE}\",  \"short\": false},
        {\"title\": \"Rolled back TO\",   \"value\": \"${STABLE_IMAGE}\",   \"short\": false},
        {\"title\": \"Executed by\",      \"value\": \"${USER:-ci-system}\", \"short\": true},
        {\"title\": \"Time\",             \"value\": \"$(date -u '+%Y-%m-%d %H:%M UTC')\", \"short\": true}
      ]
    }]
  }" \
  "${SLACK_WEBHOOK_URL}"

echo ""
echo "Slack notification sent. Rollback complete."
echo "Next step: create a post-mortem issue and identify root cause before re-deploying."