CloudWatch TreatMissingData — Why Silence Won't Page

#!/bin/bash
# publish_custom_metric.sh
# Publishes a custom business metric to CloudWatch.
# Run this from an EC2 instance, a container, or your CI/CD pipeline.
# Prerequisite: AWS CLI installed and IAM role with cloudwatch:PutMetricData permission.

APP_NAME="checkout-service"
ENVIRONMENT="production"

# Simulate reading the number of orders processed in the last minute.
# In a real app you'd query your database or read from an in-memory counter.
ORDERS_PROCESSED=142

# Publish the custom metric to a namespace we own.
# Namespace acts like a folder — use a consistent naming convention.
aws cloudwatch put-metric-data \
  --namespace "TheCodeForge/${APP_NAME}" \
  --metric-name "OrdersProcessedPerMinute" \
  --value "${ORDERS_PROCESSED}" \
  --unit "Count" \
  --dimensions \
    "Name=Environment,Value=${ENVIRONMENT}" \
    "Name=AppName,Value=${APP_NAME}" \
  --region us-east-1

# Verify the metric was accepted (exit code 0 means success)
if [ $? -eq 0 ]; then
  echo "[OK] Metric published: ${ORDERS_PROCESSED} orders for ${APP_NAME} in ${ENVIRONMENT}"
else
  echo "[ERROR] Failed to publish metric. Check IAM permissions and region."
  exit 1
fi

# cloudwatch_alarm.yaml
# CloudFormation template that creates a CloudWatch alarm on a Lambda function.
# Deploy with: aws cloudformation deploy --template-file cloudwatch_alarm.yaml \
#              --stack-name checkout-lambda-alarms --capabilities CAPABILITY_IAM

AWSTemplateFormatVersion: '2010-09-09'
Description: >-
  Alarm that fires when the checkout Lambda error rate exceeds 1% over 5 minutes.
  Sends an alert to the on-call SNS topic when triggered.

Parameters:
  LambdaFunctionName:
    Type: String
    Default: checkout-processor
    Description: The name of the Lambda function to monitor.

  OnCallSnsTopicArn:
    Type: String
    Description: ARN of the SNS topic that routes to PagerDuty or email.

Resources:

  # Alarm: triggers if Lambda errors exceed 5 in any 5-minute window.
  # EvaluationPeriods: how many periods must breach before alarm fires.
  # DatapointsToAlarm: of those periods, how many must actually breach.
  # Using 2-of-3 prevents a single noisy data point from waking someone at 3am.
  CheckoutLambdaErrorAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmName: !Sub '${LambdaFunctionName}-high-error-rate'
      AlarmDescription: >-
        Fires when checkout Lambda has more than 5 errors in a 5-minute period.
        Check Lambda logs in CloudWatch for stack traces.
      Namespace: AWS/Lambda                 # Built-in Lambda namespace — no setup needed
      MetricName: Errors
      Dimensions:
        - Name: FunctionName
          Value: !Ref LambdaFunctionName    # Scoped to our specific function
      Statistic: Sum                        # Add up all error counts in the period
      Period: 300                           # Each evaluation window is 5 minutes (300 seconds)
      EvaluationPeriods: 3                  # Look at the last 3 windows (15 minutes total)
      DatapointsToAlarm: 2                  # Alarm only if 2 out of 3 windows breach threshold
      Threshold: 5                          # More than 5 errors triggers the alarm
      ComparisonOperator: GreaterThanThreshold
      TreatMissingData: notBreaching        # No data = function not invoked = not an error
      AlarmActions:
        - !Ref OnCallSnsTopicArn            # Page on-call engineer via SNS
      OKActions:
        - !Ref OnCallSnsTopicArn            # Also notify when alarm recovers

Outputs:
  AlarmName:
    Description: Name of the created CloudWatch alarm
    Value: !Ref CheckoutLambdaErrorAlarm

#!/bin/bash
# logs_insights_query.sh
# Runs a CloudWatch Logs Insights query to find the top 10 slowest
# API endpoints in the last hour, using structured JSON logs.
#
# Assumes your app logs JSON lines like:
# {"level":"INFO","endpoint":"/api/checkout","duration_ms":342,"status":200}

LOG_GROUP_NAME="/production/checkout-service"
QUERY_LOOKBACK_SECONDS=3600  # Last 1 hour

START_TIME=$(date -u -d "-${QUERY_LOOKBACK_SECONDS} seconds" +%s)  # Linux
# On macOS use: date -u -v -${QUERY_LOOKBACK_SECONDS}S +%s
END_TIME=$(date -u +%s)

echo "Starting Logs Insights query on: ${LOG_GROUP_NAME}"
echo "Time range: last ${QUERY_LOOKBACK_SECONDS} seconds"

# Start the query — Logs Insights runs asynchronously, so we get a query ID back.
QUERY_ID=$(aws logs start-query \
  --log-group-name "${LOG_GROUP_NAME}" \
  --start-time "${START_TIME}" \
  --end-time "${END_TIME}" \
  --query-string '
    fields @timestamp, endpoint, duration_ms, status
    | filter ispresent(duration_ms)          # Only include log lines that have this field
    | stats avg(duration_ms) as avg_duration,
            max(duration_ms) as max_duration,
            count() as request_count
      by endpoint
    | sort avg_duration desc                 # Slowest endpoints first
    | limit 10
  ' \
  --query 'queryId' \
  --output text \
  --region us-east-1)

echo "Query submitted. ID: ${QUERY_ID}"
echo "Waiting for results..."

# Poll until the query finishes (usually 2-10 seconds for an hour of logs)
while true; do
  STATUS=$(aws logs get-query-results \
    --query-id "${QUERY_ID}" \
    --query 'status' \
    --output text \
    --region us-east-1)

  if [ "${STATUS}" == "Complete" ]; then
    echo "Query complete. Results:"
    # Fetch and pretty-print the results
    aws logs get-query-results \
      --query-id "${QUERY_ID}" \
      --query 'results[*].[?field==`endpoint`].value | [0] | join(`,`, @)' \
      --output json \
      --region us-east-1
    break
  elif [ "${STATUS}" == "Failed" ]; then
    echo "[ERROR] Query failed. Check query syntax and log group name."
    exit 1
  fi

  sleep 2
done

{
  "Comment": "CloudWatch Dashboard definition for the checkout service.",
  "Comment2": "Deploy with: aws cloudwatch put-dashboard --dashboard-name checkout-production --dashboard-body file://service_dashboard.json",
  "widgets": [
    {
      "type": "metric",
      "x": 0, "y": 0, "width": 8, "height": 6,
      "properties": {
        "title": "Request Rate — Checkout Lambda (invocations/min)",
        "view": "timeSeries",
        "stat": "Sum",
        "period": 60,
        "metrics": [
          [
            "AWS/Lambda",
            "Invocations",
            "FunctionName", "checkout-processor"
          ]
        ],
        "region": "us-east-1"
      }
    },
    {
      "type": "metric",
      "x": 8, "y": 0, "width": 8, "height": 6,
      "properties": {
        "title": "Error Rate — Checkout Lambda (errors/min)",
        "view": "timeSeries",
        "stat": "Sum",
        "period": 60,
        "metrics": [
          [
            "AWS/Lambda",
            "Errors",
            "FunctionName", "checkout-processor",
            { "color": "#d62728" }
          ]
        ],
        "region": "us-east-1",
        "annotations": {
          "horizontal": [
            {
              "label": "Alarm threshold",
              "value": 5,
              "color": "#ff7f0e"
            }
          ]
        }
      }
    },
    {
      "type": "metric",
      "x": 16, "y": 0, "width": 8, "height": 6,
      "properties": {
        "title": "P99 Latency — Checkout Lambda (ms)",
        "view": "timeSeries",
        "stat": "p99",
        "period": 60,
        "metrics": [
          [
            "AWS/Lambda",
            "Duration",
            "FunctionName", "checkout-processor"
          ]
        ],
        "region": "us-east-1"
      }
    },
    {
      "type": "alarm",
      "x": 0, "y": 6, "width": 24, "height": 2,
      "properties": {
        "title": "Active Alarms — Checkout Service",
        "alarms": [
          "arn:aws:cloudwatch:us-east-1:123456789012:alarm:checkout-processor-high-error-rate"
        ]
      }
    }
  ]
}