Control-M Ended Not OK: Find the Real Failure and Rerun Safely

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Simulate a job that fails after partial work, then inspect its output.

JOB_LOG="/tmp/controlm_job_$$.log"

# Simulate a job that writes a file then fails on a permission check
echo "Starting data export..." > "$JOB_LOG"
echo "Wrote 500 rows to /data/export.csv" >> "$JOB_LOG"
# This line fails because /root is not writable by the job user
touch /root/export.done 2>> "$JOB_LOG"
EXIT_CODE=$?

echo "Job exited with code $EXIT_CODE" >> "$JOB_LOG"

# The operator sees 'Ended Not OK' but the real error is here:
echo "--- Job output ---"
cat "$JOB_LOG"

# Cleanup
rm -f "$JOB_LOG"

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Demonstrate the difference between Control-M log and job sysout.

# Simulate a job that fails with a database connection error
cat > /tmp/db_job.sh <<'EOF'
#!/bin/bash
echo "Connecting to database..."
# Simulate a connection failure
echo "ERROR: ORA-12170: TNS:Connect timeout occurred" >&2
exit 1
EOF
chmod +x /tmp/db_job.sh

# Run the job and capture both stdout and stderr
/tmp/db_job.sh > /tmp/job_sysout.log 2>&1
EXIT_CODE=$?

echo "Control-M log would show: Job ended with exit code $EXIT_CODE"
echo "Sysout contains the real error:"
cat /tmp/job_sysout.log

rm -f /tmp/db_job.sh /tmp/job_sysout.log

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Simulate tracing a missing condition with ctmcontb (conceptual).

# In real Control-M, you'd run:
# ctmcontb -LIST CONDITION_NAME

# Here's a mock output showing a condition that was never set:
echo "Condition: DAILY_SALES_LOADED"
echo "Status: Not found"
echo "Expected by job: SALES_REPORT (downstream)"
echo "Upstream job: SALES_LOAD (should set condition)"
echo "Upstream job status: Ended Not OK at 01:45"

echo ""
echo "Diagnosis: Upstream job failed, so condition was never set."
echo "Fix: Rerun SALES_LOAD first, then SALES_REPORT."

# io.thecodeforge — DevOps tutorial
# Idempotent payment job: safe to rerun without duplicating transfers.

import sqlite3
from datetime import date

DB_PATH = "/tmp/payments.db"

def init_db():
    conn = sqlite3.connect(DB_PATH)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS transfers (
            idempotency_key TEXT PRIMARY KEY,
            employee_id TEXT,
            amount REAL,
            transfer_date TEXT
        )
    """)
    conn.commit()
    return conn

def transfer_payment(conn, employee_id, amount):
    # Build a deterministic key: employee + date + amount
    key = f"{employee_id}-{date.today().isoformat()}-{amount}"
    try:
        conn.execute(
            "INSERT INTO transfers (idempotency_key, employee_id, amount, transfer_date) VALUES (?, ?, ?, ?)",
            (key, employee_id, amount, date.today().isoformat())
        )
        conn.commit()
        print(f"Transferred {amount} to {employee_id}")
    except sqlite3.IntegrityError:
        # Key already exists: this is a rerun, skip safely
        print(f"Skipping {employee_id}: already transferred today")

if __name__ == "__main__":
    conn = init_db()
    # First run
    transfer_payment(conn, "EMP001", 5000.0)
    # Simulate rerun
    transfer_payment(conn, "EMP001", 5000.0)
    conn.close()

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Pre-rerun checklist for a Control-M job.

JOB_NAME="PAYROLL_TRANSFER"
RUN_DATE=$(date +%Y-%m-%d)

# 1. Check if the job already wrote data (idempotency check)
echo "Checking for existing transfers for $RUN_DATE..."
# In real life: query your database
# SELECT COUNT(*) FROM transfers WHERE transfer_date = '$RUN_DATE';
EXISTING_COUNT=0  # mock value

if [ "$EXISTING_COUNT" -gt 0 ]; then
    echo "WARNING: $EXISTING_COUNT transfers already exist for $RUN_DATE."
    echo "Do NOT rerun without manual review."
    exit 1
fi

# 2. Clear failure condition if present
echo "Clearing failure condition for $JOB_NAME..."
ctmcontb -DELETE "${JOB_NAME}_FAILED" "$(date +%m%d)"

# 3. Rerun the job by its order number
echo "Rerunning $JOB_NAME..."
ctmpsm -UPDATEAJF "$ORDERNO" RERUN

echo "Rerun initiated. Monitor sysout for errors."

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Fix a script that returns non-zero on success due to grep.

LOG_FILE="/tmp/app.log"
echo "ERROR: something happened" > "$LOG_FILE"

# BAD: grep returns 1 if no match, causing false failure
grep -q "CRITICAL" "$LOG_FILE"
# This would exit 1, marking the job Not OK even though no critical error exists.

# GOOD: capture grep result and exit 0 if no match
grep -q "CRITICAL" "$LOG_FILE"
if [ $? -eq 0 ]; then
    echo "Critical error found!"
    exit 1
else
    echo "No critical errors. Success."
    exit 0
fi

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Simulate checking downstream dependencies after a rerun.

FAILED_JOB="SALES_LOAD"
DOWNSTREAM_JOBS=("SALES_TRANSFORM" "SALES_REPORT")

echo "Rerunning $FAILED_JOB..."
# ctm run order <server> <folder> "$FAILED_JOB"

for job in "${DOWNSTREAM_JOBS[@]}"; do
    echo "Checking status of $job..."
    # In real Control-M: ctm run jobs:status::get -s "jobname=$job"
    # Mock: assume SALES_TRANSFORM already ran, SALES_REPORT did not
    if [ "$job" == "SALES_TRANSFORM" ]; then
        echo "  $job already completed. Skipping rerun to avoid duplication."
    else
        echo "  $job did not run. It will trigger automatically from condition."
    fi
done

#!/bin/bash
# io.thecodeforge — DevOps tutorial
# Detect a deterministic failure that should not be rerun.

ERROR_MSG="ORA-00001: unique constraint (HR.EMP_EMAIL_UK) violated"

echo "Job failed with: $ERROR_MSG"

if echo "$ERROR_MSG" | grep -q "unique constraint"; then
    echo "Deterministic failure detected: duplicate data exists."
    echo "Do NOT rerun. Fix the data first:"
    echo "  DELETE FROM employees WHERE email = 'duplicate@example.com';"
    echo "Then rerun the job."
    exit 1
fi

echo "Transient failure. Safe to rerun."