Confusion Matrix — Why 99% Accuracy Missed Every Fraud Case

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

# Simulated ground truth labels for 20 fraud detection predictions
# 1 = fraud, 0 = legitimate
actual_labels = [0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
                 0, 1, 0, 1, 0, 0, 1, 1, 0, 1]

# What our model predicted for each of those 20 transactions
predicted_labels = [0, 1, 0, 0, 1, 1, 0, 1, 1, 0,
                    0, 0, 0, 1, 0, 1, 1, 1, 0, 0]

# Build the confusion matrix
# sklearn orders as: [[TN, FP], [FN, TP]] by default
cm = confusion_matrix(actual_labels, predicted_labels)

# Pull out each cell so we can label them clearly
tn, fp, fn, tp = cm.ravel()

print("=== Confusion Matrix (raw counts) ===")
print(f"True Negatives  (TN): {tn}  — Legit transactions correctly cleared")
print(f"False Positives (FP): {fp}  — Legit transactions wrongly flagged (false alarm)")
print(f"False Negatives (FN): {fn}  — Fraud transactions we missed (dangerous!)")
print(f"True Positives  (TP): {tp}  — Fraud transactions correctly caught")
print()
print("Raw confusion matrix:")
print(cm)

# Visualise it as a heatmap — much easier to read at a glance
fig, ax = plt.subplots(figsize=(6, 5))
disp = ConfusionMatrixDisplay(
    confusion_matrix=cm,
    display_labels=["Legitimate", "Fraud"]
)
disp.plot(ax=ax, cmap="Blues", colorbar=False)
ax.set_title("Fraud Detection — Confusion Matrix", fontsize=14, pad=12)
plt.tight_layout()
plt.savefig("confusion_matrix_fraud.png", dpi=150)
print("Heatmap saved to confusion_matrix_fraud.png")

from sklearn.metrics import (
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    classification_report
)

# Same fraud detection scenario as before
actual_labels    = [0, 1, 1, 0, 1, 0, 0, 1, 1, 0,
                    0, 1, 0, 1, 0, 0, 1, 1, 0, 1]
predicted_labels = [0, 1, 0, 0, 1, 1, 0, 1, 1, 0,
                    0, 0, 0, 1, 0, 1, 1, 1, 0, 0]

# --- Calculate every metric manually first so we understand what's happening ---
# pos_label=1 means we treat 'Fraud' as the positive class we care about
accuracy  = accuracy_score(actual_labels, predicted_labels)
precision = precision_score(actual_labels, predicted_labels, pos_label=1)
recall    = recall_score(actual_labels, predicted_labels, pos_label=1)
f1        = f1_score(actual_labels, predicted_labels, pos_label=1)

print("=== Individual Metrics (Fraud = Positive Class) ===")
print(f"Accuracy : {accuracy:.2%}  — {int(accuracy*20)}/20 predictions correct overall")
print(f"Precision: {precision:.2%}  — Of {int(precision**-1 * recall * 10):.0f} fraud alerts, this fraction were real fraud")
print(f"Recall   : {recall:.2%}  — Of all actual fraud, this fraction was caught")
print(f"F1-Score : {f1:.2%}  — Balanced measure (harmonic mean of precision & recall)")
print()

# --- sklearn's classification_report is your best friend in production ---
# It gives you precision, recall, F1 for EACH class plus macro/weighted averages
report = classification_report(
    actual_labels,
    predicted_labels,
    target_names=["Legitimate", "Fraud"],
    digits=3
)
print("=== Full Classification Report ===")
print(report)

# --- Illustrate why accuracy is misleading on imbalanced data ---
print("=== The Accuracy Trap — Imbalanced Dataset Demonstration ===")

# Imagine 1000 transactions: 990 legit, 10 fraudulent (realistic ratio)
imbalanced_actual    = [0] * 990 + [1] * 10
# A dumb model that ALWAYS predicts 'legitimate'
dumb_model_predicted = [0] * 1000

dumb_accuracy  = accuracy_score(imbalanced_actual, dumb_model_predicted)
dumb_recall    = recall_score(imbalanced_actual, dumb_model_predicted,
                              pos_label=1, zero_division=0)
dumb_f1        = f1_score(imbalanced_actual, dumb_model_predicted,
                          pos_label=1, zero_division=0)

print(f"Dumb model accuracy : {dumb_accuracy:.2%}  ← looks great!")
print(f"Dumb model recall   : {dumb_recall:.2%}  ← caught ZERO fraud cases")
print(f"Dumb model F1-score : {dumb_f1:.2%}  ← tells the real story")

from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import (
    precision_score, recall_score, f1_score,
    classification_report, confusion_matrix
)
import numpy as np

# --- Scenario: Medical test — detecting a rare condition (5% prevalence) ---
np.random.seed(42)

# Generate a deliberately imbalanced binary classification dataset
# weights=[0.95, 0.05] means 95% healthy, 5% have the condition
features, labels = make_classification(
    n_samples=2000,
    n_features=10,
    n_informative=6,
    weights=[0.95, 0.05],   # heavy class imbalance
    flip_y=0.02,            # a little noise to make it realistic
    random_state=42
)

train_features, test_features, train_labels, test_labels = train_test_split(
    features, labels,
    test_size=0.25,
    stratify=labels,        # preserve the 95/5 ratio in both splits
    random_state=42
)

# Train a simple logistic regression model
medical_model = LogisticRegression(max_iter=1000, random_state=42)
medical_model.fit(train_features, train_labels)

# Default predictions (threshold = 0.5)
default_predictions = medical_model.predict(test_features)

# Lower threshold to 0.3 — the model now flags 'positive' at lower confidence
# This trades precision for recall, which makes sense for medical screening
positive_probabilities = medical_model.predict_proba(test_features)[:, 1]
lowered_threshold_predictions = (positive_probabilities >= 0.3).astype(int)

print("=== Medical Screening Model — Default Threshold (0.5) ===")
print(classification_report(
    test_labels,
    default_predictions,
    target_names=["Healthy", "Has Condition"],
    digits=3
))

print("=== Medical Screening Model — Lowered Threshold (0.3) ===")
print(classification_report(
    test_labels,
    lowered_threshold_predictions,
    target_names=["Healthy", "Has Condition"],
    digits=3
))

# Show the trade-off explicitly — this is what you'd put in a model card
for threshold, preds in [(0.5, default_predictions),
                          (0.3, lowered_threshold_predictions)]:
    prec  = precision_score(test_labels, preds, pos_label=1, zero_division=0)
    rec   = recall_score(test_labels, preds, pos_label=1, zero_division=0)
    f1    = f1_score(test_labels, preds, pos_label=1, zero_division=0)
    fn_count = confusion_matrix(test_labels, preds).ravel()[2]  # FN cell
    print(f"Threshold={threshold} | Precision={prec:.3f} | Recall={rec:.3f} "
          f"| F1={f1:.3f} | Missed cases (FN)={fn_count}")

from sklearn.metrics import classification_report, confusion_matrix
import numpy as np

# Simulated 3-class classification: news categories
# 0 = Sports, 1 = Politics, 2 = Technology
y_true = [0, 1, 2, 0, 1, 2, 0, 0, 1, 2,
          0, 1, 2, 1, 0, 2, 1, 0, 2, 1]
y_pred = [0, 1, 2, 0, 1, 1, 0, 0, 2, 2,
          0, 1, 2, 1, 0, 2, 2, 0, 2, 1]

target_names = ['Sports', 'Politics', 'Technology']

print("=== Confusion Matrix (3 classes) ===")
cm = confusion_matrix(y_true, y_pred)
print(cm)
print()

print("=== Classification Report ===")
# Reports per-class precision, recall, f1, support
# Also macro avg and weighted avg
report = classification_report(y_true, y_pred, target_names=target_names, digits=3)
print(report)

# Demonstrate averaging differences manually
def precision_recall_f1_per_class(y_true, y_pred, labels=None):
    cm = confusion_matrix(y_true, y_pred, labels=labels)
    tp = np.diag(cm)
    fp = cm.sum(axis=0) - tp
    fn = cm.sum(axis=1) - tp
    prec = np.where((tp+fp)==0, 0, tp/(tp+fp))
    rec = np.where((tp+fn)==0, 0, tp/(tp+fn))
    f1 = np.where((prec+rec)==0, 0, 2*prec*rec/(prec+rec))
    return prec, rec, f1

prec, rec, f1 = precision_recall_f1_per_class(y_true, y_pred)
print("\n=== Manual Per-Class Metrics ===")
for i, name in enumerate(target_names):
    print(f"{name:15s}: Precision={prec[i]:.3f}, Recall={rec[i]:.3f}, F1={f1[i]:.3f}")

macro_f1 = np.mean(f1)
weighted_f1 = np.mean(f1 * np.bincount(y_true)[:3])
print(f"\nMacro F1: {macro_f1:.3f} (all classes equal weight)")
print(f"Weighted F1: {weighted_f1:.3f} (weighted by actual class distribution)")

import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    precision_recall_curve,
    auc,
    f1_score
)
import matplotlib.pyplot as plt

# Generate imbalanced dataset
X, y = make_classification(n_samples=1000, n_classes=2,
                           weights=[0.9, 0.1], flip_y=0.05,
                           random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y,
    test_size=0.3, stratify=y, random_state=42)

model = LogisticRegression(max_iter=1000, class_weight='balanced')
model.fit(X_train, y_train)
y_scores = model.predict_proba(X_test)[:, 1]

# Compute precision-recall curve
precisions, recalls, thresholds = precision_recall_curve(y_test, y_scores)

# Find threshold that gives recall >= 0.80 with highest precision
min_recall = 0.80
valid_indices = np.where(recalls[:-1] >= min_recall)[0]
if len(valid_indices) > 0:
    best_idx = valid_indices[np.argmax(precisions[valid_indices])]
    best_threshold = thresholds[best_idx]
    print(f"Best threshold for recall >= {min_recall}: {best_threshold:.3f}")
    print(f"  -> Precision = {precisions[best_idx]:.3f}, Recall = {recalls[best_idx]:.3f}")
else:
    print(f"No threshold can achieve recall >= {min_recall}")

# Area under PR curve
pr_auc = auc(recalls, precisions)
print(f"AUC-PR: {pr_auc:.3f}")

# Plot
plt.figure(figsize=(8,6))
plt.plot(recalls, precisions, label=f'Logistic Regression (AUC-PR = {pr_auc:.3f})', lw=2)
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.legend(loc='best')
plt.grid(alpha=0.3)
plt.tight_layout()
plt.savefig('pr_curve_threshold_tuning.png', dpi=150)
print("Figure saved.")