Logistic Regression — The Threshold That Missed 3% Cancers

import numpy as np
import matplotlib.pyplot as plt

def sigmoid(z):
    """The core of logistic regression — maps any real number to (0, 1)."""
    return 1 / (1 + np.exp(-z))

# Create a range of z values to visualise the S-curve
z_values = np.linspace(-10, 10, 300)
probabilities = sigmoid(z_values)

# Annotate key points so the behaviour is obvious
key_points = {
    -5: sigmoid(-5),   # Very likely class 0
     0: sigmoid(0),    # Exactly on the decision boundary
     5: sigmoid(5),    # Very likely class 1
}

print("=== Sigmoid Output at Key Z-Values ===")
for z, prob in key_points.items():
    label = "→ class 1" if prob >= 0.5 else "→ class 0"
    print(f"  z = {z:+d}  |  P(y=1) = {prob:.4f}  {label}")

# Plot the S-curve
plt.figure(figsize=(8, 4))
plt.plot(z_values, probabilities, color='steelblue', linewidth=2.5, label='σ(z)')
plt.axhline(y=0.5, color='tomato', linestyle='--', linewidth=1.5, label='Decision boundary (0.5)')
plt.axvline(x=0,   color='gray',   linestyle=':',  linewidth=1.2)
plt.fill_between(z_values, probabilities, 0.5,
                 where=(probabilities >= 0.5), alpha=0.12, color='steelblue', label='Predict class 1')
plt.fill_between(z_values, probabilities, 0.5,
                 where=(probabilities < 0.5),  alpha=0.12, color='tomato',    label='Predict class 0')
plt.xlabel('z  (linear combination of features)')
plt.ylabel('Predicted Probability')
plt.title('The Sigmoid Function — How Logistic Regression Converts Scores to Probabilities')
plt.legend()
plt.tight_layout()
plt.savefig('sigmoid_curve.png', dpi=150)
print("\nPlot saved to sigmoid_curve.png")

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    classification_report,
    confusion_matrix,
    roc_auc_score
)

# ── 1. Load Data ──────────────────────────────────────────────────────────────
cancer_data = load_breast_cancer()
feature_matrix = cancer_data.data        # Shape: (569, 30)
target_labels  = cancer_data.target      # 0 = malignant, 1 = benign
feature_names  = cancer_data.feature_names

print(f"Dataset shape : {feature_matrix.shape}")
print(f"Class balance : {np.bincount(target_labels)} (malignant, benign)")

# ── 2. Train / Test Split ─────────────────────────────────────────────────────
# stratify= ensures both splits keep the same class ratio
(X_train, X_test,
 y_train, y_test) = train_test_split(
    feature_matrix, target_labels,
    test_size=0.20,
    random_state=42,
    stratify=target_labels
)

# ── 3. Feature Scaling — critical for gradient-descent-based models ───────────
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)   # fit only on training data!
X_test_scaled  = scaler.transform(X_test)         # apply same scale to test

# ── 4. Train the Model ───────────────────────────────────────────────────────
# max_iter=1000 because the default 100 often hits a ConvergenceWarning
logistic_model = LogisticRegression(max_iter=1000, random_state=42)
logistic_model.fit(X_train_scaled, y_train)

# ── 5. Predict & Evaluate ────────────────────────────────────────────────────
y_pred_labels = logistic_model.predict(X_test_scaled)
y_pred_proba  = logistic_model.predict_proba(X_test_scaled)[:, 1]  # P(benign)

print("\n=== Confusion Matrix ===")
cm = confusion_matrix(y_test, y_pred_labels)
print(f"  True Negatives  (Malignant correctly caught) : {cm[0,0]}")
print(f"  False Positives (Malignant missed as Benign) : {cm[0,1]}")
print(f"  False Negatives (Benign wrongly flagged)     : {cm[1,0]}")
print(f"  True Positives  (Benign correctly caught)    : {cm[1,1]}")

print("\n=== Classification Report ===")
print(classification_report(y_test, y_pred_labels,
                            target_names=['Malignant', 'Benign']))

roc_auc = roc_auc_score(y_test, y_pred_proba)
print(f"ROC-AUC Score : {roc_auc:.4f}")

# ── 6. Inspect Coefficients — this is where Logistic Regression shines ───────
print("\n=== Top 5 Features Pushing Towards Malignant (negative coefficients) ===")
coef_pairs = sorted(
    zip(feature_names, logistic_model.coef_[0]),
    key=lambda pair: pair[1]
)
for feature_name, coefficient in coef_pairs[:5]:
    print(f"  {feature_name:<35}  coef = {coefficient:+.4f}")

print("\n=== Top 5 Features Pushing Towards Benign (positive coefficients) ===")
for feature_name, coefficient in coef_pairs[-5:][::-1]:
    print(f"  {feature_name:<35}  coef = {coefficient:+.4f}")

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import precision_recall_curve, roc_curve
import matplotlib.pyplot as plt

# ── Reuse the trained model setup from the previous example ──────────────────
cancer_data    = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    cancer_data.data, cancer_data.target,
    test_size=0.20, random_state=42, stratify=cancer_data.target
)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled  = scaler.transform(X_test)
logistic_model = LogisticRegression(max_iter=1000, random_state=42)
logistic_model.fit(X_train_scaled, y_train)

# Predicted probabilities for the positive class (benign = 1)
y_proba_benign = logistic_model.predict_proba(X_test_scaled)[:, 1]

# ── Find threshold that maximises recall for MALIGNANT class ─────────────────
# Note: precision_recall_curve works with respect to the positive label.
# We flip the probabilities so 'malignant' becomes the positive class.
y_proba_malignant = 1 - y_proba_benign
y_test_malignant  = 1 - y_test        # 1 = malignant, 0 = benign (flipped)

precisions, recalls, thresholds = precision_recall_curve(
    y_test_malignant, y_proba_malignant
)

# We want recall >= 0.99 with the highest possible precision
high_recall_mask = recalls[:-1] >= 0.99   # exclude last point (no threshold)
candidates = list(zip(
    thresholds[high_recall_mask],
    precisions[:-1][high_recall_mask],
    recalls[:-1][high_recall_mask]
))

print("=== Threshold Candidates Achieving ≥99% Recall on Malignant Class ===")
print(f"  {'Threshold':>12}  {'Precision':>10}  {'Recall':>8}")
for thresh, prec, rec in candidates:
    print(f"  {thresh:>12.4f}  {prec:>10.4f}  {rec:>8.4f}")

# Pick the threshold with highest precision among our high-recall candidates
best_threshold, best_precision, best_recall = max(candidates, key=lambda t: t[1])
print(f"\n✔ Best threshold = {best_threshold:.4f}")
print(f"  At this threshold — Precision: {best_precision:.4f}, Recall: {best_recall:.4f}")

# ── Apply the chosen threshold and see its real-world impact ─────────────────
# We predict 'malignant' whenever P(malignant) >= best_threshold
y_pred_tuned = (y_proba_malignant >= best_threshold).astype(int)

malignant_actual   = np.sum(y_test_malignant == 1)
malignant_caught   = np.sum((y_pred_tuned == 1) & (y_test_malignant == 1))
malignant_missed   = malignant_actual - malignant_caught

print(f"\n=== Clinical Impact at Tuned Threshold ===")
print(f"  Total malignant tumours in test set : {malignant_actual}")
print(f"  Correctly flagged (True Positives)  : {malignant_caught}")
print(f"  Missed (False Negatives)            : {malignant_missed}  ← the dangerous ones")

# ── ROC Curve ─────────────────────────────────────────────────────────────────
fpr, tpr, roc_thresholds = roc_curve(y_test_malignant, y_proba_malignant)

plt.figure(figsize=(6, 5))
plt.plot(fpr, tpr, color='steelblue', lw=2, label='ROC Curve')
plt.plot([0, 1], [0, 1], color='gray', linestyle='--', label='Random classifier')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate (Recall)')
plt.title('ROC Curve — Malignant Detection')
plt.legend()
plt.tight_layout()
plt.savefig('roc_curve.png', dpi=150)
print("\nROC curve saved to roc_curve.png")

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import log_loss

# Use the same breast cancer data
cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, test_size=0.2, random_state=42, stratify=cancer.target
)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Train model with different regularisation strengths
for C in [0.01, 0.1, 1, 10, 100]:
    model = LogisticRegression(C=C, max_iter=1000, random_state=42)
    model.fit(X_train, y_train)
    y_pred_train_proba = model.predict_proba(X_train)[:, 1]
    y_pred_test_proba = model.predict_proba(X_test)[:, 1]
    train_loss = log_loss(y_train, y_pred_train_proba)
    test_loss = log_loss(y_test, y_pred_test_proba)
    acc = model.score(X_test, y_test)
    print(f"C={C:>6.2f} | Train log-loss: {train_loss:.4f} | Test log-loss: {test_loss:.4f} | Test Acc: {acc:.4f}")

# Observe: as C increases (less regularisation), train loss decreases, test loss may start increasing (overfitting).

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import StandardScaler

cancer = load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(
    cancer.data, cancer.target, test_size=0.2, random_state=42, stratify=cancer.target
)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Compare L1 and L2 with same C
models = {
    'L2 (default)': LogisticRegression(penalty='l2', C=1.0, solver='lbfgs', max_iter=1000),
    'L1 (lasso)': LogisticRegression(penalty='l1', C=1.0, solver='saga', max_iter=1000),
    'ElasticNet (l1_ratio=0.5)': LogisticRegression(penalty='elasticnet', C=1.0, solver='saga', l1_ratio=0.5, max_iter=1000),
}

for name, model in models.items():
    model.fit(X_train, y_train)
    nonzero_coefs = np.sum(np.abs(model.coef_) > 1e-10)
    test_acc = model.score(X_test, y_test)
    print(f"{name:20} | Non-zero coefficients: {nonzero_coefs:2d} | Test accuracy: {test_acc:.4f}")

# Output shows L1 produces sparser models (fewer non-zero coefficients).