Regularisation — 99% Accuracy Masked 3x Default Rate

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.metrics import mean_squared_error

np.random.seed(42)

# --- Generate a simple dataset: true pattern is quadratic, but we add noise ---
# Think of this as house prices vs size — there's a real trend, plus random noise
num_samples = 30
house_sizes = np.linspace(50, 300, num_samples)
true_prices = 0.5 * house_sizes**2 - 50 * house_sizes + 8000  # the real pattern
noise = np.random.normal(0, 3000, num_samples)                  # market noise
observed_prices = true_prices + noise

# Reshape for sklearn (needs 2D input)
house_sizes_2d = house_sizes.reshape(-1, 1)

# --- Fit three models: underfitting, overfitting, and regularised ---

# Degree-1: too simple, misses the curve (underfitting)
linear_model = make_pipeline(PolynomialFeatures(degree=1), LinearRegression())
linear_model.fit(house_sizes_2d, observed_prices)

# Degree-10: so flexible it chases every noise spike (overfitting)
overfitted_model = make_pipeline(PolynomialFeatures(degree=10), LinearRegression())
overfitted_model.fit(house_sizes_2d, observed_prices)

# Degree-10 with Ridge regularisation: flexible but penalised for large weights
ridge_model = make_pipeline(PolynomialFeatures(degree=10), Ridge(alpha=1000))
ridge_model.fit(house_sizes_2d, observed_prices)

# --- Evaluate on training data ---
plot_range = np.linspace(50, 300, 300).reshape(-1, 1)

linear_train_rmse   = mean_squared_error(observed_prices, linear_model.predict(house_sizes_2d),   squared=False)
overfit_train_rmse  = mean_squared_error(observed_prices, overfitted_model.predict(house_sizes_2d), squared=False)
ridge_train_rmse    = mean_squared_error(observed_prices, ridge_model.predict(house_sizes_2d),    squared=False)

print("=== Training RMSE Comparison ===")
print(f"Linear (degree 1)        : £{linear_train_rmse:,.0f}")
print(f"Overfitted (degree 10)   : £{overfit_train_rmse:,.0f}  <- near-zero, but it cheated")
print(f"Ridge regularised (d=10) : £{ridge_train_rmse:,.0f}  <- honest fit")

# Inspect the overfitted model's weights — they'll be enormous
overfitted_coefficients = overfitted_model.named_steps['linearregression'].coef_
ridge_coefficients      = ridge_model.named_steps['ridge'].coef_

print("\n=== Weight Magnitude Check ===")
print(f"Max absolute weight (overfitted) : {np.max(np.abs(overfitted_coefficients)):,.2f}")
print(f"Max absolute weight (Ridge)      : {np.max(np.abs(ridge_coefficients)):,.2f}")
print("\nRegularisation shrank those runaway weights dramatically!")

import numpy as np
from sklearn.linear_model import Ridge, Lasso, ElasticNet
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_regression

np.random.seed(0)

# --- Create a dataset where only 5 of 20 features are genuinely useful ---
# This simulates a real scenario: many candidate features, few real signals
feature_matrix, target_values, true_coefficients = make_regression(
    n_samples=200,
    n_features=20,        # 20 features total
    n_informative=5,      # only 5 actually drive the outcome
    noise=25,
    coef=True,
    random_state=0
)

# IMPORTANT: Always scale features before regularisation!
# Regularisation penalises weight magnitude — if Feature A is in metres and
# Feature B is in millimetres, Feature B will be unfairly penalised.
scaler = StandardScaler()
feature_matrix_scaled = scaler.fit_transform(feature_matrix)

# --- Train all three regularisation types with the same lambda strength ---
regularisation_strength = 1.0

ridge_model    = Ridge(alpha=regularisation_strength)
lasso_model    = Lasso(alpha=regularisation_strength, max_iter=10000)
elastic_model  = ElasticNet(alpha=regularisation_strength, l1_ratio=0.5, max_iter=10000)

ridge_model.fit(feature_matrix_scaled, target_values)
lasso_model.fit(feature_matrix_scaled, target_values)
elastic_model.fit(feature_matrix_scaled, target_values)

# --- Compare how many features each model zeroed out ---
ridge_zeros   = np.sum(np.abs(ridge_model.coef_)   < 0.01)
lasso_zeros   = np.sum(np.abs(lasso_model.coef_)   < 0.01)  # true zeroes
elastic_zeros = np.sum(np.abs(elastic_model.coef_) < 0.01)

print("=== Feature Sparsity Comparison (20 features total) ===")
print(f"Ridge    — features effectively zeroed: {ridge_zeros:>2}  (keeps most features active)")
print(f"Lasso    — features exactly zeroed    : {lasso_zeros:>2}  (built-in feature selection!)")
print(f"ElasticNet — features zeroed          : {elastic_zeros:>2}  (balanced approach)")

# --- Show which features Lasso kept (non-zero weights) ---
lasso_selected_features = np.where(np.abs(lasso_model.coef_) >= 0.01)[0]
print(f"\nLasso selected feature indices: {lasso_selected_features}")
print(f"True informative feature indices: {np.where(np.abs(true_coefficients) > 0)[0]}")

# --- Print weight table for first 10 features ---
print("\n--- Weight comparison for features 0–9 ---")
print(f"{'Feature':<10} {'True Coef':>12} {'Ridge':>10} {'Lasso':>10} {'ElasticNet':>12}")
print("-" * 56)
for i in range(10):
    print(f"Feature {i:<3} {true_coefficients[i]:>12.2f} "
          f"{ridge_model.coef_[i]:>10.2f} "
          f"{lasso_model.coef_[i]:>10.2f} "
          f"{elastic_model.coef_[i]:>12.2f}")

import numpy as np
from sklearn.linear_model import RidgeCV, LassoCV
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error

np.random.seed(7)

# --- Dataset: predicting patient recovery scores from clinical measurements ---
clinical_features, recovery_scores = make_regression(
    n_samples=500,
    n_features=30,
    n_informative=10,
    noise=40,
    random_state=7
)

# Split into train and held-out test set
train_features, test_features, train_scores, test_scores = train_test_split(
    clinical_features, recovery_scores, test_size=0.2, random_state=7
)

# Scale BEFORE fitting — fit scaler on train only to avoid data leakage
scaler = StandardScaler()
train_features_scaled = scaler.fit_transform(train_features)
test_features_scaled  = scaler.transform(test_features)  # transform only, don't refit

# --- Define lambda search space on a log scale ---
# np.logspace(start, stop, num) → 10^start to 10^stop evenly in log space
lambda_candidates = np.logspace(-3, 4, 100)  # 0.001 to 10,000, 100 values

# --- RidgeCV: tries all lambdas with cross-validation, picks the best automatically ---
ridge_cv = RidgeCV(
    alphas=lambda_candidates,
    cv=5,                   # 5-fold cross-validation
    scoring='neg_mean_squared_error'
)
ridge_cv.fit(train_features_scaled, train_scores)

# --- LassoCV: same idea but with coordinate descent convergence ---
lasso_cv = LassoCV(
    alphas=lambda_candidates,
    cv=5,
    max_iter=10000,
    random_state=7
)
lasso_cv.fit(train_features_scaled, train_scores)

# --- Evaluate both on the held-out test set ---
ridge_test_rmse = mean_squared_error(
    test_scores, ridge_cv.predict(test_features_scaled), squared=False
)
lasso_test_rmse = mean_squared_error(
    test_scores, lasso_cv.predict(test_features_scaled), squared=False
)

lasso_active_features = np.sum(np.abs(lasso_cv.coef_) > 0.001)

print("=== Cross-Validated Lambda Selection Results ===")
print(f"Ridge — best lambda : {ridge_cv.alpha_:.4f}")
print(f"Ridge — test RMSE   : {ridge_test_rmse:.3f}")
print()
print(f"Lasso — best lambda : {lasso_cv.alpha_:.4f}")
print(f"Lasso — test RMSE   : {lasso_test_rmse:.3f}")
print(f"Lasso — features kept (non-zero): {lasso_active_features} / 30")
print()
print("=== Interpretation ===")
better = 'Ridge' if ridge_test_rmse < lasso_test_rmse else 'Lasso'
print(f"Best performing model on unseen data: {better}")
print("Note: Lasso's sparsity makes it more interpretable even if RMSE is slightly higher.")

import numpy as np
from sklearn.linear_model import ElasticNetCV
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error

np.random.seed(42)
X, y = make_regression(n_samples=300, n_features=50, n_informative=10, noise=15, random_state=42)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# ElasticNetCV: cross-validates both alpha and l1_ratio
elastic_cv = ElasticNetCV(
    alphas=np.logspace(-3, 3, 50),
    l1_ratio=[0.1, 0.3, 0.5, 0.7, 0.9],
    cv=5,
    max_iter=10000,
    random_state=42
)
elastic_cv.fit(X_train_scaled, y_train)

test_rmse = mean_squared_error(y_test, elastic_cv.predict(X_test_scaled), squared=False)

print("=== Elastic Net CV Results ===")
print(f"Best alpha   : {elastic_cv.alpha_:.4f}")
print(f"Best l1_ratio: {elastic_cv.l1_ratio_:.2f}")
print(f"Test RMSE    : {test_rmse:.3f}")
print(f"Non-zero coefs: {np.sum(np.abs(elastic_cv.coef_) > 0.001)} / 50")

import torch
import torch.nn as nn

# --- PyTorch model with weight decay ---
model = nn.Sequential(
    nn.Linear(100, 64),
    nn.ReLU(),
    nn.Dropout(0.5),       # dropout regularisation
    nn.Linear(64, 32),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(32, 1)
)

# L2 regularisation via weight_decay in optimiser
optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=0.01)

# Keras equivalent:
# model = Sequential([
#     Dense(64, activation='relu', kernel_regularizer=l2(0.01)),
#     Dropout(0.5),
#     Dense(32, activation='relu', kernel_regularizer=l2(0.01)),
#     Dropout(0.3),
#     Dense(1)
# ])

print("Model defined with weight_decay=0.01 and dropout layers.")

import numpy as np
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler

# --- BAD: no scaling, default alpha ---
# model = Ridge(alpha=1.0).fit(X, y)  # WRONG for non-scaled data

# --- GOOD: scale, use CV to find alpha ---
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X_train)

# Baseline: no regularisation
ridge_none = Ridge(alpha=0).fit(X_scaled, y_train)
baseline_rmse = mean_squared_error(y_train, ridge_none.predict(X_scaled), squared=False)

# CV tuned
from sklearn.linear_model import RidgeCV
ridge_cv = RidgeCV(alphas=np.logspace(-3, 4, 100), cv=5).fit(X_scaled, y_train)
optimal_alpha = ridge_cv.alpha_

print(f"Baseline (alpha=0) RMSE: {baseline_rmse:.2f}")
print(f"Optimal alpha from CV: {optimal_alpha:.4f}")
print(f"Improvement: {(baseline_rmse - mean_squared_error(y_train, ridge_cv.predict(X_scaled), squared=False)):.2f}")