Bias-Variance Tradeoff — Diagnosing Why More Data Fails

import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# Reproducibility — always set a seed when demonstrating stochastic behaviour
np.random.seed(42)

# --- Generate synthetic data with a known underlying pattern ---
# True relationship: a gentle curve (cubic), plus some irreducible noise
n_samples = 80
X_all = np.linspace(-3, 3, n_samples)
true_signal = 0.5 * X_all**3 - X_all**2 + 2  # the ground truth we're trying to learn
irreducible_noise = np.random.normal(0, 2.5, n_samples)  # noise no model can remove
y_all = true_signal + irreducible_noise

# Reshape X for sklearn — it expects a 2D array
X_all = X_all.reshape(-1, 1)

# --- Split into training and test sets manually so we can control the story ---
split_index = 55
X_train, y_train = X_all[:split_index], y_all[:split_index]
X_test, y_test = X_all[split_index:], y_all[split_index:]

# --- Build three models of increasing complexity ---
model_configs = [
    {"degree": 1, "label": "Degree 1 (High Bias — Underfitting)"},
    {"degree": 3, "label": "Degree 3 (Sweet Spot)"},
    {"degree": 15, "label": "Degree 15 (High Variance — Overfitting)"},
]

fig, axes = plt.subplots(1, 3, figsize=(18, 5))
X_plot = np.linspace(-3, 3, 300).reshape(-1, 1)  # smooth curve for plotting

for ax, config in zip(axes, model_configs):
    model = Pipeline([
        ("poly_features", PolynomialFeatures(degree=config["degree"], include_bias=False)),
        ("linear_regression", LinearRegression())
    ])

    model.fit(X_train, y_train)

    # Predict on both sets to expose the bias-variance story
    train_predictions = model.predict(X_train)
    test_predictions = model.predict(X_test)

    train_mse = mean_squared_error(y_train, train_predictions)
    test_mse = mean_squared_error(y_test, test_predictions)

    print(f"\n{config['label']}")
    print(f"  Training MSE : {train_mse:.2f}")
    print(f"  Test MSE     : {test_mse:.2f}")
    print(f"  Gap (variance signal): {test_mse - train_mse:.2f}")

    smooth_predictions = model.predict(X_plot)
    ax.scatter(X_train, y_train, color="steelblue", alpha=0.6, s=20, label="Training data")
    ax.scatter(X_test, y_test, color="tomato", alpha=0.6, s=20, label="Test data")
    ax.plot(X_plot, smooth_predictions, color="black", linewidth=2, label="Model fit")
    ax.set_title(config["label"], fontsize=10)
    ax.set_ylim(-20, 20)
    ax.legend(fontsize=7)

plt.suptitle("Bias vs Variance: Three Models, Same Data", fontsize=14, fontweight="bold")
plt.tight_layout()
plt.savefig("bias_variance_demo.png", dpi=120)

package io.thecodeforge.ml;

import java.util.logging.Logger;

/**
 * Automates the detection of Overfitting (High Variance) and Underfitting (High Bias)
 * in the CI/CD pipeline.
 */
public class ModelHealthGuard {
    private static final Logger logger = Logger.getLogger(ModelHealthGuard.class.getName());
    
    // Thresholds tuned based on historical benchmarks for this dataset
    private static final double VARIANCE_GAP_THRESHOLD = 0.15;
    private static final double MIN_ACCEPTABLE_ACCURACY = 0.70;

    public void runHealthAudit(double trainScore, double valScore) {
        double gap = Math.abs(trainScore - valScore);

        if (trainScore < MIN_ACCEPTABLE_ACCURACY && valScore < MIN_ACCEPTABLE_ACCURACY) {
            logger.severe("STATUS: HIGH BIAS detected. Model is too simple to capture signal.");
            suggestFix("Increase model complexity or reduce regularization alpha.");
        } else if (gap > VARIANCE_GAP_THRESHOLD) {
            logger.warning("STATUS: HIGH VARIANCE detected. Gap is " + (gap * 100) + "%");
            suggestFix("Add more training data, apply L2 regularization, or use Dropout.");
        } else {
            logger.info("STATUS: OPTIMAL. Model generalization within acceptable limits.");
        }
    }

    private void suggestFix(String fix) {
        System.out.println("Forge Recommendation: " + fix);
    }

    public static void main(String[] args) {
        ModelHealthGuard guard = new ModelHealthGuard();
        // Example of a model failing due to High Variance
        guard.runHealthAudit(0.98, 0.72);
    }
}

import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

# Implementation of Learning Curve diagnostic to decouple Bias from Variance
def compute_learning_curve(model, X_train_full, y_train_full, X_val, y_val):
    training_sizes = range(10, len(X_train_full), 5)
    train_errors, val_errors = [], []

    for size in training_sizes:
        X_subset, y_subset = X_train_full[:size], y_train_full[:size]
        model.fit(X_subset, y_subset)

        train_mse = mean_squared_error(y_subset, model.predict(X_subset))
        val_mse = mean_squared_error(y_val, model.predict(X_val))

        train_errors.append(train_mse)
        val_errors.append(val_mse)

    return list(training_sizes), train_errors, val_errors

from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler

# io.thecodeforge best practice: Scale features before regularization
ridge_pipeline = Pipeline([
    ("poly", PolynomialFeatures(degree=12, include_bias=False)),
    ("scaler", StandardScaler()),
    ("ridge", Ridge(alpha=10.0)) # Alpha controls the trade-off
])

ridge_pipeline.fit(X_train, y_train)
print(f"Regularized Test MSE: {mean_squared_error(y_test, ridge_pipeline.predict(X_test)):.2f}")

from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBRegressor
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# io.thecodeforge best practice: Compare ensemble vs simple models on the same data
rf = RandomForestRegressor(n_estimators=100, random_state=42)
xgb = XGBRegressor(n_estimators=100, learning_rate=0.1, random_state=42)
linear = LinearRegression()

for name, model in [('Linear (high bias)', linear), ('Random Forest (variance reduction)', rf), ('XGBoost (bias reduction)', xgb)]:
    model.fit(X_train, y_train)
    train_mse = mean_squared_error(y_train, model.predict(X_train))
    test_mse = mean_squared_error(y_test, model.predict(X_test))
    print(f"{name}: Train MSE = {train_mse:.2f}, Test MSE = {test_mse:.2f}, Gap = {test_mse - train_mse:.2f}")