Train Test Split & Cross Validation Explained — With Real Code

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score

# Load a real-world dataset: breast cancer classification
# 569 samples, 30 features, binary target (malignant / benign)
cancer_data = load_breast_cancer()
features = cancer_data.data      # shape: (569, 30)
targets   = cancer_data.target   # shape: (569,)

# Split: 80% train, 20% test
# random_state=42 locks the shuffle so results are reproducible
# stratify=targets ensures BOTH splits have the same class ratio
#   (critical for imbalanced datasets — without this, your test set
#    might accidentally contain mostly one class)
X_train, X_test, y_train, y_test = train_test_split(
    features,
    targets,
    test_size=0.20,
    random_state=42,
    stratify=targets
)

print(f"Training samples : {X_train.shape[0]}")
print(f"Test samples     : {X_test.shape[0]}")
print(f"Train class dist : {np.bincount(y_train) / len(y_train)}")
print(f"Test class dist  : {np.bincount(y_test)  / len(y_test)}")

# Train the model — it ONLY sees X_train and y_train
forest_model = RandomForestClassifier(n_estimators=100, random_state=42)
forest_model.fit(X_train, y_train)

# Evaluate — the model sees X_test for the first time here
train_accuracy = accuracy_score(y_train, forest_model.predict(X_train))
test_accuracy  = accuracy_score(y_test,  forest_model.predict(X_test))

print(f"\nTrain accuracy : {train_accuracy:.4f}")
print(f"Test accuracy  : {test_accuracy:.4f}")
print(f"Gap (overfit signal): {train_accuracy - test_accuracy:.4f}")

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

cancer_data = load_breast_cancer()
features = cancer_data.data
targets   = cancer_data.target

# Wrap scaler + model in a Pipeline so the scaler ONLY learns
# from training folds — never from the test fold.
# This is critical: scaling before splitting leaks test set statistics.
evaluation_pipeline = Pipeline([
    ('scaler', StandardScaler()),           # fit_transform on train fold, transform on test fold
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42))
])

# StratifiedKFold preserves class balance in every fold
# shuffle=True prevents ordering bias (e.g., if data was sorted by class)
stratified_kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# cross_val_score handles the entire loop: split -> fit -> score -> repeat
fold_scores = cross_val_score(
    evaluation_pipeline,
    features,
    targets,
    cv=stratified_kfold,
    scoring='accuracy'
)

print("Per-fold accuracy scores:")
for fold_index, fold_score in enumerate(fold_scores, start=1):
    print(f"  Fold {fold_index}: {fold_score:.4f}")

print(f"\nMean accuracy : {fold_scores.mean():.4f}")
print(f"Std deviation : {fold_scores.std():.4f}")
print(f"95% CI estimate: ({fold_scores.mean() - 2*fold_scores.std():.4f}, "
      f"{fold_scores.mean() + 2*fold_scores.std():.4f})")

# High std deviation is a red flag — it means your model is unstable
# and performance depends heavily on which samples it sees
if fold_scores.std() > 0.03:
    print("\n⚠  High variance detected — consider more data or simpler model")
else:
    print("\n✓  Low variance — model generalises consistently across folds")

import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import (
    train_test_split, GridSearchCV, StratifiedKFold, cross_val_score
)
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report

cancer_data = load_breast_cancer()
features = cancer_data.data
targets   = cancer_data.target

# ── STEP 1 ──────────────────────────────────────────────────
# Lock away the final test set immediately. Do NOT touch it again
# until the very end. This is your "exam paper in a sealed envelope".
X_develop, X_final_test, y_develop, y_final_test = train_test_split(
    features, targets,
    test_size=0.20,
    random_state=42,
    stratify=targets
)

# ── STEP 2 ──────────────────────────────────────────────────
# Build pipeline (scaler must be inside to prevent leakage)
model_pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('classifier', RandomForestClassifier(random_state=42))
])

# ── STEP 3 ──────────────────────────────────────────────────
# Hyperparameter search using inner 5-fold CV on the development set ONLY
hyperparam_grid = {
    'classifier__n_estimators': [50, 100, 200],
    'classifier__max_depth':    [None, 10, 20]
}

inner_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

grid_search = GridSearchCV(
    model_pipeline,
    hyperparam_grid,
    cv=inner_cv,          # inner CV: tunes hyperparameters
    scoring='accuracy',
    n_jobs=-1,
    refit=True            # refit best model on full development set
)
grid_search.fit(X_develop, y_develop)

print("Best hyperparameters found:")
print(f"  {grid_search.best_params_}")
print(f"\nBest CV score (development set): {grid_search.best_score_:.4f}")

# ── STEP 4 ──────────────────────────────────────────────────
# NOW — and only now — open the sealed envelope and evaluate
best_model = grid_search.best_estimator_
final_predictions = best_model.predict(X_final_test)

print("\n── Final Held-Out Test Set Results ──")
print(classification_report(
    y_final_test,
    final_predictions,
    target_names=cancer_data.target_names
))
print("This score is your honest, reportable model performance.")