Data Preprocessing in ML: The Complete Practical Guide

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split

# --- Simulate a messy real-world dataset ---
np.random.seed(42)
raw_data = pd.DataFrame({
    'age':    [25, np.nan, 47, 31, np.nan, 52, 29, 40],
    'salary': [48000, 72000, np.nan, 61000, 58000, np.nan, 43000, 95000],
    'bought': [0, 1, 1, 0, 1, 1, 0, 1]   # target label
})

print("Raw data:")
print(raw_data)
print(f"\nMissing counts:\n{raw_data.isnull().sum()}")

# --- Step 1: Add binary indicator columns BEFORE imputing ---
# These columns tell the model WHERE data was missing — that pattern has meaning.
for col in ['age', 'salary']:
    raw_data[f'{col}_was_missing'] = raw_data[col].isnull().astype(int)

# --- Step 2: Split BEFORE fitting the imputer ---
# Fitting on all data first would leak test-set statistics into training.
features = raw_data.drop(columns='bought')
labels   = raw_data['bought']

X_train, X_test, y_train, y_test = train_test_split(
    features, labels, test_size=0.25, random_state=42
)

# --- Step 3: Fit imputer on TRAINING data only ---
# strategy='median' is more robust to outliers than 'mean'
numeric_cols = ['age', 'salary']
imputer = SimpleImputer(strategy='median')

# fit_transform on train, transform-only on test
X_train[numeric_cols] = imputer.fit_transform(X_train[numeric_cols])
X_test[numeric_cols]  = imputer.transform(X_test[numeric_cols])   # NO fit here!

print("\nTraining set after imputation:")
print(X_train.round(1))
print("\nImputer medians learned from training data:", imputer.statistics_)

import pandas as pd
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
from sklearn.compose import ColumnTransformer

# --- Dataset with two types of categorical columns ---
clothing_data = pd.DataFrame({
    'city':        ['London', 'Paris', 'Tokyo', 'London', 'Berlin', 'Paris'],
    'size':        ['small', 'large', 'medium', 'large', 'small', 'medium'],
    'price_usd':   [120, 85, 200, 115, 95, 90]
})

print("Original data:")
print(clothing_data)

# 'city' is NOMINAL — no natural order, use One-Hot Encoding
# 'size' is ORDINAL — small < medium < large, order is meaningful

nominal_features = ['city']
ordinal_features  = ['size']
numeric_features  = ['price_usd']

# Define the correct ordering for the ordinal column explicitly
size_order = [['small', 'medium', 'large']]

preprocessor = ColumnTransformer(transformers=[
    ('one_hot',  OneHotEncoder(handle_unknown='ignore', sparse_output=False), nominal_features),
    ('ordinal',  OrdinalEncoder(categories=size_order),                       ordinal_features),
    ('passthrough', 'passthrough',                                             numeric_features)
])

# fit_transform on the full set here just for demonstration
encoded_array = preprocessor.fit_transform(clothing_data)

# Recover column names for readability
ohe_feature_names = preprocessor.named_transformers_['one_hot'].get_feature_names_out(nominal_features)
all_column_names  = list(ohe_feature_names) + ordinal_features + numeric_features

encoded_df = pd.DataFrame(encoded_array, columns=all_column_names)

print("\nEncoded data:")
print(encoded_df)
print("\n'size' ordinal mapping — small=0, medium=1, large=2 (correct order preserved)")

import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

# --- Sensor readings dataset with one outlier row (row index 5) ---
sensor_data = pd.DataFrame({
    'temperature_c': [22.1, 23.5, 21.8, 24.0, 22.9, 999.9, 23.1],  # 999.9 is a broken sensor
    'humidity_pct':  [45,   50,   43,   55,   48,   47,    51  ]
})

print("Original sensor readings:")
print(sensor_data)
print()

# Fit each scaler independently on the same data for comparison
standard_scaler = StandardScaler()
minmax_scaler   = MinMaxScaler()
robust_scaler   = RobustScaler()   # uses median + IQR, ignores outliers effectively

std_scaled    = standard_scaler.fit_transform(sensor_data)
minmax_scaled = minmax_scaler.fit_transform(sensor_data)
robust_scaled = robust_scaler.fit_transform(sensor_data)

# Package results for easy comparison
result = pd.DataFrame({
    'temp_original':  sensor_data['temperature_c'],
    'temp_standard':  std_scaled[:, 0].round(3),
    'temp_minmax':    minmax_scaled[:, 0].round(3),
    'temp_robust':    robust_scaled[:, 0].round(3),
})

print("Scaling comparison (temperature column only):")
print(result)
print()
print("Notice how MinMaxScaler crushes all normal readings to near-zero")
print("because the outlier 999.9 dominates the range.")
print("RobustScaler keeps normal readings well spread — outlier is still large but harmless.")

import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import joblib

# --- Realistic loan application dataset ---
np.random.seed(0)
loan_data = pd.DataFrame({
    'age':         [34, np.nan, 28, 45, 52, 31, np.nan, 40, 37, 29],
    'income':      [52000, 81000, np.nan, 120000, 95000, 43000, 67000, np.nan, 58000, 74000],
    'employment':  ['employed', 'self-employed', 'employed', np.nan,
                    'employed', 'unemployed', 'employed', 'self-employed', np.nan, 'employed'],
    'city':        ['London', 'Manchester', 'London', 'Birmingham',
                    'London', 'Manchester', 'Birmingham', 'London', 'Manchester', 'London'],
    'loan_approved': [1, 1, 0, 1, 1, 0, 1, 0, 0, 1]   # target
})

features = loan_data.drop(columns='loan_approved')
labels   = loan_data['loan_approved']

# Identify which columns need which treatment
numeric_cols     = ['age', 'income']
categorical_cols = ['employment', 'city']

X_train, X_test, y_train, y_test = train_test_split(
    features, labels, test_size=0.3, random_state=42
)

# --- Build the numeric sub-pipeline ---
# Impute first (median is robust to outliers), then scale
numeric_pipeline = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler',  StandardScaler())
])

# --- Build the categorical sub-pipeline ---
# Impute with most frequent value, then one-hot encode
categorical_pipeline = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(handle_unknown='ignore', sparse_output=False))
])

# --- Combine into one ColumnTransformer ---
preprocessor = ColumnTransformer(transformers=[
    ('numeric',      numeric_pipeline,      numeric_cols),
    ('categorical',  categorical_pipeline,  categorical_cols)
])

# --- Full pipeline: preprocessing + model ---
full_pipeline = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('classifier',   LogisticRegression(random_state=42, max_iter=500))
])

# One call to fit — everything happens in the correct order, no leakage
full_pipeline.fit(X_train, y_train)

predictions = full_pipeline.predict(X_test)
accuracy    = accuracy_score(y_test, predictions)

print(f"Test accuracy: {accuracy:.2f}")
print(f"Predictions:   {predictions}")
print(f"Actual labels: {y_test.values}")

# --- Save the entire pipeline for deployment ---
# In production: load this file and call full_pipeline.predict(raw_input_df)
joblib.dump(full_pipeline, 'loan_approval_pipeline.pkl')
print("\nPipeline saved to loan_approval_pipeline.pkl")
print("In production, load with: pipeline = joblib.load('loan_approval_pipeline.pkl')")