Math for Machine Learning — Learning Rate 1.0 Divergence

# TheCodeForge — Linear Algebra for ML
import numpy as np

# VECTORS: a single data point with multiple features
# A customer described by 3 numbers: [age, income, tenure_months]
customer = np.array([35, 75000, 24])
print(f'Vector shape: {customer.shape}')  # (3,) — 1 sample, 3 features

# MATRICES: a batch of data points (rows = samples, columns = features)
# 5 customers, each with 3 features
data = np.array([
    [35, 75000, 24],   # customer 1
    [28, 52000, 12],   # customer 2
    [42, 98000, 36],   # customer 3
    [31, 61000, 18],   # customer 4
    [55, 120000, 48],  # customer 5
])
print(f'Matrix shape: {data.shape}')  # (5, 3) = 5 samples, 3 features

# MATRIX MULTIPLICATION: the core operation in every ML model
# Neural network layer: output = input @ weights + bias
# (5,3) @ (3,2) = (5,2) — 5 samples transformed from 3 features to 2 outputs
np.random.seed(42)
weights = np.random.randn(3, 2)  # 3 input features -> 2 output neurons
bias = np.array([0.5, -0.3])
output = data @ weights + bias
print(f'Layer output shape: {output.shape}')  # (5, 2)
print(f'First sample output: {output[0].round(3)}')

# DOT PRODUCT: measures similarity between two vectors
# Used in recommendation systems and attention mechanisms
user_embedding = np.array([0.2, 0.8, 0.1])
item_embedding = np.array([0.3, 0.7, 0.2])
similarity = np.dot(user_embedding, item_embedding)
print(f'Dot product similarity: {similarity:.3f}')

# COSINE SIMILARITY: normalized dot product — ignores magnitude, measures direction
def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

print(f'Cosine similarity: {cosine_similarity(user_embedding, item_embedding):.3f}')

# TRANSPOSE: flip rows and columns — essential for shape compatibility
print(f'Original: {data.shape}')       # (5, 3)
print(f'Transposed: {data.T.shape}')   # (3, 5)

# EIGENDECOMPOSITION: powers PCA (dimensionality reduction)
# Covariance matrix reveals which features vary together
normalized_data = (data - data.mean(axis=0)) / data.std(axis=0)
cov_matrix = np.cov(normalized_data.T)
eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)
print(f'\nPCA — explained variance ratios: {(eigenvalues / eigenvalues.sum()).round(3)}')
print(f'First principal component: {eigenvectors[:, 0].round(3)}')

# NORM: measures vector magnitude — used in regularization and gradient clipping
weight_vector = np.array([2.5, -1.3, 0.8])
l2_norm = np.linalg.norm(weight_vector)      # Euclidean distance from origin
l1_norm = np.sum(np.abs(weight_vector))       # Manhattan distance — promotes sparsity
print(f'\nL2 norm: {l2_norm:.3f} (used in Ridge/weight decay)')
print(f'L1 norm: {l1_norm:.3f} (used in Lasso/feature selection)')

# TheCodeForge — Calculus for ML: Gradient Descent from Scratch
import numpy as np

# THE SETUP: we have data and want to find the best weight w
# y = w * x — find the w that minimizes prediction error
X = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
y_true = np.array([2.1, 3.9, 6.2, 7.8, 10.1])  # approximately y = 2x

# LOSS FUNCTION: measures how wrong the model is
# Mean Squared Error: L = (1/n) * sum((w*x - y)^2)
def compute_loss(w, X, y):
    predictions = w * X
    return np.mean((predictions - y) ** 2)

# DERIVATIVE: the slope of the loss function at the current w
# dL/dw = (2/n) * sum((w*x - y) * x)
# Positive derivative -> w is too large -> decrease w
# Negative derivative -> w is too small -> increase w
def compute_gradient(w, X, y):
    predictions = w * X
    errors = predictions - y
    return (2.0 / len(X)) * np.sum(errors * X)

# GRADIENT DESCENT: iteratively follow the slope downhill
w = 0.0  # start with a guess
learning_rate = 0.01
losses = []

for step in range(100):
    loss = compute_loss(w, X, y_true)
    gradient = compute_gradient(w, X, y_true)
    w = w - learning_rate * gradient  # the fundamental update rule
    losses.append(loss)

    if step % 20 == 0:
        print(f'Step {step:3d} | w = {w:.4f} | loss = {loss:.6f} | gradient = {gradient:+.4f}')

print(f'\nConverged weight: {w:.4f} (true value is approximately 2.0)')
print(f'Final loss: {losses[-1]:.8f}')

# LEARNING RATE EFFECT: the most important hyperparameter
print('\n--- Learning Rate Comparison ---')
for lr in [0.0001, 0.001, 0.01, 0.1, 1.0]:
    w_test = 0.0
    for _ in range(50):
        grad = compute_gradient(w_test, X, y_true)
        w_test = w_test - lr * grad
    final_loss = compute_loss(w_test, X, y_true)
    status = 'DIVERGED' if np.isnan(final_loss) or final_loss > 1e10 else f'loss={final_loss:.6f}'
    print(f'  LR={lr:<6} | w={w_test:.4f} | {status}')

# PARTIAL DERIVATIVES: when there are multiple parameters
# y = w1*x1 + w2*x2 + b — gradient has one component per parameter
def multi_param_gradient(w1, w2, b, X1, X2, y):
    pred = w1 * X1 + w2 * X2 + b
    errors = pred - y
    n = len(y)
    dw1 = (2.0 / n) * np.sum(errors * X1)
    dw2 = (2.0 / n) * np.sum(errors * X2)
    db  = (2.0 / n) * np.sum(errors)
    return dw1, dw2, db

print('\nPartial derivatives enable multi-parameter optimization.')
print('Each parameter gets its own gradient component.')
print('This scales to millions of parameters — same principle, computed by autograd.')

# TheCodeForge — Probability for ML
import numpy as np
from scipy import stats

# PROBABILITY BASICS: how likely is an event?
# P(spam) = spam emails / total emails
spam_count = 200
total_count = 1000
p_spam = spam_count / total_count
print(f'P(spam) = {p_spam}')  # 0.2

# CONDITIONAL PROBABILITY + BAYES' THEOREM
# Question: if an email contains the word "winner", what is P(spam)?
p_word_given_spam = 0.80   # 80% of spam contains "winner"
p_word_given_ham = 0.05    # 5% of legitimate email contains "winner"
p_ham = 1 - p_spam         # 0.8

# Bayes: P(spam | word) = P(word | spam) * P(spam) / P(word)
p_word = (p_word_given_spam * p_spam) + (p_word_given_ham * p_ham)
p_spam_given_word = (p_word_given_spam * p_spam) / p_word
print(f'P(spam | "winner") = {p_spam_given_word:.3f}')  # prior 0.2 updated to 0.8

# PROBABILITY DISTRIBUTIONS: describe how data is spread
# Normal (Gaussian): most values near mean, symmetric tails
normal = stats.norm(loc=100, scale=15)  # mean=100, std=15
print(f'\nP(85 < X < 115) = {normal.cdf(115) - normal.cdf(85):.3f}')  # ~68% within 1 std
print(f'P(X > 130) = {1 - normal.cdf(130):.4f}')  # ~2.3% in upper tail

# SOFTMAX: converts raw model outputs (logits) to probabilities
# Used in every classification neural network's final layer
def softmax(logits):
    # Subtract max for numerical stability — prevents exp() overflow
    shifted = logits - np.max(logits)
    exp_values = np.exp(shifted)
    return exp_values / exp_values.sum()

logits = np.array([2.0, 1.0, 0.1])  # raw scores from neural network
probabilities = softmax(logits)
print(f'\nLogits: {logits}')
print(f'Softmax probabilities: {probabilities.round(3)}')  # sums to 1.0
print(f'Predicted class: {np.argmax(probabilities)}')

# TEMPERATURE: controls confidence sharpness in LLM token sampling
def softmax_with_temperature(logits, temperature):
    scaled = logits / temperature
    return softmax(scaled)

print('\n--- Temperature effect on probability distribution ---')
for temp in [0.1, 0.5, 1.0, 2.0, 5.0]:
    probs = softmax_with_temperature(logits, temp)
    print(f'  T={temp:<3} | probs={probs.round(3)} | max_prob={probs.max():.3f}')

# CROSS-ENTROPY LOSS: measures distance between predicted and true distributions
# Lower = predicted probabilities closer to ground truth
def cross_entropy(y_true, y_pred, epsilon=1e-15):
    y_pred = np.clip(y_pred, epsilon, 1 - epsilon)  # prevent log(0)
    return -np.sum(y_true * np.log(y_pred))

y_true = np.array([1, 0, 0])  # true class is 0
y_pred_good = np.array([0.9, 0.05, 0.05])  # confident and correct
y_pred_bad = np.array([0.1, 0.6, 0.3])     # confident but wrong
y_pred_uncertain = np.array([0.4, 0.3, 0.3])  # uncertain
print(f'\nCross-entropy (confident correct): {cross_entropy(y_true, y_pred_good):.3f}')
print(f'Cross-entropy (confident wrong):   {cross_entropy(y_true, y_pred_bad):.3f}')
print(f'Cross-entropy (uncertain):         {cross_entropy(y_true, y_pred_uncertain):.3f}')
print('Lower loss = better calibrated predictions')

# TheCodeForge — Statistics for ML
import numpy as np
from scipy import stats

# DESCRIPTIVE STATISTICS: summarize what the data looks like
np.random.seed(42)
# Simulating real-world income data — right-skewed, not normal
income = np.concatenate([
    np.random.exponential(scale=40000, size=800),   # majority of earners
    np.random.normal(loc=200000, scale=50000, size=200)  # high earners
])

mean = np.mean(income)
median = np.median(income)
std = np.std(income)
print(f'Mean:   ${mean:>10,.0f}   (pulled up by high earners)')
print(f'Median: ${median:>10,.0f}   (more representative of typical earner)')
print(f'Std:    ${std:>10,.0f}   (high spread indicates mixed population)')
print(f'25th percentile: ${np.percentile(income, 25):>10,.0f}')
print(f'75th percentile: ${np.percentile(income, 75):>10,.0f}')
print(f'Mean-Median gap: ${mean - median:>10,.0f}   (positive gap = right skew)')

# HYPOTHESIS TESTING: is Model B actually better than Model A?
# Scenario: Model A accuracy 85%, Model B accuracy 87% on 1000 test samples
# Question: is the 2% gap real or could it be sampling luck?
np.random.seed(42)
model_a_correct = np.random.binomial(1, 0.85, 1000)  # 1=correct, 0=wrong
model_b_correct = np.random.binomial(1, 0.87, 1000)

t_stat, p_value = stats.ttest_ind(model_a_correct, model_b_correct)
print(f'\n--- Hypothesis Test: Model A vs Model B ---')
print(f'Model A accuracy: {model_a_correct.mean():.3f}')
print(f'Model B accuracy: {model_b_correct.mean():.3f}')
print(f'T-statistic: {t_stat:.3f}')
print(f'P-value: {p_value:.4f}')
print(f'Significant at alpha=0.05? {"YES — real improvement" if p_value < 0.05 else "NO — could be noise"}')

# CONFIDENCE INTERVALS: range of plausible accuracy values
def confidence_interval(data, confidence=0.95):
    n = len(data)
    mean = np.mean(data)
    std_err = stats.sem(data)  # standard error of the mean
    margin = std_err * stats.t.ppf((1 + confidence) / 2, n - 1)
    return mean, mean - margin, mean + margin

mean_b, ci_low, ci_high = confidence_interval(model_b_correct)
print(f'\nModel B accuracy: {mean_b:.3f}')
print(f'95% CI: [{ci_low:.3f}, {ci_high:.3f}]')
print(f'Interpretation: we are 95% confident true accuracy is in this range')

# CORRELATION: which features move together?
# High correlation between features = potential multicollinearity problem
np.random.seed(42)
age = np.random.normal(40, 10, 200)
experience = age - 22 + np.random.normal(0, 3, 200)  # correlated with age
salary = 30000 + 1500 * experience + np.random.normal(0, 5000, 200)

print(f'\n--- Feature Correlations ---')
print(f'Age vs Experience:  r = {np.corrcoef(age, experience)[0,1]:.3f}  (high — potential multicollinearity)')
print(f'Experience vs Salary: r = {np.corrcoef(experience, salary)[0,1]:.3f}  (strong positive relationship)')
print(f'Age vs Salary:      r = {np.corrcoef(age, salary)[0,1]:.3f}  (indirect through experience)')

# BIAS-VARIANCE TRADEOFF: the most important concept in ML
# High bias (underfitting): model too simple, misses patterns
# High variance (overfitting): model too complex, memorizes noise
train_acc = 0.99
test_acc = 0.72
gap = train_acc - test_acc
print(f'\n--- Bias-Variance Diagnostic ---')
print(f'Train accuracy: {train_acc:.2f}')
print(f'Test accuracy:  {test_acc:.2f}')
print(f'Gap: {gap:.2f}')
if gap > 0.15:
    print('Diagnosis: HIGH VARIANCE (overfitting) — add regularization, reduce complexity, or get more data')
elif test_acc < 0.70:
    print('Diagnosis: HIGH BIAS (underfitting) — increase model capacity or improve features')
else:
    print('Diagnosis: reasonable tradeoff — monitor for drift')

# TheCodeForge — Math Behind Common ML Algorithms
import numpy as np

# ====================================================================
# LINEAR REGRESSION: Linear Algebra + Calculus + Statistics
# ====================================================================
np.random.seed(42)
X = np.random.randn(100, 3)  # 100 samples, 3 features
true_weights = np.array([2.5, -1.3, 0.8])
y = X @ true_weights + np.random.randn(100) * 0.5  # y = Xw + noise

# METHOD 1: Closed-form solution (Linear Algebra)
# Normal equation: w = (X^T X)^-1 X^T y
X_bias = np.column_stack([X, np.ones(100)])  # add bias column
w_closed = np.linalg.inv(X_bias.T @ X_bias) @ X_bias.T @ y
print('--- Linear Regression ---')
print(f'Closed-form weights: {w_closed[:3].round(3)}')
print(f'True weights:        {true_weights}')

# METHOD 2: Gradient descent (Calculus)
w_gd = np.zeros(3)
lr = 0.01
for step in range(500):
    predictions = X @ w_gd
    errors = predictions - y
    gradient = (2.0 / len(y)) * (X.T @ errors)  # vector of partial derivatives
    w_gd = w_gd - lr * gradient

print(f'Gradient descent weights: {w_gd.round(3)}')

# R-squared (Statistics): how much variance does the model explain?
y_pred = X @ w_gd
ss_res = np.sum((y - y_pred) ** 2)
ss_tot = np.sum((y - y.mean()) ** 2)
r_squared = 1 - ss_res / ss_tot
print(f'R-squared: {r_squared:.4f}')

# ====================================================================
# LOGISTIC REGRESSION: Linear Algebra + Calculus + Probability
# ====================================================================
def sigmoid(z):
    """Probability function: maps any real number to (0, 1)"""
    return 1.0 / (1.0 + np.exp(-np.clip(z, -500, 500)))

# Sigmoid converts linear output to probability
linear_outputs = np.array([-2, -1, 0, 1, 2])
probabilities = sigmoid(linear_outputs)
print(f'\n--- Logistic Regression ---')
print(f'Linear outputs: {linear_outputs}')
print(f'Sigmoid probs:  {probabilities.round(3)}')
print('Sigmoid(0) = 0.5 — the decision boundary')
print('Linear Algebra + Calculus + Probability = Logistic Regression')

# ====================================================================
# ATTENTION MECHANISM (Transformers): Linear Algebra + Probability
# ====================================================================
def scaled_dot_product_attention(Q, K, V):
    """Core attention computation used in every Transformer model.
    Q, K, V: query, key, value matrices
    Returns: weighted combination of values based on query-key similarity
    """
    d_k = K.shape[-1]
    # Step 1: compute similarity scores (Linear Algebra: matrix multiply)
    scores = Q @ K.T / np.sqrt(d_k)
    # Step 2: convert scores to probabilities (Probability: softmax)
    exp_scores = np.exp(scores - np.max(scores, axis=-1, keepdims=True))
    attention_weights = exp_scores / exp_scores.sum(axis=-1, keepdims=True)
    # Step 3: weighted sum of values (Linear Algebra: matrix multiply)
    output = attention_weights @ V
    return output, attention_weights

# Simulate 4 tokens with 8-dimensional embeddings
np.random.seed(42)
seq_len, d_model = 4, 8
Q = np.random.randn(seq_len, d_model)
K = np.random.randn(seq_len, d_model)
V = np.random.randn(seq_len, d_model)

output, weights = scaled_dot_product_attention(Q, K, V)
print(f'\n--- Transformer Attention ---')
print(f'Query shape:  {Q.shape}')
print(f'Output shape: {output.shape}')
print(f'Attention weights (row = query, col = key):')
print(weights.round(3))
print('Each row sums to 1.0 — softmax makes it a probability distribution over keys')
print('Attention = Linear Algebra (matmul) + Probability (softmax)')