PyTorch Basics Explained: Tensors, Autograd, and Real Model Training

import torch

# --- Creating tensors from real data ---
# Simulating a tiny dataset: 4 house sizes (sq ft) and their prices ($k)
house_sizes = torch.tensor([750.0, 1200.0, 1800.0, 2400.0])  # 1D tensor, shape (4,)
house_prices = torch.tensor([150.0, 220.0, 310.0, 410.0])    # 1D tensor, shape (4,)

print("Sizes tensor:", house_sizes)
print("Shape:", house_sizes.shape)        # torch.Size([4])
print("Data type:", house_sizes.dtype)    # torch.float32 — default for floats

# --- 2D tensor: batch of data (rows = samples, cols = features) ---
feature_matrix = torch.tensor([
    [750.0,  3.0, 1.0],   # size, bedrooms, bathrooms
    [1200.0, 4.0, 2.0],
    [1800.0, 4.0, 3.0],
    [2400.0, 5.0, 3.0],
])
print("\nFeature matrix shape:", feature_matrix.shape)  # torch.Size([4, 3])

# --- Device awareness: check and move to GPU if available ---
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("\nUsing device:", device)

# Move tensor to the target device — always do this before operations
# In PyTorch 2.x, doing this at creation time avoids an extra host-to-device copy
feature_matrix = feature_matrix.to(device)
print("Feature matrix device:", feature_matrix.device)

# --- Useful tensor operations ---
# Normalise features: (x - mean) / std — critical for stable training
# dim=0 means we compute one mean per column (per feature), across all rows (samples)
means = feature_matrix.mean(dim=0)
stds  = feature_matrix.std(dim=0)
normalised = (feature_matrix - means) / stds

print("\nNormalised features (first row):", normalised[0])

# --- requires_grad: opting a tensor INTO gradient tracking ---
# We do NOT set this on input data — only on learnable parameters
# Input data is fixed; we want gradients w.r.t. parameters, not the data itself
weight = torch.tensor([0.15], requires_grad=True)  # our model's single weight
bias   = torch.tensor([10.0], requires_grad=True)  # our model's bias term

print("\nWeight requires grad:", weight.requires_grad)       # True
print("House sizes requires grad:", house_sizes.requires_grad)  # False — data, not a parameter

# --- Checking tensor metadata in one place ---
# Useful diagnostic pattern during debugging
for name, t in [("weight", weight), ("bias", bias), ("sizes", house_sizes)]:
    print(f"{name:8s} | dtype: {t.dtype} | device: {t.device} | requires_grad: {t.requires_grad}")

import torch

# Seed for reproducibility — always set this in experiments
# Without it, two runs with identical code produce different results and debugging becomes a nightmare
torch.manual_seed(42)

# --- Toy linear regression: predict house price from size ---
# Ground truth relationship: price ≈ 0.18 * size + 5  (the model must discover this)
house_sizes  = torch.tensor([750.0, 1200.0, 1800.0, 2400.0])
house_prices = torch.tensor([140.0, 221.0, 329.0, 437.0])

# Learnable parameters — these are the knobs autograd will compute gradients for
weight = torch.tensor([0.01], requires_grad=True)  # terrible initial guess, intentionally
bias   = torch.tensor([0.01], requires_grad=True)

# Learning rate is tiny because our raw inputs are in the hundreds-to-thousands range
# Without normalisation, you need a proportionally smaller step to avoid overshooting
learning_rate = 1e-7

for epoch in range(6):
    # FORWARD PASS: compute predictions using current weight and bias
    # Broadcasting applies weight and bias across all 4 house sizes simultaneously
    predicted_prices = weight * house_sizes + bias

    # COMPUTE LOSS: Mean Squared Error — average squared error across all predictions
    loss = ((predicted_prices - house_prices) ** 2).mean()

    # ZERO GRADIENTS: must do this before backward()
    # .grad accumulates by default — if we skip this, epoch 2 adds to epoch 1's gradients
    if weight.grad is not None:
        weight.grad.zero_()
        bias.grad.zero_()
    # (In real code you'd use optimizer.zero_grad() instead of this manual approach)

    # BACKWARD PASS: autograd traverses the graph and fills .grad on weight and bias
    # This computes d(loss)/d(weight) and d(loss)/d(bias) via the Chain Rule
    loss.backward()

    # PARAMETER UPDATE: move weight and bias in the direction that reduces loss
    # torch.no_grad() here because we don't want this update operation itself tracked
    with torch.no_grad():
        weight -= learning_rate * weight.grad
        bias   -= learning_rate * bias.grad

    print(f"Epoch {epoch+1:2d} | Loss: {loss.item():.2f} | "
          f"weight: {weight.item():.5f} | bias: {bias.item():.5f} | "
          f"grad_w: {weight.grad.item():.4f}")

print("\nFinal model: price =", round(weight.item(), 4), "* size +", round(bias.item(), 4))
print("Target model:  price = 0.18 * size + 5")
print("Note: bias is far from 5.0 — this is expected with unnormalised features and only 6 epochs")

# Inference — graph construction is wasted work here; inference_mode is faster than no_grad
with torch.inference_mode():
    test_size = torch.tensor([2000.0])
    predicted = weight * test_size + bias
    print(f"\nPredicted price for 2000 sq ft: ${predicted.item():.1f}k")

import torch
import torch.nn as nn
import torch.optim as optim

torch.manual_seed(42)

# --- Dataset: synthetic house price prediction ---
# 100 samples, 3 features: normalised size, bedrooms, age
num_samples  = 100
num_features = 3

# Generate synthetic features and a linear target with realistic noise
raw_features = torch.randn(num_samples, num_features)
true_weights  = torch.tensor([0.5, 0.3, -0.2])  # size helps, age hurts price
target_prices = raw_features @ true_weights + 0.1 * torch.randn(num_samples)

# Train / validation split: 80 / 20
train_size = int(0.8 * num_samples)
train_features, val_features = raw_features[:train_size], raw_features[train_size:]
train_targets,  val_targets  = target_prices[:train_size], target_prices[train_size:]


# --- Model definition ---
class HousePriceNet(nn.Module):
    def __init__(self, input_features: int):
        super().__init__()  # always call parent __init__ — skipping this breaks parameter registration

        self.network = nn.Sequential(
            nn.Linear(input_features, 16),  # input -> hidden (16 neurons)
            nn.ReLU(),                       # non-linearity: clamps negatives to zero
            nn.Linear(16, 8),               # hidden -> smaller hidden
            nn.ReLU(),
            nn.Linear(8, 1),                # final layer: one price prediction per sample
        )

    def forward(self, feature_batch: torch.Tensor) -> torch.Tensor:
        # squeeze(1) removes the trailing dimension: (batch_size, 1) -> (batch_size,)
        # This matches the shape of target_prices for MSELoss
        return self.network(feature_batch).squeeze(1)


model     = HousePriceNet(input_features=num_features)
loss_fn   = nn.MSELoss()                             # Mean Squared Error for regression
optimizer = optim.Adam(model.parameters(), lr=1e-3)  # Adam adapts step size per parameter

# Quick sanity check before training begins
print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
print(model)
print()

# --- Training loop ---
for epoch in range(1, 51):
    # Always set training mode at the top of the training loop body
    model.train()  # enables Dropout masking and BatchNorm batch statistics updates

    # Step 1: Zero stale gradients from the previous iteration
    # This is the line that the production incident was missing
    optimizer.zero_grad()

    # Step 2: Forward pass — model(x) not model.forward(x)
    # model() wires up forward hooks; model.forward() bypasses them
    train_predictions = model(train_features)

    # Step 3: Compute loss
    train_loss = loss_fn(train_predictions, train_targets)

    # Step 4: Backprop — autograd fills .grad on every parameter
    train_loss.backward()

    # Optional but recommended: log gradient norms before the update step
    # A norm above ~10.0 is worth investigating; above ~100.0 is a red flag
    if epoch % 10 == 0:
        total_norm = sum(p.grad.norm().item() ** 2 for p in model.parameters() if p.grad is not None) ** 0.5

    # Step 5: Update every parameter using its stored gradient
    optimizer.step()

    # --- Validation pass every 10 epochs ---
    if epoch % 10 == 0:
        model.eval()  # disables Dropout, freezes BatchNorm running stats
        with torch.no_grad():  # no graph construction needed — saves memory
            val_predictions = model(val_features)
            val_loss = loss_fn(val_predictions, val_targets)

        print(f"Epoch {epoch:3d} | Train Loss: {train_loss.item():.4f} | "
              f"Val Loss: {val_loss.item():.4f} | Grad Norm: {total_norm:.4f}")

# --- Single inference example ---
# inference_mode is faster than no_grad for serving — disables version tracking too
model.eval()
with torch.inference_mode():
    new_house = torch.tensor([[1.2, 0.5, -0.8]])  # one sample, 3 normalised features
    price_pred = model(new_house)
    expected   = new_house[0] @ true_weights
    print(f"\nPredicted: {price_pred.item():.4f} | Expected (approx): {expected.item():.4f}")