Neural Networks Explained: Architecture, Training, and Real-World Use

import numpy as np

def sigmoid(raw_output):
    """Squishes any number into the range (0, 1).
    Values far below zero approach 0; far above zero approach 1.
    This lets the neuron express 'confidence' as a probability."""
    return 1 / (1 + np.exp(-raw_output))

def single_neuron_forward(inputs, weights, bias):
    """Simulates one forward pass through a single neuron."""
    # Dot product: multiply each input by its weight, then sum them all.
    # This is the core of what a neuron does before activation.
    weighted_sum = np.dot(inputs, weights) + bias

    # The activation function decides: how 'fired up' is this neuron?
    activated_output = sigmoid(weighted_sum)
    return weighted_sum, activated_output

# --- Example: predicting whether a house is 'expensive' ---
# Input features: [square_footage_scaled, num_bedrooms_scaled]
house_inputs   = np.array([0.85, 0.60])   # normalised to 0-1 range
initial_weights = np.array([0.40, 0.35])  # learned during training
bias_term       = -0.20                   # shifts the decision threshold

raw, output = single_neuron_forward(house_inputs, initial_weights, bias_term)

print(f"Weighted sum (before activation): {raw:.4f}")
print(f"Sigmoid output (prediction):      {output:.4f}")
print(f"Interpretation: {output*100:.1f}% confidence the house is expensive")

import numpy as np

np.random.seed(42)  # Makes results reproducible — critical for debugging

# ── Activation functions ──────────────────────────────────────────────────────

def sigmoid(z):
    """Maps any real number to (0, 1). Good for output layers in binary tasks."""
    return 1 / (1 + np.exp(-z))

def sigmoid_derivative(z):
    """Gradient of sigmoid — needed during backpropagation.
    Note: we pass the pre-activation value z, not the activated output."""
    s = sigmoid(z)
    return s * (1 - s)  # Derivative of sigmoid is elegantly self-referential

def relu(z):
    """Rectified Linear Unit: max(0, z). Fast, simple, works great in hidden layers."""
    return np.maximum(0, z)

def relu_derivative(z):
    """Gradient of ReLU: 1 where input > 0, else 0."""
    return (z > 0).astype(float)

# ── Network initialisation ────────────────────────────────────────────────────

def initialise_network(layer_sizes):
    """Creates weight matrices and bias vectors for a network of any depth.

    layer_sizes: list of integers, e.g. [2, 4, 4, 1] means
                 2 inputs -> hidden layer of 4 -> hidden layer of 4 -> 1 output.

    We use He initialisation (weights scaled by sqrt(2/n_inputs)) because
    it keeps gradients from vanishing or exploding in deep ReLU networks.
    """
    parameters = {}
    num_layers  = len(layer_sizes) - 1  # number of weight matrices

    for layer_idx in range(1, len(layer_sizes)):
        fan_in   = layer_sizes[layer_idx - 1]  # neurons feeding INTO this layer
        fan_out  = layer_sizes[layer_idx]       # neurons IN this layer

        # He initialisation: variance = 2 / fan_in
        parameters[f"W{layer_idx}"] = np.random.randn(fan_out, fan_in) * np.sqrt(2 / fan_in)
        parameters[f"b{layer_idx}"] = np.zeros((fan_out, 1))  # biases start at zero

    return parameters

# ── Forward propagation ───────────────────────────────────────────────────────

def forward_pass(input_data, parameters, num_layers):
    """Passes input through every layer and returns predictions + a cache.

    The cache stores pre-activation values (Z) and activations (A) for each
    layer — we'll need them during backpropagation to compute gradients.
    """
    cache      = {"A0": input_data}  # A0 is just the raw input
    current_A  = input_data

    for layer_idx in range(1, num_layers + 1):
        W = parameters[f"W{layer_idx}"]
        b = parameters[f"b{layer_idx}"]

        # Linear step: Z = W·A_prev + b
        Z = np.dot(W, current_A) + b
        cache[f"Z{layer_idx}"] = Z  # save for backprop

        # Activation step: use ReLU for hidden layers, sigmoid for output
        if layer_idx < num_layers:
            current_A = relu(Z)      # hidden layers stay non-linear with ReLU
        else:
            current_A = sigmoid(Z)   # output layer gives probability [0,1]

        cache[f"A{layer_idx}"] = current_A

    prediction = current_A  # final layer output
    return prediction, cache

# ── Loss function ─────────────────────────────────────────────────────────────

def binary_cross_entropy_loss(predictions, true_labels):
    """Measures how wrong our predictions are for a binary classification task.

    Cross-entropy penalises confident wrong predictions very heavily —
    far more than mean squared error would. This is WHY we use it for
    classification rather than regression losses.
    """
    num_samples = true_labels.shape[1]
    epsilon     = 1e-15  # prevents log(0) which is undefined

    predictions = np.clip(predictions, epsilon, 1 - epsilon)
    loss = -np.mean(
        true_labels * np.log(predictions) +
        (1 - true_labels) * np.log(1 - predictions)
    )
    return loss

# ── Backpropagation ───────────────────────────────────────────────────────────

def backward_pass(predictions, true_labels, cache, parameters, num_layers):
    """Computes gradients for every weight and bias using the chain rule.

    Backpropagation works BACKWARDS from the output error, distributing
    blame to each weight based on how much it contributed to the mistake.
    """
    gradients   = {}
    num_samples = true_labels.shape[1]

    # Gradient of loss with respect to output layer activation
    # (This formula is the derivative of binary cross-entropy + sigmoid combined)
    dA_output = -(true_labels / predictions) + (1 - true_labels) / (1 - predictions)

    current_dA = dA_output

    for layer_idx in reversed(range(1, num_layers + 1)):
        Z      = cache[f"Z{layer_idx}"]
        A_prev = cache[f"A{layer_idx - 1}"]
        W      = parameters[f"W{layer_idx}"]

        # Gradient through activation function
        if layer_idx == num_layers:
            dZ = current_dA * sigmoid_derivative(Z)
        else:
            dZ = current_dA * relu_derivative(Z)

        # Gradients for weights, biases, and the previous layer's activation
        dW             = np.dot(dZ, A_prev.T) / num_samples
        db             = np.sum(dZ, axis=1, keepdims=True) / num_samples
        dA_prev        = np.dot(W.T, dZ)

        gradients[f"dW{layer_idx}"] = dW
        gradients[f"db{layer_idx}"] = db
        current_dA                  = dA_prev  # propagate error to previous layer

    return gradients

# ── Weight update (Gradient Descent) ─────────────────────────────────────────

def update_weights(parameters, gradients, learning_rate, num_layers):
    """Nudges every weight in the direction that reduces loss.

    learning_rate controls step size — too large and we overshoot the minimum,
    too small and training takes forever. 0.01 is a safe starting point.
    """
    for layer_idx in range(1, num_layers + 1):
        parameters[f"W{layer_idx}"] -= learning_rate * gradients[f"dW{layer_idx}"]
        parameters[f"b{layer_idx}"] -= learning_rate * gradients[f"db{layer_idx}"]
    return parameters

# ── Full training loop ────────────────────────────────────────────────────────

def train(input_data, true_labels, layer_sizes, learning_rate=0.01, num_epochs=1000):
    """Ties everything together: initialise, forward, compute loss, backward, update."""
    num_layers = len(layer_sizes) - 1
    parameters = initialise_network(layer_sizes)

    for epoch in range(num_epochs):
        predictions, cache = forward_pass(input_data, parameters, num_layers)
        loss               = binary_cross_entropy_loss(predictions, true_labels)
        gradients          = backward_pass(predictions, true_labels, cache, parameters, num_layers)
        parameters         = update_weights(parameters, gradients, learning_rate, num_layers)

        if epoch % 200 == 0:
            print(f"Epoch {epoch:>5} | Loss: {loss:.4f}")

    return parameters

# ── XOR problem: the classic test for multi-layer networks ────────────────────
# XOR cannot be solved by a single neuron (it's not linearly separable).
# A two-layer network solves it easily — great proof that depth adds power.

XOR_inputs  = np.array([[0, 0, 1, 1],
                          [0, 1, 0, 1]])  # shape: (2 features, 4 samples)
XOR_outputs = np.array([[0, 1, 1, 0]])   # shape: (1 output, 4 samples)

# Architecture: 2 inputs -> 4 hidden neurons -> 4 hidden neurons -> 1 output
architecture = [2, 4, 4, 1]

print("=== Training Neural Network on XOR Problem ===")
trained_params = train(
    input_data   = XOR_inputs,
    true_labels  = XOR_outputs,
    layer_sizes  = architecture,
    learning_rate= 0.1,
    num_epochs   = 1001
)

# Final predictions
final_preds, _ = forward_pass(XOR_inputs, trained_params, len(architecture) - 1)
print("\n=== Final Predictions ===")
for i in range(4):
    input_pair = XOR_inputs[:, i]
    predicted  = final_preds[0, i]
    expected   = XOR_outputs[0, i]
    print(f"Input: {input_pair} | Expected: {expected} | Predicted: {predicted:.4f}")

import numpy as np

# ── Demonstrating gradient descent on a simple 1D loss surface ────────────────
# Imagine weight_value is ONE weight in the network.
# The loss function below is a simplified stand-in for real cross-entropy loss.
# Goal: find the weight value that minimises loss.

def simplified_loss(weight_value):
    """A bowl-shaped loss surface. Minimum is at weight_value = 2.0.
    In real networks, the loss surface is millions of dimensions — same idea."""
    return (weight_value - 2.0) ** 2 + 0.5

def loss_gradient(weight_value):
    """The derivative of the loss with respect to the weight.
    Tells us which direction to nudge the weight to reduce loss.
    For (w-2)^2 + 0.5, the derivative is 2*(w-2)."""
    return 2 * (weight_value - 2.0)

# Start with a random weight far from optimal
current_weight = -3.0
learning_rate  = 0.15
num_steps      = 20

print("=== Gradient Descent — Watching a Weight Find Its Optimal Value ===")
print(f"{'Step':>5} | {'Weight':>10} | {'Loss':>10} | {'Gradient':>10}")
print("-" * 46)

for step in range(num_steps):
    current_loss     = simplified_loss(current_weight)
    gradient         = loss_gradient(current_weight)

    # The core update rule: move OPPOSITE to the gradient
    # Negative gradient means loss decreases as weight increases — so increase weight
    current_weight  -= learning_rate * gradient  # w = w - lr * dL/dw

    if step % 4 == 0 or step == num_steps - 1:
        print(f"{step:>5} | {current_weight:>10.4f} | {current_loss:>10.4f} | {gradient:>10.4f}")

print(f"\nFinal weight: {current_weight:.4f} (target was 2.0)")
print(f"Final loss:   {simplified_loss(current_weight):.6f} (minimum is 0.5)")