LLM Fine-Tuning Guide — How a Bad LoRA Rank Cost Us $4k/Month and 23% Accuracy

import torch
import torch.nn as nn
from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, AutoTokenizer

# Load base model (7B parameters)
model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", torch_dtype=torch.bfloat16)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
tokenizer.pad_token = tokenizer.eos_token  # Fix: Llama-2 has no pad token by default

# Configure LoRA — only q_proj and v_proj for production stability
lora_config = LoraConfig(
    r=16,  # Rank: higher = more capacity, but more memory
    lora_alpha=32,  # Scaling factor: higher = stronger adaptation
    target_modules=["q_proj", "v_proj"],  # Only attention query and value projections
    lora_dropout=0.05,  # Prevents overfitting on small datasets
    bias="none",  # Don't train bias terms — adds memory with little gain
    task_type="CAUSAL_LM"  # For GPT-style models
)

# Apply LoRA — this freezes all original weights and adds adapters
peft_model = get_peft_model(model, lora_config)

# Verify: only LoRA parameters are trainable
print(f"Trainable params: {sum(p.numel() for p in peft_model.parameters() if p.requires_grad)}")
# Output: ~8,388,608 for rank 16 (vs 7B for full fine-tuning)

# Forward pass — same interface as original model
inputs = tokenizer("The contract states that", return_tensors="pt", padding=True, truncation=True, max_length=512)
outputs = peft_model(**inputs, labels=inputs["input_ids"])
loss = outputs.loss
print(f"Initial loss: {loss.item():.4f}")  # Should be ~2-3 for a random batch

# After training, merge LoRA weights for inference (optional, speeds up by 10-20%)
merged_model = peft_model.merge_and_unload()  # Combines base + LoRA into original weight matrices

import torch
from transformers import (
    AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer,
    DataCollatorForLanguageModeling
)
from peft import LoraConfig, get_peft_model
from datasets import load_dataset
import wandb

# 1. Load dataset (assumes a JSONL file with 'prompt' and 'completion' fields)
dataset = load_dataset("json", data_files="train.jsonl", split="train")
dataset = dataset.train_test_split(test_size=0.1, seed=42)
train_data = dataset["train"]
eval_data = dataset["test"]

# 2. Tokenize with consistent max_length
def tokenize_function(examples):
    # Combine prompt and completion with EOS token
    texts = [p + c + tokenizer.eos_token for p, c in zip(examples["prompt"], examples["completion"])]
    return tokenizer(texts, truncation=True, padding="max_length", max_length=512)

tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
tokenizer.pad_token = tokenizer.eos_token

tokenized_train = train_data.map(tokenize_function, batched=True, remove_columns=train_data.column_names)
tokenized_eval = eval_data.map(tokenize_function, batched=True, remove_columns=eval_data.column_names)

# 3. Load model with LoRA
model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", torch_dtype=torch.bfloat16)
lora_config = LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], lora_dropout=0.05, bias="none", task_type="CAUSAL_LM")
peft_model = get_peft_model(model, lora_config)

# 4. Training arguments — production tuning
# Use bf16 if available, else fp16. Effective batch size = per_device * gradient_accumulation * num_gpus
training_args = TrainingArguments(
    output_dir="./llm-finetune-output",
    per_device_train_batch_size=4,  # Max for 7B on 40GB A100
    per_device_eval_batch_size=4,
    gradient_accumulation_steps=16,  # Effective batch size = 4*16*1 = 64
    num_train_epochs=3,
    learning_rate=2e-4,
    warmup_ratio=0.1,  # 10% of steps for warmup
    logging_steps=10,
    evaluation_strategy="steps",
    eval_steps=100,
    save_strategy="steps",
    save_steps=100,
    load_best_model_at_end=True,
    metric_for_best_model="eval_loss",
    greater_is_better=False,
    fp16=False,  # Use bf16 instead
    bf16=torch.cuda.is_bf16_supported(),  # Check hardware support
    report_to="wandb",  # Log to Weights & Biases
    run_name="llm-finetune-v1",
    seed=42,
)

# 5. Data collator for causal LM (creates labels automatically)
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)

# 6. Trainer
trainer = Trainer(
    model=peft_model,
    args=training_args,
    train_dataset=tokenized_train,
    eval_dataset=tokenized_eval,
    data_collator=data_collator,
)

# 7. Train and save
trainer.train()
trainer.save_model("./llm-finetune-final")

# 8. For deployment, merge LoRA weights (optional)
merged_model = peft_model.merge_and_unload()
merged_model.save_pretrained("./llm-finetune-merged")

from transformers import pipeline
from datasets import load_dataset

# Load your dataset (e.g., sentiment analysis)
dataset = load_dataset("imdb", split="test[:100]")  # 100 samples for quick eval

# Create a zero-shot classifier using the base model
classifier = pipeline("text-classification", model="meta-llama/Llama-2-7b-hf", device=0)

# Measure baseline accuracy
correct = 0
for example in dataset:
    result = classifier(example["text"], top_k=None)
    # Llama-2 returns labels like "POSITIVE" or "NEGATIVE"
    predicted = result[0]["label"]
    if predicted == example["label"]:
        correct += 1

accuracy = correct / len(dataset)
print(f"Zero-shot accuracy: {accuracy:.2%}")  # If >80%, reconsider fine-tuning

# If accuracy is low, check if prompt engineering helps
prompt = "Classify the sentiment of this movie review as POSITIVE or NEGATIVE. Review: {text}"
correct_prompt = 0
for example in dataset:
    result = classifier(prompt.format(text=example["text"]), top_k=None)
    predicted = result[0]["label"]
    if predicted == example["label"]:
        correct_prompt += 1

accuracy_prompt = correct_prompt / len(dataset)
print(f"With prompt engineering: {accuracy_prompt:.2%}")

# If prompt engineering gives >85%, skip fine-tuning and use RAG or better prompts

import torch
from transformers import AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model
from datasets import load_dataset

# 1. Quantization config — 4-bit NormalFloat (QLoRA)
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",  # NormalFloat 4-bit
    bnb_4bit_compute_dtype=torch.bfloat16,  # Compute in bf16
    bnb_4bit_use_double_quant=True,  # Double quantization saves memory
)

# 2. Load model in 4-bit
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    quantization_config=bnb_config,
    device_map="auto",  # Automatically distribute across GPUs
    torch_dtype=torch.bfloat16,
)

# 3. LoRA config (same as before)
lora_config = LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], lora_dropout=0.05)
peft_model = get_peft_model(model, lora_config)

# 4. Training — note: we can use larger batch size because 4-bit reduces memory
# On a single 48GB GPU, we can fit batch_size=8 for 7B model
training_args = TrainingArguments(
    output_dir="./qlora-output",
    per_device_train_batch_size=8,
    gradient_accumulation_steps=8,  # Effective batch size = 64
    num_train_epochs=3,
    learning_rate=2e-4,
    fp16=False,
    bf16=True,
    logging_steps=10,
    save_strategy="epoch",
)

# 5. Train (same as before)
trainer = Trainer(model=peft_model, args=training_args, train_dataset=train_dataset)
trainer.train()

# 6. Save adapter only (not the base model — it's already saved)
peft_model.save_pretrained("./qlora-adapter")

# For inference, load base model in 4-bit and adapter separately
from peft import PeftModel
base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf", quantization_config=bnb_config, device_map="auto")
adapter_model = PeftModel.from_pretrained(base_model, "./qlora-adapter")
# Now you can generate with the fine-tuned model

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model

# Mistake 1: No pad_token
model_name = "meta-llama/Llama-2-7b-hf"
tokenizer = AutoTokenizer.from_pretrained(model_name)
print(f"Before fix: pad_token = {tokenizer.pad_token}")  # None
tokenizer.pad_token = tokenizer.eos_token  # Fix
print(f"After fix: pad_token = {tokenizer.pad_token}")  # </s>

# Mistake 2: Wrong target_modules
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
# Check actual module names
attn = model.model.layers[0].self_attn
print(f"Attention module keys: {list(attn.state_dict().keys())}")
# Output: ['q_proj.weight', 'k_proj.weight', 'v_proj.weight', 'o_proj.weight']
# So target_modules should be ['q_proj', 'v_proj', 'k_proj', 'o_proj'] — but we only use q and v

lora_config = LoraConfig(r=16, target_modules=["q_proj", "v_proj"])  # Correct for Llama

# Mistake 3: use_cache=True during training
print(f"Before fix: use_cache = {model.config.use_cache}")  # True
model.config.use_cache = False  # Fix for training
print(f"After fix: use_cache = {model.config.use_cache}")  # False

# Now apply LoRA
peft_model = get_peft_model(model, lora_config)
print("Model ready for training — no more crashes at step 1")

import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model, PrefixTuningConfig, TaskType

model_name = "meta-llama/Llama-2-7b-hf"
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token

# Measure memory for each method
def measure_memory(method_name, model):
    torch.cuda.reset_peak_memory_stats()
    model.to("cuda")
    mem = torch.cuda.max_memory_allocated() / 1024**3  # GB
    print(f"{method_name}: {mem:.2f} GB")
    return mem

# 1. Full fine-tuning (no LoRA)
model_full = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
mem_full = measure_memory("Full Fine-Tuning", model_full)

# 2. LoRA (rank 16)
model_lora = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
lora_config = LoraConfig(r=16, lora_alpha=32, target_modules=["q_proj", "v_proj"], task_type=TaskType.CAUSAL_LM)
peft_model_lora = get_peft_model(model_lora, lora_config)
mem_lora = measure_memory("LoRA (r=16)", peft_model_lora)

# 3. Prefix Tuning (virtual tokens=20)
model_prefix = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
prefix_config = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, num_virtual_tokens=20)
peft_model_prefix = get_peft_model(model_prefix, prefix_config)
mem_prefix = measure_memory("Prefix Tuning (20 tokens)", peft_model_prefix)

# Results (on A100 40GB):
# Full Fine-Tuning: ~14.5 GB (just model, no optimizer)
# LoRA (r=16): ~2.1 GB (model + adapters)
# Prefix Tuning: ~0.3 GB (just virtual tokens)

print(f"Memory savings: LoRA = {mem_full/mem_lora:.1f}x, Prefix = {mem_full/mem_prefix:.1f}x")

import json
import numpy as np
from transformers import pipeline
from datetime import datetime

# Load fine-tuned model (merged)
classifier = pipeline("text-classification", model="./llm-finetune-merged", device=0)

# Simulate production predictions
predictions = []
confidence_scores = []

# Example: batch of 1000 requests
for i in range(1000):
    text = f"Sample input {i}"  # Replace with actual request
    result = classifier(text, top_k=None)
    pred = result[0]["label"]
    conf = result[0]["score"]
    predictions.append(pred)
    confidence_scores.append(conf)

# Monitor: check if mean confidence dropped
mean_conf = np.mean(confidence_scores)
print(f"Mean confidence: {mean_conf:.4f}")

# Alert if confidence drops below threshold (e.g., 0.7)
if mean_conf < 0.7:
    print("ALERT: Model confidence dropped — consider retraining")
    # Trigger retraining pipeline
    # e.g., call an API endpoint to start a new fine-tuning job

# Log to monitoring system (e.g., Prometheus)
log_entry = {
    "timestamp": datetime.utcnow().isoformat(),
    "mean_confidence": mean_conf,
    "num_predictions": len(predictions),
    "label_distribution": {label: predictions.count(label) for label in set(predictions)}
}
with open("monitoring_log.jsonl", "a") as f:
    f.write(json.dumps(log_entry) + "\n")

# Also track input length distribution
input_lengths = [len(text.split()) for text in [f"Sample input {i}" for i in range(1000)]]
print(f"Mean input length: {np.mean(input_lengths):.1f} tokens")
# If mean length deviates >20% from training data, flag it

from datasets import load_dataset

dataset = load_dataset("mteb/tweet_sentiment_extraction")

train_test = dataset["train"].train_test_split(
    test_size=0.2,
    seed=42,
    stratify_by_column="label"
)

train_dataset = train_test["train"]
eval_dataset = train_test["test"]

print(train_dataset.features["label"].names)
print(train_dataset.to_pandas()["label"].value_counts().to_dict())
print(eval_dataset.to_pandas()["label"].value_counts().to_dict())

from transformers import AutoModelForSequenceClassification
import torch

model = AutoModelForSequenceClassification.from_pretrained("gpt2", num_labels=3)

# Freeze all transformer layers initially
for param in model.transformer.parameters():
    param.requires_grad = False

# Unfreeze last 2 layers (layers 10 and 11 in GPT-2 12-layer)
for layer in model.transformer.h[-2:]:
    for param in layer.parameters():
        param.requires_grad = True

# Always keep the classification head trainable
for param in model.lm_head.parameters():
    param.requires_grad = True

# Train for 3 epochs, then thaw layers 8-9 and repeat