Text Classification Failure — OOV Crash Recall to 0.51

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer

# Simulating a small customer review dataset
# In a real project this would come from a CSV or database
reviews = [
    "This product is absolutely amazing and works perfectly",
    "Terrible quality, broke after two days, total waste of money",
    "Pretty good value for the price, happy with my purchase",
    "Worst purchase I have ever made, completely useless product",
    "Exceeded my expectations, will definitely buy again"
]

labels = ["positive", "negative", "positive", "negative", "positive"]

# TfidfVectorizer handles tokenisation, lowercasing, and IDF weighting
# max_features limits vocabulary size — important for memory on large datasets
# stop_words='english' removes common words like 'the', 'is', 'and'
vectorizer = TfidfVectorizer(max_features=20, stop_words='english')

# fit_transform: learns the vocabulary AND converts text to numbers in one step
# Returns a sparse matrix — rows are documents, columns are words
tfidf_matrix = vectorizer.fit_transform(reviews)

# Let's see what vocabulary was learned
learned_vocabulary = vectorizer.get_feature_names_out()
print("Learned vocabulary:")
print(learned_vocabulary)
print()

# Convert sparse matrix to a readable DataFrame
tfidf_df = pd.DataFrame(
    tfidf_matrix.toarray(),
    columns=learned_vocabulary,
    index=[f"Review {i+1}" for i in range(len(reviews))]
)

print("TF-IDF scores per document (higher = more distinctive word):")
print(tfidf_df.round(3).to_string())

import numpy as np
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report, accuracy_score
from sklearn.model_selection import train_test_split

# Using a real benchmark dataset — 20 newsgroup posts across different topics
# We're selecting 3 categories to keep it manageable and interpretable
categories_to_classify = ['sci.space', 'rec.sport.hockey', 'talk.politics.guns']

print("Loading 20 Newsgroups dataset...")
newsgroups_data = fetch_20newsgroups(
    subset='all',
    categories=categories_to_classify,
    remove=('headers', 'footers', 'quotes')  # remove metadata that makes classification trivially easy
)

post_texts = newsgroups_data.data
category_labels = newsgroups_data.target
category_names = newsgroups_data.target_names

print(f"Total documents: {len(post_texts)}")
print(f"Categories: {category_names}")
print()

# Split into training and test sets — 80/20 is a solid default
# stratify=category_labels ensures each class has proportional representation in both sets
train_texts, test_texts, train_labels, test_labels = train_test_split(
    post_texts, category_labels,
    test_size=0.2,
    random_state=42,
    stratify=category_labels
)

# --- PIPELINE 1: Naive Bayes ---
# Pipeline chains steps so the same transformations apply consistently to train and test
# This is the production-safe pattern — no data leakage possible
naive_bayes_pipeline = Pipeline([
    ('tfidf_vectorizer', TfidfVectorizer(
        max_features=10000,
        stop_words='english',
        ngram_range=(1, 2)  # include both single words AND two-word phrases
    )),
    ('naive_bayes_classifier', MultinomialNB(alpha=0.1))  # alpha is the smoothing parameter
])

naive_bayes_pipeline.fit(train_texts, train_labels)
nb_predictions = naive_bayes_pipeline.predict(test_texts)
nb_accuracy = accuracy_score(test_labels, nb_predictions)

print(f"=== Naive Bayes Results ===")
print(f"Accuracy: {nb_accuracy:.3f}")
print(classification_report(test_labels, nb_predictions, target_names=category_names))

# --- PIPELINE 2: Logistic Regression ---
logistic_pipeline = Pipeline([
    ('tfidf_vectorizer', TfidfVectorizer(
        max_features=10000,
        stop_words='english',
        ngram_range=(1, 2)
    )),
    ('logistic_classifier', LogisticRegression(
        max_iter=1000,      # increase from default 100 to ensure convergence
        C=1.0,              # regularisation strength — lower C = more regularisation
        solver='lbfgs',
        multi_class='multinomial'
    ))
])

logistic_pipeline.fit(train_texts, train_labels)
lr_predictions = logistic_pipeline.predict(test_texts)
lr_accuracy = accuracy_score(test_labels, lr_predictions)

print(f"\n=== Logistic Regression Results ===")
print(f"Accuracy: {lr_accuracy:.3f}")
print(classification_report(test_labels, lr_predictions, target_names=category_names))

# --- Making predictions on new, unseen text ---
new_posts = [
    "The astronauts launched successfully to the International Space Station",
    "The goalie made an incredible save in overtime to win the championship",
    "The senate voted on the second amendment legislation today"
]

print("\n=== Predictions on new posts ===")
new_predictions = logistic_pipeline.predict(new_posts)
new_probabilities = logistic_pipeline.predict_proba(new_posts)

for post, prediction, probabilities in zip(new_posts, new_predictions, new_probabilities):
    predicted_category = category_names[prediction]
    confidence = max(probabilities)
    print(f"Post: '{post[:55]}...'")
    print(f"  Predicted: {predicted_category} (confidence: {confidence:.1%})")
    print()

# Install first: pip install sentence-transformers scikit-learn
import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split

# Real-world scenario: classifying customer support tickets
# Notice how many rows have paraphrased meaning — TF-IDF would struggle here
support_tickets = [
    # Billing issues
    "I was charged twice for my subscription this month",
    "There's a duplicate payment on my credit card statement",
    "My invoice shows the wrong amount, please fix this",
    "You billed me for a plan I never signed up for",
    "I need a refund for the extra charge on my account",
    "The payment went through twice and I want my money back",
    # Technical issues
    "The app keeps crashing every time I open it",
    "Your software won't start on my Windows 11 laptop",
    "I'm getting a black screen when I launch the application",
    "The program freezes after about 30 seconds of use",
    "Cannot log into the platform, it just hangs on the loading screen",
    "The mobile app stopped working after the latest update",
    # Account access
    "I forgot my password and the reset email never arrived",
    "Locked out of my account after too many login attempts",
    "My account was suspended but I didn't violate any rules",
    "Can't access my profile, says my email is not recognised",
    "The two-factor authentication code isn't working for me",
    "I need to recover access to my account urgently"
]

ticket_categories = (
    ["billing"] * 6 +
    ["technical"] * 6 +
    ["account_access"] * 6
)

# Load a lightweight, fast model — good balance of speed and quality
# 'all-MiniLM-L6-v2' produces 384-dimensional embeddings and runs in ~50ms per sentence on CPU
print("Loading sentence transformer model (downloads ~80MB on first run)...")
embedding_model = SentenceTransformer('all-MiniLM-L6-v2')

# Encode all tickets into dense vector representations
# Each ticket becomes a 384-dimensional vector — semantically similar tickets will cluster together
print("Generating sentence embeddings...")
ticket_embeddings = embedding_model.encode(
    support_tickets,
    show_progress_bar=True,
    batch_size=16  # process in batches to manage memory
)

print(f"\nEmbedding shape: {ticket_embeddings.shape}")
print(f"Each ticket is now a vector of {ticket_embeddings.shape[1]} numbers")

# Split data — stratify ensures all 3 classes appear in both sets
train_embeddings, test_embeddings, train_labels, test_labels = train_test_split(
    ticket_embeddings, ticket_categories,
    test_size=0.33,
    random_state=42,
    stratify=ticket_categories
)

# Logistic Regression works beautifully on top of embeddings
# The embeddings do the heavy lifting; LR just learns the decision boundary
classifier = LogisticRegression(max_iter=1000, C=1.0)
classifier.fit(train_embeddings, train_labels)

test_predictions = classifier.predict(test_embeddings)
print("\n=== Classification Report ===")
print(classification_report(test_labels, test_predictions))

# --- The real power: paraphrase robustness ---
# These sentences use completely different words from the training data
unseen_tickets = [
    "I've been double-billed and demand an immediate reimbursement",   # billing
    "The desktop client is unresponsive and will not open at all",      # technical
    "My login credentials are no longer being accepted by the system"   # account_access
]

print("\n=== Paraphrase Robustness Test ===")
unseen_embeddings = embedding_model.encode(unseen_tickets)
unseen_predictions = classifier.predict(unseen_embeddings)
unseen_probabilities = classifier.predict_proba(unseen_embeddings)

for ticket, prediction, probs in zip(unseen_tickets, unseen_predictions, unseen_probabilities):
    confidence = max(probs)
    print(f"Ticket:    '{ticket}'")
    print(f"Predicted: {prediction} ({confidence:.1%} confidence)")
    print()

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.metrics import (
    classification_report,
    confusion_matrix,
    precision_recall_curve,
    average_precision_score
)
from sklearn.model_selection import train_test_split, cross_val_score

# Using medical vs non-medical newsgroups to simulate a high-stakes classification scenario
medical_categories = ['sci.med', 'sci.space', 'rec.sport.hockey']

newsgroups = fetch_20newsgroups(
    subset='all',
    categories=medical_categories,
    remove=('headers', 'footers', 'quotes')
)

train_texts, test_texts, train_labels, test_labels = train_test_split(
    newsgroups.data,
    newsgroups.target,
    test_size=0.2,
    random_state=42,
    stratify=newsgroups.target
)

classification_pipeline = Pipeline([
    ('vectorizer', TfidfVectorizer(max_features=15000, stop_words='english', ngram_range=(1, 2))),
    ('classifier', LogisticRegression(max_iter=1000, C=0.5))
])

classification_pipeline.fit(train_texts, train_labels)
test_predictions = classification_pipeline.predict(test_texts)
test_probabilities = classification_pipeline.predict_proba(test_texts)

category_names = newsgroups.target_names

# --- 1. Full Classification Report ---
print("=== Full Classification Report ===")
print(classification_report(test_labels, test_predictions, target_names=category_names))

# --- 2. Confusion Matrix ---
cm = confusion_matrix(test_labels, test_predictions)

plt.figure(figsize=(8, 6))
sns.heatmap(
    cm,
    annot=True,
    fmt='d',                     # show integer counts, not scientific notation
    cmap='Blues',
    xticklabels=category_names,
    yticklabels=category_names
)
plt.ylabel('True Label', fontsize=12)
plt.xlabel('Predicted Label', fontsize=12)
plt.title('Confusion Matrix — Text Classifier')
plt.tight_layout()
plt.savefig('confusion_matrix.png', dpi=150)
print("Confusion matrix saved to confusion_matrix.png")

# --- 3. Cross-validation for robust accuracy estimate ---
# Single train/test split can get lucky or unlucky
# 5-fold CV gives you mean +/- std — a much more honest picture
cv_scores = cross_val_score(
    classification_pipeline,
    newsgroups.data,
    newsgroups.target,
    cv=5,
    scoring='f1_macro',
    n_jobs=-1  # use all available CPU cores
)

print(f"\n=== 5-Fold Cross-Validation ===")
print(f"F1 Macro scores: {cv_scores.round(3)}")
print(f"Mean F1: {cv_scores.mean():.3f} (+/- {cv_scores.std() * 2:.3f})")

# --- 4. Identify the model's worst confusions ---
print("\n=== Top Misclassifications (first 5) ===")
test_texts_array = np.array(test_texts)
incorrect_mask = test_predictions != test_labels
incorrect_texts = test_texts_array[incorrect_mask]
incorrect_true = np.array(test_labels)[incorrect_mask]
incorrect_predicted = np.array(test_predictions)[incorrect_mask]

for i in range(min(3, len(incorrect_texts))):
    true_category = category_names[incorrect_true[i]]
    predicted_category = category_names[incorrect_predicted[i]]
    snippet = incorrect_texts[i][:100].replace('\n', ' ')
    print(f"\nTrue: {true_category} | Predicted: {predicted_category}")
    print(f"Text: '{snippet}...'")

import time
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sentence_transformers import SentenceTransformer

sample_texts = [
    "This product is amazing and works perfectly",
    "Terrible quality, broke after two days",
    "Pretty good value for the price",
    "Worst purchase ever, completely useless",
    "Exceeded my expectations, will buy again"
] * 200  # 1000 texts for benchmarking

# ---- TF-IDF + Logistic Regression ----
vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')
X_tfidf = vectorizer.fit_transform(sample_texts)
labels = np.random.randint(0, 2, len(sample_texts))
model_lr = LogisticRegression(max_iter=1000)
model_lr.fit(X_tfidf, labels)

start = time.perf_counter()
for _ in range(100):
    model_lr.predict(X_tfidf[:1])
print(f"TF-IDF + LR inference (1 sample): {(time.perf_counter()-start)/100*1000:.2f} ms")

# ---- Sentence Transformer + LR ----
embedder = SentenceTransformer('all-MiniLM-L6-v2')
X_emb = embedder.encode(sample_texts[:1])  # warm up
model_emb = LogisticRegression(max_iter=1000)
model_emb.fit(X_emb, labels[:1])

start = time.perf_counter()
for _ in range(100):
    emb = embedder.encode(sample_texts[:1])
    model_emb.predict(emb)
print(f"SentenceTransformer + LR inference (1 sample): {(time.perf_counter()-start)/100*1000:.2f} ms")