Introduction to NLP: How Machines Learn to Understand Human Language

import spacy

# Load the small English model — run `python -m spacy download en_core_web_sm` first
nlp = spacy.load("en_core_web_sm")

raw_review = "The battery life on the new iPhone 15 Pro isn't great, but the camera is absolutely stunning!"

# spaCy processes the text in one call — it runs the full pipeline internally
doc = nlp(raw_review)

print("=== TOKENS AND THEIR PROPERTIES ===")
for token in doc:
    # token.is_stop  → True if this word carries little meaning (e.g. 'the', 'is')
    # token.lemma_   → the dictionary root form of the word
    # token.pos_     → coarse-grained part of speech (NOUN, VERB, ADJ…)
    print(f"  {token.text:<15} lemma={token.lemma_:<12} POS={token.pos_:<8} stop={token.is_stop}")

print("\n=== MEANINGFUL TOKENS ONLY (stop words removed) ===")
meaningful_tokens = [
    token.lemma_.lower()          # normalise to lowercase root form
    for token in doc
    if not token.is_stop           # skip stop words
    and not token.is_punct         # skip punctuation
    and not token.is_space         # skip whitespace tokens
]
print(meaningful_tokens)

print("\n=== NAMED ENTITIES ===")
for entity in doc.ents:
    # entity.label_ tells you WHAT kind of entity it is
    print(f"  '{entity.text}' → {entity.label_} ({spacy.explain(entity.label_)})")

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
import spacy

# spaCy's medium model includes 300-dimensional GloVe word vectors
# Run: python -m spacy download en_core_web_md
nlp = spacy.load("en_core_web_md")

# These are the words we want to compare semantically
word_pairs = [
    ("dog", "puppy"),       # should be very similar
    ("dog", "cat"),         # similar (both animals) but less so
    ("dog", "skyscraper"),  # should be very dissimilar
    ("king", "queen"),      # similar by role, opposite by gender
]

print("=== SEMANTIC SIMILARITY (cosine similarity: 1.0 = identical, 0.0 = unrelated) ===")
for word_a, word_b in word_pairs:
    token_a = nlp(word_a)
    token_b = nlp(word_b)
    # .similarity() computes cosine similarity between the two word vectors
    score = token_a.similarity(token_b)
    print(f"  '{word_a}' ↔ '{word_b}': {score:.4f}")

print("\n=== THE FAMOUS KING - MAN + WOMAN ANALOGY ===")
# Fetch individual word vectors
king_vec   = nlp("king").vector
man_vec    = nlp("man").vector
woman_vec  = nlp("woman").vector

# Arithmetic on vectors: king - man + woman should point toward 'queen'
analogy_vec = king_vec - man_vec + woman_vec

# Compare our analogy vector against a set of candidate words
candidates = ["queen", "princess", "monarch", "prince", "knight", "duchess"]
candidate_vecs = np.array([nlp(word).vector for word in candidates])

# cosine_similarity expects 2D arrays
similarities = cosine_similarity([analogy_vec], candidate_vecs)[0]

# Rank candidates by similarity
ranked = sorted(zip(candidates, similarities), key=lambda pair: pair[1], reverse=True)
print("  king - man + woman is most similar to:")
for rank, (candidate_word, score) in enumerate(ranked, start=1):
    print(f"    {rank}. '{candidate_word}' → {score:.4f}")

# Install dependencies first:
# pip install vaderSentiment transformers torch

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from transformers import pipeline

# --- Approach 1: VADER (rule-based, no GPU needed) ---
vader_analyzer = SentimentIntensityAnalyzer()

# --- Approach 2: HuggingFace transformer (fine-tuned DistilBERT) ---
# 'sentiment-analysis' downloads distilbert-base-uncased-finetuned-sst-2-english
transformer_analyzer = pipeline(
    task="sentiment-analysis",
    model="distilbert-base-uncased-finetuned-sst-2-english",
    truncation=True   # truncate inputs longer than 512 tokens automatically
)

# Sentences chosen deliberately to expose each model's strengths/weaknesses
test_sentences = [
    "This product is absolutely incredible!",            # clear positive
    "The food was okay but nothing special.",             # mild / neutral
    "I can't say I didn't enjoy it.",                    # double negation — tricky
    "This film is so bad it's actually good.",           # irony — very tricky
    "Worst. Purchase. Ever. 🤦",                          # emoji + sarcasm
]

def vader_label(compound_score: float) -> str:
    """Convert VADER compound score to a human-readable label."""
    if compound_score >= 0.05:
        return "POSITIVE"
    elif compound_score <= -0.05:
        return "NEGATIVE"
    return "NEUTRAL"

print(f"{'Sentence':<45} {'VADER':<12} {'Transformer':<12}")
print("-" * 70)

for sentence in test_sentences:
    # VADER returns a dict: {'neg': 0.0, 'neu': 0.5, 'pos': 0.5, 'compound': 0.63}
    vader_scores = vader_analyzer.polarity_scores(sentence)
    vader_result = vader_label(vader_scores["compound"])
    vader_conf   = abs(vader_scores["compound"])  # compound ranges -1 to +1

    # Transformer returns a list of dicts: [{'label': 'POSITIVE', 'score': 0.99}]
    transformer_result = transformer_analyzer(sentence)[0]
    transformer_label  = transformer_result["label"]
    transformer_conf   = transformer_result["score"]

    # Truncate sentence display for clean table formatting
    display_sentence = sentence[:42] + "..." if len(sentence) > 42 else sentence
    print(
        f"{display_sentence:<45} "
        f"{vader_result:<8}({vader_conf:.2f})  "
        f"{transformer_label:<8}({transformer_conf:.2f})"
    )

# Demonstrates the same task (tokenise + POS tag) across NLTK and spaCy
# so you can feel the API difference directly
#
# pip install nltk spacy
# python -m spacy download en_core_web_sm
# python -c "import nltk; nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"

import nltk
import spacy
import time

sample_text = "Apple is acquiring a London-based startup for $1.3 billion to strengthen its AI division."

# ─────────────────────────────────────────────
# APPROACH 1 — NLTK (classic, educational)
# ─────────────────────────────────────────────
print("=== NLTK ===")
nltk_start = time.perf_counter()

# Step 1: tokenise — nltk needs an explicit call per step
nltk_tokens = nltk.word_tokenize(sample_text)

# Step 2: POS tag — separate call, returns list of (word, tag) tuples
nltk_pos_tags = nltk.pos_tag(nltk_tokens)

nltk_duration = time.perf_counter() - nltk_start

for word, tag in nltk_pos_tags:
    print(f"  {word:<20} {tag}")
print(f"  ⏱  {nltk_duration*1000:.2f}ms")

# ─────────────────────────────────────────────
# APPROACH 2 — spaCy (production, fast)
# ─────────────────────────────────────────────
print("\n=== spaCy ===")
nlp = spacy.load("en_core_web_sm")
spacy_start = time.perf_counter()

# One call does tokenisation, tagging, parsing, AND NER simultaneously
doc = nlp(sample_text)

spacy_duration = time.perf_counter() - spacy_start

for token in doc:
    print(f"  {token.text:<20} {token.tag_:<8} ({token.pos_})")
print(f"  ⏱  {spacy_duration*1000:.2f}ms")

# Bonus: spaCy also gives you entities for free in the same pass
print("\n  Entities detected:")
for ent in doc.ents:
    print(f"    {ent.text:<25} → {ent.label_}")