LLM Tokenization Explained — How a BPE Merge Table Cost Us $4k/Month in Token Waste

import tiktoken

# Load the GPT-4 tokenizer (cl100k_base)
enc = tiktoken.get_encoding("cl100k_base")

# Example: a common word vs a rare word
common = "tokenization"
rare = "pneumonoultramicroscopicsilicovolcanoconiosis"

print(f"'{common}' -> {len(enc.encode(common))} tokens: {enc.encode(common)}")
print(f"'{rare}' -> {len(enc.encode(rare))} tokens: {enc.encode(rare)}")

# The rare word is split into many tokens because the merge table doesn't have it
# Output: 'tokenization' -> 2 tokens: [1929, 1438]
#         'pneumonoultramicroscopicsilicovolcanoconiosis' -> 15 tokens

import tiktoken
from openai import OpenAI

client = OpenAI()

# Map model to encoding
MODEL_ENCODINGS = {
    "gpt-4": "cl100k_base",
    "gpt-4o": "o200k_base",
    "gpt-3.5-turbo": "cl100k_base",
}

def count_tokens(text: str, model: str) -> int:
    encoding_name = MODEL_ENCODINGS.get(model, "cl100k_base")
    enc = tiktoken.get_encoding(encoding_name)
    return len(enc.encode(text))

def safe_chat_completion(messages, model="gpt-4", max_tokens=4096):
    total_tokens = sum(count_tokens(m["content"], model) for m in messages)
    # Account for special tokens: each message adds 2 tokens (role + content)
    total_tokens += 2 * len(messages)
    if total_tokens > max_tokens:
        raise ValueError(f"Prompt too long: {total_tokens} tokens, max {max_tokens}")
    return client.chat.completions.create(model=model, messages=messages)

# Usage
messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Explain tokenization."}
]
response = safe_chat_completion(messages)
print(response.choices[0].message.content)

import tiktoken

# Compare token counts for code across different encodings
code_snippet = """
def fibonacci(n):
    if n <= 1:
        return n
    return fibonacci(n-1) + fibonacci(n-2)
"""

# GPT-4 tokenizer (cl100k_base)
enc_gpt4 = tiktoken.get_encoding("cl100k_base")
print(f"GPT-4 tokens: {len(enc_gpt4.encode(code_snippet))}")

# GPT-4o tokenizer (o200k_base) - optimized for code
enc_gpt4o = tiktoken.get_encoding("o200k_base")
print(f"GPT-4o tokens: {len(enc_gpt4o.encode(code_snippet))}")

# GPT-3.5 tokenizer (p50k_base)
enc_gpt35 = tiktoken.get_encoding("p50k_base")
print(f"GPT-3.5 tokens: {len(enc_gpt35.encode(code_snippet))}")

# Output: GPT-4: 38, GPT-4o: 32, GPT-3.5: 42

import hashlib
import tiktoken
from openai import OpenAI

client = OpenAI()
enc = tiktoken.get_encoding("cl100k_base")

# In-memory cache. In production, use Redis with TTL.
cache = {}

def get_cached_response(prompt: str, model: str = "gpt-4"):
    # Tokenize the prompt
    tokens = enc.encode(prompt)
    # Create a hash of the token IDs
    token_hash = hashlib.sha256(str(tokens).encode()).hexdigest()
    
    if token_hash in cache:
        return cache[token_hash]
    
    # Send to API
    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        temperature=0.0  # Deterministic output for caching
    )
    result = response.choices[0].message.content
    cache[token_hash] = result
    return result

# Test
print(get_cached_response("Hello, world!"))
print(get_cached_response("Hello, world!"))  # This will hit the cache

import tiktoken

enc = tiktoken.get_encoding("cl100k_base")

# Mistake 1: Whitespace
prompt_with_newlines = "Hello\n\n\n\nWorld"
prompt_stripped = "Hello\nWorld"
print(f"With newlines: {len(enc.encode(prompt_with_newlines))} tokens")
print(f"Stripped: {len(enc.encode(prompt_stripped))} tokens")
# Output: 5 vs 3 tokens

# Mistake 2: Special tokens in chat
messages = [
    {"role": "system", "content": "You are helpful."},
    {"role": "user", "content": "Hi"},
    {"role": "assistant", "content": "Hello"},
]
# OpenAI adds 2 special tokens per message
# Simulate: <|im_start|>system\nYou are helpful.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\nHello<|im_end|>
chat_text = "<|im_start|>system\nYou are helpful.<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\nHello<|im_end|>"
print(f"Chat with special tokens: {len(enc.encode(chat_text))} tokens")
# Output: 12 tokens (3 messages * 2 special tokens + 3 content tokens + 3 newlines)

# Mistake 3: Unicode
print(f"'a' tokens: {len(enc.encode('a'))}")  # 1
print(f"'á' tokens: {len(enc.encode('á'))}")  # 2 (accented character)

import tiktoken
from transformers import AutoTokenizer

# BPE (GPT-4)
bpe_enc = tiktoken.get_encoding("cl100k_base")

# SentencePiece (Llama 3)
sp_enc = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")

# WordPiece (BERT)
wp_enc = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

text = "Hello, world! This is a test of tokenization."

print(f"BPE tokens: {len(bpe_enc.encode(text))}")
print(f"SentencePiece tokens: {len(sp_enc.encode(text))}")
print(f"WordPiece tokens: {len(wp_enc.encode(text))}")

# Output varies by tokenizer. Typically BPE ~10, SentencePiece ~11, WordPiece ~12.

import logging
import json
from datetime import datetime
from openai import OpenAI

client = OpenAI()

# Configure structured logging (JSON)
logging.basicConfig(level=logging.INFO, format='{"time": "%(asctime)s", "level": "%(levelname)s", "message": %(message)s}')
logger = logging.getLogger(__name__)

def log_completion(model, messages, response):
    log_entry = {
        "timestamp": datetime.utcnow().isoformat(),
        "model": model,
        "input_tokens": response.usage.prompt_tokens,
        "output_tokens": response.usage.completion_tokens,
        "total_tokens": response.usage.total_tokens,
        "latency_ms": response.response_ms,
        "cache_hit": False,  # Set based on your caching logic
    }
    logger.info(json.dumps(log_entry))

# Usage
response = client.chat.completions.create(
    model="gpt-4",
    messages=[{"role": "user", "content": "Hello"}]
)
log_completion("gpt-4", [{"role": "user", "content": "Hello"}], response)

# No code needed here. The future is uncertain, but the principles are timeless.
# However, here's a thought experiment:
# If a tokenization-free model costs $0.01/1k bytes, and BPE costs $0.01/1k tokens,
# compare: "Hello, world!" is 13 bytes vs 4 tokens.
# Bytes: $0.00013, Tokens: $0.00004. BPE is cheaper for English.
# But for Japanese: "こんにちは" is 15 bytes vs 8 tokens.
# Bytes: $0.00015, Tokens: $0.00008. BPE still cheaper.
# The advantage of tokenization-free is not cost but fairness (no language bias).