Advanced RAG Techniques — The 800ms P99 That Taught Us Chunking Isn't Free

from langchain_experimental.text_splitter import SemanticChunker
from langchain_openai import OpenAIEmbeddings

# Default threshold is 0.5, tuned for generic text
# For financial docs, lower threshold to 0.3
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
splitter = SemanticChunker(
    embeddings,
    breakpoint_threshold=0.3,  # Tuned for dense entity text
    # Add this to prevent silent truncation
    max_chunk_size=2000  # Keep chunks under 2000 chars for safety
)

with open("financial_report.txt") as f:
    text = f.read()

chunks = splitter.split_text(text)

# Validate chunk boundaries
for i, chunk in enumerate(chunks):
    if not chunk.rstrip().endswith('.'):
        print(f"WARNING: Chunk {i} ends mid-sentence: {chunk[-100:]}")

from langchain_openai import ChatOpenAI
from openai import OpenAI
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
embed_client = OpenAI()

def rewrite_query(original_query: str) -> str:
    prompt = f"""Rewrite the following query for better retrieval. 
    CRITICAL: Preserve all negations (not, never, without, etc.) and logical operators (AND, OR, NOT).
    Original: {original_query}
    Rewritten: """
    response = llm.invoke(prompt)
    rewritten = response.content.strip()
    
    # Validate semantic similarity
    orig_emb = embed_client.embeddings.create(
        input=original_query, model="text-embedding-3-small"
    ).data[0].embedding
    rew_emb = embed_client.embeddings.create(
        input=rewritten, model="text-embedding-3-small"
    ).data[0].embedding
    
    similarity = cosine_similarity([orig_emb], [rew_emb])[0][0]
    
    if similarity < 0.6:
        print(f"WARNING: Rewrite changed meaning. Similarity: {similarity:.2f}. Falling back to original.")
        return original_query
    
    return rewritten

from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = Chroma(embedding_function=embeddings, persist_directory="./chroma_db")

def hyde_retrieve(query: str, k: int = 5):
    # Step 1: Generate hypothetical document
    hyde_prompt = f"""Generate a short, factual document that would answer the following query.
    Do NOT invent any facts. If you don't know, say 'I don't know'.
    Query: {query}
    Document: """
    hyde_doc = llm.invoke(hyde_prompt).content
    
    # Step 2: Check for hallucinations
    if "I don't know" in hyde_doc or len(hyde_doc) < 20:
        print("HyDE failed to generate a valid document. Falling back to query.")
        return vectorstore.similarity_search(query, k=k)
    
    # Step 3: Use HyDE document for retrieval
    return vectorstore.similarity_search(hyde_doc, k=k)

from sentence_transformers import CrossEncoder
import numpy as np

# Load cross-encoder model (download once)
reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')

def rerank(query: str, documents: list[str], top_k: int = 5):
    # Create pairs: (query, doc) for each document
    pairs = [(query, doc) for doc in documents]
    
    # Batch predict for efficiency
    scores = reranker.predict(pairs, batch_size=32)
    
    # Sort by score descending
    ranked_indices = np.argsort(scores)[::-1]
    
    # Return top_k documents
    return [documents[i] for i in ranked_indices[:top_k]]

# Usage: retriever returns 20 docs, reranker picks top 5
retrieved_docs = retriever.get_relevant_documents(query, k=20)
final_docs = rerank(query, [doc.page_content for doc in retrieved_docs], top_k=5)

import time
from rank_bm25 import BM25Okapi
from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

# Simple BM25
corpus = ["Paris is the capital of France.", "London is the capital of the UK."]
tokenized_corpus = [doc.split() for doc in corpus]
bm25 = BM25Okapi(tokenized_corpus)

query = "capital of France"
tokenized_query = query.split()
start = time.time()
bm25_scores = bm25.get_scores(tokenized_query)
bm25_time = time.time() - start
print(f"BM25 latency: {bm25_time*1000:.2f}ms")

# Advanced RAG
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = Chroma.from_texts(corpus, embeddings)
start = time.time()
results = vectorstore.similarity_search(query, k=1)
advanced_time = time.time() - start
print(f"Advanced RAG latency: {advanced_time*1000:.2f}ms")

import redis
import hashlib
from langchain_openai import OpenAIEmbeddings

# Redis cache for embeddings
cache = redis.Redis(host='localhost', port=6379, decode_responses=True)
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

def get_embedding_with_cache(text: str):
    # Hash the text to use as cache key
    text_hash = hashlib.sha256(text.encode()).hexdigest()
    
    # Check cache
    cached = cache.get(f"emb:{text_hash}")
    if cached:
        return cached
    
    # Compute embedding
    embedding = embeddings.embed_query(text)
    
    # Cache for 1 hour
    cache.setex(f"emb:{text_hash}", 3600, str(embedding))
    
    return embedding

from langchain.text_splitter import RecursiveCharacterTextSplitter

def clean_chunks(chunks: list[str], min_chars: int = 50) -> list[str]:
    """Remove empty or near-empty chunks."""
    return [c for c in chunks if len(c.strip()) >= min_chars]

# Example: split a document and filter
splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=50)
raw_chunks = splitter.split_text(document)
clean = clean_chunks(raw_chunks)
print(f"Removed {len(raw_chunks) - len(clean)} empty chunks")

# Pseudo-code for hybrid approach
# Step 1: Fine-tune a model on output format (e.g., legal summaries)
# from openai import OpenAI
# client = OpenAI()
# client.fine_tuning.jobs.create(
#     training_file="legal_summaries.jsonl",
#     model="gpt-4o-mini-2024-07-18"
# )

# Step 2: Use the fine-tuned model with RAG
from langchain_openai import ChatOpenAI
from langchain_community.vectorstores import Chroma

# Use the fine-tuned model ID
ft_model = "ft:gpt-4o-mini:my-org:legal-summarizer:abc123"
llm = ChatOpenAI(model=ft_model, temperature=0)

# Retrieve context
vectorstore = Chroma(...)
context = vectorstore.similarity_search(query, k=5)

# Generate with fine-tuned model + RAG context
response = llm.invoke(f"Context: {context}\n\nQuery: {query}")