Building RAG with LlamaIndex

Overview

This tutorial walks you through building a production-ready Retrieval-Augmented Generation (RAG) system using LlamaIndex. You'll learn how to ingest data from multiple sources, build an index, implement advanced retrieval strategies, and deploy a working RAG application.

What is RAG?

Retrieval-Augmented Generation (RAG) is a pattern that combines:

1. Retrieval: Finding relevant information from your data
2. Generation: Using an LLM to generate answers based on retrieved context

RAG enables AI systems to answer questions about your specific data without retraining the model.

Prerequisites

pip install llama-index llama-index-llms-openai llama-index-embeddings-openai
pip install llama-index-readers-file

Step 1: Setting Up Your Environment

import os
from llama_index.core import Settings
from llama_index.llms.openai import OpenAI
from llama_index.embeddings.openai import OpenAIEmbedding

# Configure LlamaIndex
os.environ["OPENAI_API_KEY"] = "your-api-key"

Settings.llm = OpenAI(model="gpt-4o")
Settings.embed_model = OpenAIEmbedding(model="text-embedding-3-small")

Step 2: Loading Your Data

From Files

from llama_index.core import SimpleDirectoryReader

# Load all documents from a directory
documents = SimpleDirectoryReader("./data").load_data()

print(f"Loaded {len(documents)} documents")
for doc in documents[:3]:
    print(f"- {doc.metadata['file_name']}")

From Multiple Sources

from llama_index.readers.file import PDFReader, DocxReader
from llama_index.readers.web import SimpleWebPageReader
from llama_index.readers.discord import DiscordReader

# PDF documents
pdf_documents = PDFReader().load_data("./reports/*.pdf")

# Web pages
web_documents = SimpleWebPageReader(html_to_text=True).load_data(
    ["https://example.com/docs/page1", "https://example.com/docs/page2"]
)

# Combine all documents
all_documents = documents + pdf_documents + web_documents

From Databases

from llama_index.readers.database import DatabaseReader

# Connect to PostgreSQL
db_reader = DatabaseReader(
    scheme="postgresql",
    host="localhost",
    port=5432,
    user="postgres",
    password="password",
    database="mydb"
)

# Query data
documents = db_reader.load_data(
    query="SELECT title, content FROM articles WHERE published = true"
)

Step 3: Building the Index

Basic Vector Index

from llama_index.core import VectorStoreIndex

# Build index from documents
index = VectorStoreIndex.from_documents(documents)

# Create query engine
query_engine = index.as_query_engine()

# Ask a question
response = query_engine.query("What is the return policy?")
print(response)

Advanced Index Types

Summary Index

from llama_index.core import SummaryIndex

# Good for document-level queries
summary_index = SummaryIndex.from_documents(documents)
summary_query_engine = summary_index.as_query_engine()

Keyword Table Index

from llama_index.core import KeywordTableIndex

# Good for keyword-based retrieval
keyword_index = KeywordTableIndex.from_documents(documents)

Tree Index

from llama_index.core import TreeIndex

# Good for hierarchical queries
tree_index = TreeIndex.from_documents(documents)

Step 4: Advanced Retrieval Strategies

Hybrid Search

from llama_index.core import VectorStoreIndex, StorageContext
from llama_index.vector_stores.chroma import ChromaVectorStore
import chromadb

# Create Chroma client
chroma_client = chromadb.PersistentClient(path="./chroma_db")
chroma_collection = chroma_client.create_collection("my_collection")

# Create vector store
vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
storage_context = StorageContext.from_defaults(vector_store=vector_store)

# Build index with hybrid search
index = VectorStoreIndex.from_documents(
    documents,
    storage_context=storage_context,
    show_progress=True
)

# Configure hybrid search
query_engine = index.as_query_engine(
    similarity_top_k=5,
    vector_store_kwargs={"vector_store": vector_store}
)

Sentence Window Retrieval

from llama_index.core.node_parser import SentenceWindowNodeParser
from llama_index.core import StorageContext, VectorStoreIndex
from llama_index.postprocessor.metadata_retriever import MetadataReplacementPostProcessor

# Parse documents with window context
node_parser = SentenceWindowNodeParser(
    window_size=3,
    window_metadata_key="window",
    original_text_metadata_key="original_text"
)

# Build index
nodes = node_parser.get_nodes_from_documents(documents)
storage_context = StorageContext.from_defaults()
storage_context.docstore.add_documents(nodes)

index = VectorStoreIndex(nodes, storage_context=storage_context)

# Query with metadata replacement
query_engine = index.as_query_engine(
    node_postprocessors=[
        MetadataReplacementPostProcessor(target_metadata_key="window")
    ]
)

Auto-merging Retrieval

from llama_index.core import StorageContext, VectorStoreIndex
from llama_index.core.node_parser import HierarchicalNodeParser
from llama_index.retrievers.bm25 import BM25Retriever
from llama_index.core.retrievers import RouterRetriever
from llama_index.core.query_engine import RetrieverQueryEngine

# Create hierarchical nodes
node_parser = HierarchicalNodeParser.from_defaults(
    chunk_sizes=[2048, 512, 128]
)
nodes = node_parser.get_nodes_from_documents(documents)

# Store all levels
storage_context = StorageContext.from_defaults()
storage_context.docstore.add_documents(nodes)

# Get leaf nodes for retrieval
leaf_nodes = [n for n in nodes if n.level == 0]

# Create retrievers
vector_retriever = index.as_retriever(similarity_top_k=2)
bm25_retriever = BM25Retriever.from_defaults(
    nodes=leaf_nodes,
    similarity_top_k=2
)

# Router retriever
retriever = RouterRetriever.from_lists(
    retrievers=[vector_retriever, bm25_retriever],
    query_engine_tools=[],
    llm=Settings.llm
)

query_engine = RetrieverQueryEngine.from_args(retriever)

Step 5: Creating a Chat Engine

Context Chat Engine

from llama_index.core import Settings
from llama_index.core.chat_engine import ContextChatEngine

# Create chat engine
chat_engine = index.as_chat_engine(
    chat_mode="context",
    system_prompt="""
    You are a helpful assistant that answers questions based on the provided context.
    Always cite the source of your information.
    If the answer is not in the context, say so.
    """,
    similarity_top_k=5
)

# Chat
response = chat_engine.chat("What is the pricing model?")
print(response)

# Continue conversation
response = chat_engine.chat("How does it compare to competitors?")
print(response)

Conversation Chat Engine

from llama_index.core.chat_engine import ConversationChatEngine

chat_engine = index.as_chat_engine(
    chat_mode="conversation",
    verbose=True
)

response = chat_engine.chat("What are the main features?")

Step 6: Adding Metadata Filtering

from llama_index.core import VectorStoreIndex, StorageContext
from llama_index.vector_stores.chroma import ChromaVectorStore
import chromadb

# Initialize Chroma with metadata
chroma_client = chromadb.PersistentClient(path="./chroma_db")
chroma_collection = chroma_client.create_collection(
    "documents",
    metadata={"hnsw:space": "cosine"}
)

# Add metadata to documents
for doc in documents:
    doc.metadata["category"] = "documentation"
    doc.metadata["year"] = "2024"

# Build index
index = VectorStoreIndex.from_documents(
    documents,
    storage_context=StorageContext.from_defaults(
        vector_store=ChromaVectorStore(chroma_collection)
    )
)

# Query with filters
from llama_index.core.vector_stores.types import MetadataFilters, MetadataFilter

filters = MetadataFilters(
    filters=[
        MetadataFilter(key="category", value="documentation"),
        MetadataFilter(key="year", value="2024", operator="=="),
    ]
)

query_engine = index.as_query_engine(
    filters=filters,
    similarity_top_k=5
)

Step 7: Building a Complete RAG Application

FastAPI Backend

# main.py
from fastapi import FastAPI
from pydantic import BaseModel
from llama_index.core import VectorStoreIndex

app = FastAPI()

# Load and index data once
documents = SimpleDirectoryReader("./data").load_data()
index = VectorStoreIndex.from_documents(documents)
query_engine = index.as_query_engine()

class QueryRequest(BaseModel):
    query: str
    top_k: int = 5

class QueryResponse(BaseModel):
    answer: str
    sources: list[str]
    confidence: float

@app.post("/query", response_model=QueryResponse)
async def query_endpoint(request: QueryRequest):
    response = query_engine.query(request.query)
    
    return QueryResponse(
        answer=str(response),
        sources=[n.node.text[:100] for n in response.source_nodes[:3]],
        confidence=response.metadata.get("confidence", 0.0)
    )

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

Simple Frontend

<!-- index.html -->
<!DOCTYPE html>
<html>
<head>
    <title>RAG Chat</title>
    <style>
        body { font-family: system-ui; max-width: 800px; margin: 0 auto; padding: 20px; }
        .message { margin: 10px 0; padding: 10px; border-radius: 8px; }
        .user { background: #e0f2fe; }
        .assistant { background: #f0fdf4; }
        .sources { font-size: 0.9em; color: #666; margin-top: 8px; }
    </style>
</head>
<body>
    <h1>RAG Chat</h1>
    <div id="chat"></div>
    <input type="text" id="query" placeholder="Ask a question..." 
           onkeypress="if(event.key==='Enter')sendQuery()">
    <button onclick="sendQuery()">Send</button>

    <script>
        async function sendQuery() {
            const input = document.getElementById('query');
            const query = input.value;
            if (!query) return;
            
            // Add user message
            addMessage(query, 'user');
            input.value = '';
            
            // Get response
            const res = await fetch('/query', {
                method: 'POST',
                headers: {'Content-Type': 'application/json'},
                body: JSON.stringify({query})
            });
            const data = await res.json();
            
            // Add assistant message
            addMessage(data.answer, 'assistant', data.sources);
        }
        
        function addMessage(text, type, sources = []) {
            const div = document.createElement('div');
            div.className = `message ${type}`;
            div.innerHTML = `<p>${text}</p>`;
            if (sources.length) {
                div.innerHTML += `<div class="sources">Sources: ${sources.join(' | ')}</div>`;
            }
            document.getElementById('chat').appendChild(div);
        }
    </script>
</body>
</html>

Step 8: Evaluating Your RAG System

Using LlamaIndex Evaluation

from llama_index.core.evaluation import DatasetGenerator, FaithfulnessEvaluator, ResponseEvaluator

# Generate evaluation dataset
eval_documents = documents[:20]
data_generator = DatasetGenerator.from_documents(eval_documents)
eval_questions = data_generator.generate_questions_from_nodes()

# Evaluate faithfulness
faithfulness_evaluator = FaithfulnessEvaluator()

for q in eval_questions[:3]:
    response = query_engine.query(q)
    result = faithfulness_evaluator.evaluate_response(response)
    print(f"Question: {q}")
    print(f"Faithfulness: {result.passing}")
    print(f"Score: {result.score}")
    print()

Common Issues and Solutions

Issue: Low Retrieval Accuracy

Solutions:

1. Increase similarity_top_k
2. Use hybrid search (vector + keyword)
3. Improve chunking strategy
4. Add metadata filters
5. Use re-ranking

Issue: Slow Response Time

Solutions:

1. Use smaller embeddings model
2. Cache frequent queries
3. Use async queries
4. Optimize chunk sizes
5. Use a faster vector store (Qdrant, Weaviate)

Issue: Hallucinated Answers

Solutions:

1. Use faithfulness evaluation
2. Add "I don't know" handling
3. Use stricter similarity thresholds
4. Implement answer verification
5. Use citation requirements

Best Practices

1. Chunk Size: Start with 512-1024 tokens, adjust based on your data
2. Embeddings: Use text-embedding-3-small for cost efficiency, text-embedding-3-large for accuracy
3. Hybrid Search: Always consider hybrid search for better recall
4. Evaluation: Always evaluate before deploying
5. Monitoring: Track query performance and user feedback

Resources

• LlamaIndex Documentation
• LlamaIndex GitHub
• RAG Best Practices
• LlamaHub (Data Connectors)