Multi-Modal Agent Template
AgentTemplate for building agents that can process text, images, audio, and video.
Multi-Modal Agent Template
Overview
This template provides a foundation for building AI agents that can process and reason across multiple modalities: text, images, audio, and video. Multi-modal agents are essential for real-world applications where information comes in various formats.
Architecture
┌─────────────────────────────────────────────────┐
│ Multi-Modal Agent │
├─────────────────────────────────────────────────┤
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ Text │ │ Image │ │ Audio │ │
│ │ Encoder │ │ Encoder │ │ Encoder │ │
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
│ │ │ │ │
│ └─────────────┴──────┬──────┘ │
│ │ │
│ ┌───────▼───────┐ │
│ │ Fusion Layer │ │
│ └───────┬───────┘ │
│ │ │
│ ┌───────▼───────┐ │
│ │ Reasoning │ │
│ │ Engine │ │
│ └───────┬───────┘ │
│ │ │
│ ┌─────────────────────────▼────────────────┐ │
│ │ Output Handler │ │
│ └──────────────────────────────────────────┘ │
└─────────────────────────────────────────────────┘
Setup
Prerequisites
pip install openai anthropic pillow pydub
pip install transformers torch torchvision
Project Structure
multi-modal-agent/
├── agents/
│ ├── __init__.py
│ ├── base_agent.py
│ ├── text_agent.py
│ ├── image_agent.py
│ ├── audio_agent.py
│ └── multi_modal_agent.py
├── encoders/
│ ├── __init__.py
│ ├── text_encoder.py
│ ├── image_encoder.py
│ └── audio_encoder.py
├── fusion/
│ ├── __init__.py
│ └── fusion_layer.py
├── prompts/
│ ├── system_prompt.txt
│ └── multimodal_prompt.txt
├── utils/
│ ├── __init__.py
│ └── media_processing.py
├── config.py
└── main.py
Core Components
1. Base Agent
# agents/base_agent.py
from abc import ABC, abstractmethod
from typing import Any, Optional
from dataclasses import dataclass
@dataclass
class AgentOutput:
text: str
confidence: float
sources: list[str]
metadata: dict[str, Any]
class BaseAgent(ABC):
def __init__(self, model: str = "claude-3-5-sonnet-20241022"):
self.model = model
@abstractmethod
def process(self, input: Any) -> AgentOutput:
pass
@abstractmethod
def can_handle(self, input_type: str) -> bool:
pass
2. Text Agent
# agents/text_agent.py
from anthropic import Anthropic
from .base_agent import BaseAgent, AgentOutput
class TextAgent(BaseAgent):
def __init__(self, model: str = "claude-3-5-sonnet-20241022"):
super().__init__(model)
self.client = Anthropic()
def can_handle(self, input_type: str) -> bool:
return input_type == "text"
def process(self, text: str) -> AgentOutput:
response = self.client.messages.create(
model=self.model,
max_tokens=1024,
messages=[{"role": "user", "content": text}]
)
return AgentOutput(
text=response.content[0].text,
confidence=1.0,
sources=["text_input"],
metadata={"tokens_used": response.usage.input_tokens}
)
3. Image Agent
# agents/image_agent.py
import base64
from pathlib import Path
from .base_agent import BaseAgent, AgentOutput
class ImageAgent(BaseAgent):
def can_handle(self, input_type: str) -> bool:
return input_type in ["image", "image_url"]
def _encode_image(self, image_path: str) -> str:
with open(image_path, "rb") as f:
return base64.b64encode(f.read()).decode("utf-8")
def process(self, image_path: str, prompt: str = "") -> AgentOutput:
base64_image = self._encode_image(image_path)
content = [
{"type": "text", "text": prompt or "Describe this image."},
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": base64_image,
},
},
]
response = self.client.messages.create(
model=self.model,
max_tokens=1024,
messages=[{"role": "user", "content": content}]
)
return AgentOutput(
text=response.content[0].text,
confidence=0.95,
sources=[image_path],
metadata={"image_size": Path(image_path).stat().st_size}
)
4. Audio Agent
# agents/audio_agent.py
from pydub import AudioSegment
from .base_agent import BaseAgent, AgentOutput
class AudioAgent(BaseAgent):
def can_handle(self, input_type: str) -> bool:
return input_type in ["audio", "audio_url"]
def _transcribe_audio(self, audio_path: str) -> str:
# Use OpenAI Whisper or similar
from openai import OpenAI
client = OpenAI()
audio_file = open(audio_path, "rb")
transcript = client.audio.transcriptions.create(
model="whisper-1",
file=audio_file
)
return transcript.text
def process(self, audio_path: str, prompt: str = "") -> AgentOutput:
transcript = self._transcribe_audio(audio_path)
if prompt:
full_prompt = f"{prompt}\n\nTranscript: {transcript}"
else:
full_prompt = f"Summarize this audio transcript:\n\n{transcript}"
# Process as text
text_agent = TextAgent(self.model)
return text_agent.process(full_prompt)
5. Fusion Layer
# fusion/fusion_layer.py
from typing import List
from agents.base_agent import AgentOutput
class FusionLayer:
"""Combines outputs from multiple modalities into a unified context."""
def __init__(self):
self.context_window = []
def add(self, output: AgentOutput, modality: str):
"""Add a modality output to the context."""
self.context_window.append({
"modality": modality,
"content": output.text,
"confidence": output.confidence,
"sources": output.sources
})
def get_context(self, max_items: int = 5) -> str:
"""Get the fused context for reasoning."""
context_parts = []
for item in self.context_window[-max_items:]:
context_parts.append(
f"[{item['modality'].upper()}] {item['content']}"
)
return "\n\n".join(context_parts)
def clear(self):
"""Clear the context window."""
self.context_window = []
6. Multi-Modal Agent
# agents/multi_modal_agent.py
from typing import Union, List
from pathlib import Path
from .base_agent import BaseAgent, AgentOutput
from .text_agent import TextAgent
from .image_agent import ImageAgent
from .audio_agent import AudioAgent
from fusion.fusion_layer import FusionLayer
class MultiModalAgent:
def __init__(self):
self.text_agent = TextAgent()
self.image_agent = ImageAgent()
self.audio_agent = AudioAgent()
self.fusion = FusionLayer()
self.agents = [
self.text_agent,
self.image_agent,
self.audio_agent,
]
def _find_agent(self, input_type: str):
for agent in self.agents:
if agent.can_handle(input_type):
return agent
raise ValueError(f"No agent can handle: {input_type}")
def process(self, inputs: List[Union[str, Path]]) -> AgentOutput:
"""Process multiple inputs of different modalities."""
self.fusion.clear()
for input_path in inputs:
input_str = str(input_path)
# Determine modality
if input_str.endswith(('.txt', '.md', '.json')):
modality = "text"
with open(input_path) as f:
content = f.read()
output = self.text_agent.process(content)
elif input_str.endswith(('.png', '.jpg', '.jpeg', '.gif')):
modality = "image"
output = self.image_agent.process(input_path)
elif input_str.endswith(('.mp3', '.wav', '.m4a')):
modality = "audio"
output = self.audio_agent.process(input_path)
else:
# Treat as URL or raw text
modality = "text"
output = self.text_agent.process(input_str)
self.fusion.add(output, modality)
# Generate unified response
context = self.fusion.get_context()
final_prompt = f"""
You are analyzing multiple inputs from different modalities.
Fused Context:
{context}
Please provide a comprehensive analysis that considers all inputs.
"""
return self.text_agent.process(final_prompt)
Usage Examples
Example 1: Analyze a Document with Diagram
from agents.multi_modal_agent import MultiModalAgent
agent = MultiModalAgent()
# Process a document and its diagram together
result = agent.process([
"requirements.md",
"architecture_diagram.png"
])
print(result.text)
Example 2: Meeting Analysis
# Analyze meeting recording with slides
result = agent.process([
"meeting_transcript.txt",
"presentation_slides.pdf", # Convert to images first
"whiteboard_photo.jpg"
])
print(result.text)
Example 3: Product Review
# Analyze product with images and description
result = agent.process([
"product_description.txt",
"product_image_1.jpg",
"product_image_2.jpg",
"customer_review.txt"
])
print(result.text)
Configuration
# config.py
from dataclasses import dataclass
@dataclass
class AgentConfig:
# Model settings
primary_model: str = "claude-3-5-sonnet-20241022"
fallback_model: str = "claude-3-haiku-20240307"
# Processing limits
max_context_items: int = 5
max_image_size_mb: float = 10.0
max_audio_duration_sec: int = 300
# Output settings
default_max_tokens: int = 1024
confidence_threshold: float = 0.7
# API settings
openai_api_key: str = ""
anthropic_api_key: str = ""
config = AgentConfig()
Best Practices
- Order matters: Process inputs in a logical order (e.g., text first, then images)
- Context window: Don't overload the fusion layer with too many items
- Error handling: Always handle cases where a modality fails to process
- Cost optimization: Use smaller models for simple tasks, reserve Claude 3.5 Sonnet for complex reasoning
- Caching: Cache processed media to avoid re-processing the same inputs
Pros
- ✅ Handles real-world multi-modal inputs
- ✅ Modular architecture for easy extension
- ✅ Works with existing Claude capabilities
- ✅ Flexible for various use cases
- ✅ Clear separation of concerns
Cons
- ❌ Higher API costs for multi-modal processing
- ❌ More complex error handling
- ❌ Requires multiple API keys (OpenAI for Whisper, Anthropic for reasoning)
- ❌ Image/audio processing can be slow
When to Use
- Document analysis: PDFs with charts, diagrams, and text
- Meeting analysis: Audio recordings + slides + notes
- Product analysis: Images + descriptions + reviews
- Research: Papers with figures, tables, and text
- Any scenario with mixed media inputs