Build desktop automation agents that can see, understand, and interact with UI elements.

Multi-modal Agents with UI-TARS

Overview

UI-TARS (User Interface - Task Automation and Reasoning System) is ByteDance's open-source multimodal AI agent stack for desktop UI automation. This tutorial shows you how to build agents that can see, understand, and interact with desktop applications.

Prerequisites

Python 3.10+
UI-TARS model access (via API or local deployment)
Desktop OS (Windows, macOS, or Linux)
~16GB RAM for local inference

Installation

# Install UI-TARS
pip install ui-tars

# Or with all dependencies
pip install ui-tars[desktop,vision]

# For local model (requires API access)
pip install ui-tars[local]

Architecture Overview

┌─────────────────────────────────────────────────────────────┐
│                      UI-TARS Agent                          │
├─────────────────────────────────────────────────────────────┤
│  ┌──────────┐    ┌──────────┐    ┌──────────┐             │
│  │  Vision  │───▶│ Planning │───▶│ Action   │             │
│  │ Encoder  │    │  Engine  │    │ Executor │             │
│  └──────────┘    └──────────┘    └──────────┘             │
│       │               │               │                    │
│       ▼               ▼               ▼                    │
│  ┌──────────┐    ┌──────────┐    ┌──────────┐             │
│  │  Screen  │    │  Task    │    │  UI      │             │
│  │ Capture  │    │  Queue   │    │ Control  │             │
│  └──────────┘    └──────────┘    └──────────┘             │
└─────────────────────────────────────────────────────────────┘

Step 1: Basic Setup

Configuration

# config.py
from ui_tars import UIAgent

# Initialize agent
agent = UIAgent(
    model="ui-tars-desktop",
    vision_encoder="clip-vit-large",
    action_space="desktop"
)

Screen Capture

# src/capture.py
from ui_tars.vision import ScreenCapture

capture = ScreenCapture()

# Capture current screen
screenshot = capture.capture()

# Capture specific region
region_screenshot = capture.capture_region(x=100, y=100, width=500, height=300)

# Continuous capture for real-time interaction
for frame in capture.stream():
    process_frame(frame)

Step 2: Visual Understanding

Element Detection

# src/detection.py
from ui_tars.vision import ElementDetector

detector = ElementDetector()

# Detect all interactive elements
elements = detector.detect_elements(screenshot)

for element in elements:
    print(f"Element: {element.type}")
    print(f"  Position: {element.bbox}")
    print(f"  Text: {element.text}")
    print(f"  Confidence: {element.confidence}")

UI Hierarchy Parsing

# src/hierarchy.py
from ui_tars.vision import UIHierarchyParser

parser = UIHierarchyParser()

# Parse UI hierarchy
hierarchy = parser.parse(screenshot)

# Navigate hierarchy
def find_button(hierarchy, text: str):
    for element in hierarchy.iter_elements():
        if element.type == "button" and text.lower() in element.text.lower():
            return element
    return None

button = find_button(hierarchy, "Submit")

Step 3: Task Planning

Goal-Oriented Planning

# src/planning.py
from ui_tars.planning import TaskPlanner

planner = TaskPlanner(model="ui-tars-planner")

# Decompose complex task
task = "Book a flight from New York to London for next Friday"
subtasks = planner.decompose(task)

for i, subtask in enumerate(subtasks):
    print(f"{i+1}. {subtask}")

# Output:
# 1. Open browser
# 2. Navigate to flight booking website
# 3. Enter departure city: New York
# 4. Enter destination: London
# 5. Select date: next Friday
# 6. Search for flights
# 7. Select best option
# 8. Enter passenger details
# 9. Complete booking

State Tracking

# src/state.py
from ui_tars.state import AgentState

state = AgentState()

# Update state after each action
state.update(
    current_task="Book flight",
    completed_steps=["Open browser", "Navigate to website"],
    current_step="Enter departure city",
    remaining_steps=["Enter destination", "Select date", ...],
    screenshots=[screenshot_1, screenshot_2]
)

# Get progress
progress = state.get_progress()
print(f"Progress: {progress['percentage']}%")

Step 4: Action Execution

Basic Actions

# src/actions.py
from ui_tars.actions import ActionExecutor

executor = ActionExecutor()

# Click on element
executor.click(element=browser_address_bar)

# Type text
executor.type(text="https://example.com", clear_first=True)

# Scroll
executor.scroll(direction="down", amount=500)

# Keyboard shortcuts
executor.hotkey("ctrl", "f")  # Open find dialog

Complex Interactions

# Drag and drop
executor.drag_and_drop(
    source=drag_source_element,
    target=drag_target_element
)

# Right-click context menu
executor.right_click(element=file_element)
executor.click(element=menu_delete_option)

# Hover for tooltips
executor.hover(element=help_icon)
tooltip_text = executor.capture_tooltip()

Step 5: Complete Workflow

End-to-End Example

# src/workflow.py
from ui_tars import UIAgent
from ui_tars.vision import ScreenCapture, ElementDetector
from ui_tars.planning import TaskPlanner
from ui_tars.actions import ActionExecutor

class AutomationWorkflow:
    def __init__(self):
        self.agent = UIAgent()
        self.capture = ScreenCapture()
        self.detector = ElementDetector()
        self.planner = TaskPlanner()
        self.executor = ActionExecutor()
    
    def run(self, task: str):
        """Execute complete automation workflow."""
        
        # 1. Plan the task
        subtasks = self.planner.decompose(task)
        
        # 2. Execute each subtask
        for subtask in subtasks:
            print(f"Executing: {subtask}")
            
            # Capture current state
            screenshot = self.capture.capture()
            
            # Detect relevant elements
            elements = self.detector.detect_elements(screenshot)
            
            # Plan action for this subtask
            action = self.agent.plan_action(subtask, elements, screenshot)
            
            # Execute action
            result = self.executor.execute(action)
            
            # Verify result
            if not self.verify_result(subtask, screenshot):
                print(f"Warning: {subtask} may not have completed correctly")
        
        print("Task completed!")

# Usage
workflow = AutomationWorkflow()
workflow.run("Open Chrome, navigate to google.com, search for 'AI agents'")

Advanced Features

Multi-Step Reasoning

# src/reasoning.py
from ui_tars.reasoning import ReasoningEngine

engine = ReasoningEngine()

def handle_error(error_screenshot: str, last_action: str) -> dict:
    """Reason about errors and suggest recovery."""
    
    analysis = engine.analyze(
        screenshot=error_screenshot,
        last_action=last_action,
        expected_state="login_success"
    )
    
    return {
        'error_type': analysis.error_type,
        'cause': analysis.cause,
        'recovery_action': analysis.recovery_action
    }

# Example: Login failed
result = handle_error(
    error_screenshot=login_error_screen,
    last_action="click_login_button"
)
print(f"Error: {result['error_type']}")
print(f"Recovery: {result['recovery_action']}")

Learning from Experience

# src/learning.py
from ui_tars.learning import ExperienceLearner

learner = ExperienceLearner()

# Store successful action sequences
learner.record_success(
    task="login_to_website",
    action_sequence=[
        ("click", "username_field"),
        ("type", "my_username"),
        ("click", "password_field"),
        ("type", "my_password"),
        ("click", "login_button")
    ],
    screenshot_sequence=[s1, s2, s3, s4, s5]
)

# Retrieve similar patterns
similar = learner.find_similar("sign_in_to_app")

Cross-Application Workflows

# src/cross_app.py
async def cross_application_workflow():
    """Work across multiple applications."""
    
    # Start in Chrome
    chrome_screenshot = capture.capture()
    chrome_elements = detector.detect_elements(chrome_screenshot)
    
    # Find and click link that opens another app
    link = find_element(chrome_elements, "Open in Desktop App")
    executor.click(link)
    
    # Wait for new app to open
    await asyncio.sleep(2)
    
    # Now work in the new app
    new_app_screenshot = capture.capture()
    new_app_elements = detector.detect_elements(new_app_screenshot)
    
    # Continue workflow
    ...

Best Practices

1. Error Recovery

async def robust_execute(action: dict, max_retries: int = 3):
    """Execute action with error recovery."""
    
    for attempt in range(max_retries):
        screenshot_before = capture.capture()
        result = executor.execute(action)
        
        if verify_action_success(action, screenshot_before):
            return result
        
        # Analyze failure
        error_analysis = engine.analyze_error(
            screenshot_before,
            result.error_message if result else None
        )
        
        # Adjust action
        action = error_analysis.adjusted_action
    
    raise MaxRetriesExceeded(f"Action failed after {max_retries} attempts")

2. Performance Optimization

# Cache element positions
from functools import lru_cache

@lru_cache(maxsize=100)
def get_element_position(element_id: str) -> tuple:
    """Cache element positions for faster access."""
    return detector.find_element(element_id).bbox

# Batch element detection
elements = detector.detect_elements_batch([
    screenshot1, screenshot2, screenshot3
])

3. Human-in-the-Loop

def human_confirmation_required(action: dict) -> bool:
    """Determine if human confirmation is needed."""
    
    sensitive_actions = ['delete', 'purchase', 'transfer']
    
    if any(keyword in action['type'] for keyword in sensitive_actions):
        return True
    
    return False

# Request confirmation
if human_confirmation_required(action):
    confirmation = request_human_confirmation(action)
    if not confirmation:
        skip_action(action)

Troubleshooting

Common Issues

Screen capture fails:

# Check permissions (macOS)
# System Preferences > Security & Privacy > Privacy > Screen Recording

# Check permissions (Windows)
# Ensure app has screen capture rights

Element detection inaccurate:

# Increase detection confidence threshold
elements = detector.detect_elements(
    screenshot,
    confidence_threshold=0.8  # Higher = more accurate but fewer results
)

# Use specific element types
elements = detector.detect_elements(
    screenshot,
    element_types=['button', 'input', 'link']
)

Action execution fails:

# Add delays for UI responsiveness
await asyncio.sleep(0.5)  # Wait for UI to update

# Use explicit waits
executor.wait_for_element("submit_button", timeout=10)

Resources

UI-TARS GitHub: https://github.com/bytedance/UI-TARS-desktop
UI-TARS Documentation: https://bytedance.github.io/UI-TARS-desktop/
Model Access: Contact ByteDance for API access

Last updated: May 2026