system-prompts-and-models-of-ai-tools/ultimate_ai_agent.py

#!/usr/bin/env python3
"""
Ultimate AI Agent - Multi-Modal AI Agent System
Integrates all workflow capabilities, AI tools, and agent frameworks
"""

import os
import json
import asyncio
import sqlite3
import requests
import subprocess
from datetime import datetime
from typing import Dict, List, Any, Optional, Union
from pathlib import Path
import uvicorn
from fastapi import FastAPI, HTTPException, BackgroundTasks, Request
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel, Field
import aiofiles
import aiofiles.os
from jinja2 import Environment, FileSystemLoader
import openai
import anthropic
import replicate
from dataclasses import dataclass
import logging

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Initialize FastAPI app
app = FastAPI(
    title="Ultimate AI Agent System",
    description="Multi-modal AI agent with integrated workflows and tools",
    version="3.0.0"
)

# Add CORS middleware
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Configuration
CONFIG = {
    "openai_api_key": os.getenv("OPENAI_API_KEY"),
    "anthropic_api_key": os.getenv("ANTHROPIC_API_KEY"),
    "replicate_api_key": os.getenv("REPLICATE_API_KEY"),
    "database_path": "ultimate_agent.db",
    "workflows_dir": "n8n-workflows/workflows",
    "tools_dir": ".",
    "max_concurrent_tasks": 10,
    "default_model": "gpt-4",
    "claude_model": "claude-3-sonnet-20240229"
}

# Data Models
class AgentTask(BaseModel):
    task_id: str
    task_type: str
    description: str
    parameters: Dict[str, Any] = {}
    priority: int = Field(default=1, ge=1, le=10)
    status: str = "pending"
    created_at: datetime = Field(default_factory=datetime.now)
    updated_at: datetime = Field(default_factory=datetime.now)

class AgentResponse(BaseModel):
    task_id: str
    status: str
    result: Optional[Dict[str, Any]] = None
    error: Optional[str] = None
    execution_time: float = 0.0

class WorkflowExecution(BaseModel):
    workflow_name: str
    workflow_data: Dict[str, Any]
    input_data: Dict[str, Any] = {}
    execution_mode: str = "sync"  # sync, async, streaming

@dataclass
class AgentCapability:
    name: str
    description: str
    category: str
    tools: List[str]
    models: List[str]
    workflows: List[str]

class UltimateAIAgent:
    """Ultimate AI Agent with multi-modal capabilities"""
    
    def __init__(self):
        self.capabilities = self._load_capabilities()
        self.active_tasks = {}
        self.workflow_cache = {}
        self.tool_registry = self._load_tools()
        self.model_registry = self._load_models()
        
    def _load_capabilities(self) -> Dict[str, AgentCapability]:
        """Load all agent capabilities from various frameworks"""
        capabilities = {
            "code_generation": AgentCapability(
                name="Code Generation",
                description="Generate, analyze, and modify code across multiple languages",
                category="development",
                tools=["codebase_search", "file_read", "file_write", "run_terminal_cmd"],
                models=["gpt-4", "claude-3-sonnet", "codellama"],
                workflows=["code_review", "bug_fix", "feature_implementation"]
            ),
            "workflow_automation": AgentCapability(
                name="Workflow Automation",
                description="Create and execute complex n8n workflows",
                category="automation",
                tools=["workflow_executor", "api_integration", "data_processing"],
                models=["gpt-4", "claude-3-sonnet"],
                workflows=["data_pipeline", "api_automation", "notification_system"]
            ),
            "ai_analysis": AgentCapability(
                name="AI Analysis",
                description="Analyze data, generate insights, and create reports",
                category="analytics",
                tools=["data_analysis", "visualization", "report_generation"],
                models=["gpt-4", "claude-3-sonnet", "dall-e-3"],
                workflows=["data_analysis", "report_generation", "insight_extraction"]
            ),
            "multimodal_processing": AgentCapability(
                name="Multimodal Processing",
                description="Process text, images, audio, and video content",
                category="multimodal",
                tools=["image_processing", "audio_processing", "video_processing"],
                models=["gpt-4-vision", "claude-3-sonnet", "whisper", "stable-diffusion"],
                workflows=["content_analysis", "media_processing", "creative_generation"]
            ),
            "system_integration": AgentCapability(
                name="System Integration",
                description="Integrate with external APIs and services",
                category="integration",
                tools=["api_client", "webhook_handler", "database_connector"],
                models=["gpt-4", "claude-3-sonnet"],
                workflows=["api_integration", "data_sync", "service_orchestration"]
            )
        }
        return capabilities
    
    def _load_tools(self) -> Dict[str, Dict]:
        """Load all available tools from various frameworks"""
        tools = {}
        
        # Load Cursor tools
        try:
            with open("Cursor Prompts/Agent Tools v1.0.json", "r") as f:
                cursor_tools = json.load(f)
                for tool in cursor_tools:
                    tools[f"cursor_{tool['name']}"] = tool
        except FileNotFoundError:
            logger.warning("Cursor tools not found")
        
        # Load Manus tools
        try:
            with open("Manus Agent Tools & Prompt/tools.json", "r") as f:
                manus_tools = json.load(f)
                for tool in manus_tools:
                    if "function" in tool:
                        tools[f"manus_{tool['function']['name']}"] = tool
        except FileNotFoundError:
            logger.warning("Manus tools not found")
        
        # Add custom tools
        tools.update({
            "workflow_executor": {
                "name": "workflow_executor",
                "description": "Execute n8n workflows with custom parameters",
                "parameters": {
                    "workflow_name": {"type": "string"},
                    "input_data": {"type": "object"},
                    "execution_mode": {"type": "string", "enum": ["sync", "async", "streaming"]}
                }
            },
            "ai_model_caller": {
                "name": "ai_model_caller",
                "description": "Call various AI models for different tasks",
                "parameters": {
                    "model": {"type": "string"},
                    "prompt": {"type": "string"},
                    "parameters": {"type": "object"}
                }
            },
            "data_processor": {
                "name": "data_processor",
                "description": "Process and transform data using various methods",
                "parameters": {
                    "data": {"type": "object"},
                    "operation": {"type": "string"},
                    "parameters": {"type": "object"}
                }
            }
        })
        
        return tools
    
    def _load_models(self) -> Dict[str, Dict]:
        """Load available AI models"""
        return {
            "gpt-4": {
                "provider": "openai",
                "capabilities": ["text", "code", "reasoning"],
                "max_tokens": 8192
            },
            "gpt-4-vision": {
                "provider": "openai",
                "capabilities": ["text", "image", "code", "reasoning"],
                "max_tokens": 4096
            },
            "claude-3-sonnet": {
                "provider": "anthropic",
                "capabilities": ["text", "code", "reasoning"],
                "max_tokens": 200000
            },
            "claude-3-sonnet-vision": {
                "provider": "anthropic",
                "capabilities": ["text", "image", "code", "reasoning"],
                "max_tokens": 200000
            },
            "codellama": {
                "provider": "replicate",
                "capabilities": ["code", "text"],
                "max_tokens": 4096
            },
            "stable-diffusion": {
                "provider": "replicate",
                "capabilities": ["image_generation"],
                "max_tokens": None
            },
            "whisper": {
                "provider": "openai",
                "capabilities": ["audio_transcription"],
                "max_tokens": None
            }
        }
    
    async def execute_task(self, task: AgentTask) -> AgentResponse:
        """Execute a task using the appropriate capability and tools"""
        start_time = datetime.now()
        
        try:
            # Determine capability based on task type
            capability = self._get_capability_for_task(task.task_type)
            
            # Execute task using capability
            if capability.name == "Code Generation":
                result = await self._execute_code_generation(task)
            elif capability.name == "Workflow Automation":
                result = await self._execute_workflow_automation(task)
            elif capability.name == "AI Analysis":
                result = await self._execute_ai_analysis(task)
            elif capability.name == "Multimodal Processing":
                result = await self._execute_multimodal_processing(task)
            elif capability.name == "System Integration":
                result = await self._execute_system_integration(task)
            else:
                result = await self._execute_generic_task(task)
            
            execution_time = (datetime.now() - start_time).total_seconds()
            
            return AgentResponse(
                task_id=task.task_id,
                status="completed",
                result=result,
                execution_time=execution_time
            )
            
        except Exception as e:
            execution_time = (datetime.now() - start_time).total_seconds()
            logger.error(f"Task execution failed: {e}")
            
            return AgentResponse(
                task_id=task.task_id,
                status="failed",
                error=str(e),
                execution_time=execution_time
            )
    
    def _get_capability_for_task(self, task_type: str) -> AgentCapability:
        """Determine the appropriate capability for a task type"""
        task_mapping = {
            "code_generation": "code_generation",
            "workflow_automation": "workflow_automation",
            "ai_analysis": "ai_analysis",
            "multimodal_processing": "multimodal_processing",
            "system_integration": "system_integration"
        }
        
        capability_name = task_mapping.get(task_type, "code_generation")
        return self.capabilities[capability_name]
    
    async def _execute_code_generation(self, task: AgentTask) -> Dict[str, Any]:
        """Execute code generation tasks"""
        # Use AI model to generate code
        model = CONFIG.get("default_model", "gpt-4")
        prompt = f"""
        Task: {task.description}
        Parameters: {json.dumps(task.parameters, indent=2)}
        
        Generate high-quality, production-ready code that accomplishes this task.
        Include proper error handling, documentation, and follow best practices.
        """
        
        response = await self._call_ai_model(model, prompt)
        
        # Extract and format code
        code_blocks = self._extract_code_blocks(response)
        
        return {
            "generated_code": code_blocks,
            "model_used": model,
            "task_type": "code_generation"
        }
    
    async def _execute_workflow_automation(self, task: AgentTask) -> Dict[str, Any]:
        """Execute workflow automation tasks"""
        workflow_name = task.parameters.get("workflow_name")
        input_data = task.parameters.get("input_data", {})
        
        # Load workflow if not cached
        if workflow_name not in self.workflow_cache:
            workflow_data = await self._load_workflow(workflow_name)
            self.workflow_cache[workflow_name] = workflow_data
        
        # Execute workflow
        result = await self._execute_workflow(workflow_name, input_data)
        
        return {
            "workflow_name": workflow_name,
            "execution_result": result,
            "task_type": "workflow_automation"
        }
    
    async def _execute_ai_analysis(self, task: AgentTask) -> Dict[str, Any]:
        """Execute AI analysis tasks"""
        data = task.parameters.get("data")
        analysis_type = task.parameters.get("analysis_type", "general")
        
        # Generate analysis prompt
        prompt = f"""
        Analyze the following data and provide insights:
        
        Data: {json.dumps(data, indent=2)}
        Analysis Type: {analysis_type}
        
        Provide a comprehensive analysis including:
        1. Key insights and patterns
        2. Recommendations
        3. Potential issues or anomalies
        4. Actionable next steps
        """
        
        response = await self._call_ai_model(CONFIG["default_model"], prompt)
        
        return {
            "analysis": response,
            "analysis_type": analysis_type,
            "task_type": "ai_analysis"
        }
    
    async def _execute_multimodal_processing(self, task: AgentTask) -> Dict[str, Any]:
        """Execute multimodal processing tasks"""
        content_type = task.parameters.get("content_type")
        content_data = task.parameters.get("content_data")
        
        if content_type == "image":
            return await self._process_image(content_data, task.parameters)
        elif content_type == "audio":
            return await self._process_audio(content_data, task.parameters)
        elif content_type == "video":
            return await self._process_video(content_data, task.parameters)
        else:
            return await self._process_text(content_data, task.parameters)
    
    async def _execute_system_integration(self, task: AgentTask) -> Dict[str, Any]:
        """Execute system integration tasks"""
        integration_type = task.parameters.get("integration_type")
        api_data = task.parameters.get("api_data", {})
        
        if integration_type == "api_call":
            return await self._make_api_call(api_data)
        elif integration_type == "webhook":
            return await self._handle_webhook(api_data)
        elif integration_type == "database":
            return await self._database_operation(api_data)
        else:
            raise ValueError(f"Unknown integration type: {integration_type}")
    
    async def _execute_generic_task(self, task: AgentTask) -> Dict[str, Any]:
        """Execute generic tasks using AI reasoning"""
        prompt = f"""
        Task: {task.description}
        Parameters: {json.dumps(task.parameters, indent=2)}
        
        Provide a comprehensive solution for this task.
        """
        
        response = await self._call_ai_model(CONFIG["default_model"], prompt)
        
        return {
            "solution": response,
            "task_type": "generic",
            "model_used": CONFIG["default_model"]
        }
    
    async def _call_ai_model(self, model: str, prompt: str, **kwargs) -> str:
        """Call AI model based on provider"""
        model_info = self.model_registry.get(model)
        if not model_info:
            raise ValueError(f"Unknown model: {model}")
        
        provider = model_info["provider"]
        
        if provider == "openai":
            return await self._call_openai_model(model, prompt, **kwargs)
        elif provider == "anthropic":
            return await self._call_anthropic_model(model, prompt, **kwargs)
        elif provider == "replicate":
            return await self._call_replicate_model(model, prompt, **kwargs)
        else:
            raise ValueError(f"Unknown provider: {provider}")
    
    async def _call_openai_model(self, model: str, prompt: str, **kwargs) -> str:
        """Call OpenAI model"""
        if not CONFIG["openai_api_key"]:
            raise ValueError("OpenAI API key not configured")
        
        client = openai.AsyncOpenAI(api_key=CONFIG["openai_api_key"])
        
        response = await client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": prompt}],
            **kwargs
        )
        
        return response.choices[0].message.content
    
    async def _call_anthropic_model(self, model: str, prompt: str, **kwargs) -> str:
        """Call Anthropic model"""
        if not CONFIG["anthropic_api_key"]:
            raise ValueError("Anthropic API key not configured")
        
        client = anthropic.AsyncAnthropic(api_key=CONFIG["anthropic_api_key"])
        
        response = await client.messages.create(
            model=model,
            max_tokens=kwargs.get("max_tokens", 4096),
            messages=[{"role": "user", "content": prompt}]
        )
        
        return response.content[0].text
    
    async def _call_replicate_model(self, model: str, prompt: str, **kwargs) -> str:
        """Call Replicate model"""
        if not CONFIG["replicate_api_key"]:
            raise ValueError("Replicate API key not configured")
        
        # This would need to be implemented based on specific Replicate model
        # For now, return a placeholder
        return f"Replicate model {model} response for: {prompt[:100]}..."
    
    def _extract_code_blocks(self, text: str) -> List[str]:
        """Extract code blocks from AI response"""
        import re
        code_blocks = re.findall(r'```[\w]*\n(.*?)\n```', text, re.DOTALL)
        return code_blocks
    
    async def _load_workflow(self, workflow_name: str) -> Dict[str, Any]:
        """Load workflow from file system"""
        workflow_path = Path(CONFIG["workflows_dir"]) / f"{workflow_name}.json"
        
        if not workflow_path.exists():
            raise FileNotFoundError(f"Workflow not found: {workflow_name}")
        
        async with aiofiles.open(workflow_path, 'r') as f:
            content = await f.read()
            return json.loads(content)
    
    async def _execute_workflow(self, workflow_name: str, input_data: Dict[str, Any]) -> Dict[str, Any]:
        """Execute n8n workflow"""
        # This would integrate with n8n execution engine
        # For now, return a mock result
        return {
            "workflow_name": workflow_name,
            "status": "completed",
            "output": f"Executed workflow {workflow_name} with input: {input_data}",
            "execution_time": 1.5
        }
    
    async def _process_image(self, image_data: Any, parameters: Dict[str, Any]) -> Dict[str, Any]:
        """Process image using AI models"""
        # Implementation for image processing
        return {
            "content_type": "image",
            "analysis": "Image analysis result",
            "processed_data": image_data
        }
    
    async def _process_audio(self, audio_data: Any, parameters: Dict[str, Any]) -> Dict[str, Any]:
        """Process audio using AI models"""
        # Implementation for audio processing
        return {
            "content_type": "audio",
            "transcription": "Audio transcription result",
            "processed_data": audio_data
        }
    
    async def _process_video(self, video_data: Any, parameters: Dict[str, Any]) -> Dict[str, Any]:
        """Process video using AI models"""
        # Implementation for video processing
        return {
            "content_type": "video",
            "analysis": "Video analysis result",
            "processed_data": video_data
        }
    
    async def _process_text(self, text_data: Any, parameters: Dict[str, Any]) -> Dict[str, Any]:
        """Process text using AI models"""
        # Implementation for text processing
        return {
            "content_type": "text",
            "analysis": "Text analysis result",
            "processed_data": text_data
        }
    
    async def _make_api_call(self, api_data: Dict[str, Any]) -> Dict[str, Any]:
        """Make API call to external service"""
        url = api_data.get("url")
        method = api_data.get("method", "GET")
        headers = api_data.get("headers", {})
        data = api_data.get("data")
        
        response = requests.request(method, url, headers=headers, json=data)
        
        return {
            "status_code": response.status_code,
            "response": response.json() if response.headers.get("content-type", "").startswith("application/json") else response.text
        }
    
    async def _handle_webhook(self, webhook_data: Dict[str, Any]) -> Dict[str, Any]:
        """Handle webhook events"""
        # Implementation for webhook handling
        return {
            "webhook_type": webhook_data.get("type"),
            "status": "processed",
            "data": webhook_data
        }
    
    async def _database_operation(self, db_data: Dict[str, Any]) -> Dict[str, Any]:
        """Perform database operations"""
        # Implementation for database operations
        return {
            "operation": db_data.get("operation"),
            "status": "completed",
            "result": "Database operation result"
        }

# Initialize the agent
agent = UltimateAIAgent()

# API Routes
@app.get("/", response_class=HTMLResponse)
async def root():
    """Main dashboard"""
    html_content = """
    <!DOCTYPE html>
    <html>
    <head>
        <title>Ultimate AI Agent System</title>
        <style>
            body { font-family: Arial, sans-serif; margin: 40px; }
            .container { max-width: 1200px; margin: 0 auto; }
            .header { text-align: center; margin-bottom: 40px; }
            .capabilities { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); gap: 20px; }
            .capability { border: 1px solid #ddd; padding: 20px; border-radius: 8px; }
            .capability h3 { color: #333; margin-top: 0; }
            .status { padding: 10px; border-radius: 4px; margin: 10px 0; }
            .status.success { background-color: #d4edda; color: #155724; }
            .status.info { background-color: #d1ecf1; color: #0c5460; }
        </style>
    </head>
    <body>
        <div class="container">
            <div class="header">
                <h1>🤖 Ultimate AI Agent System</h1>
                <p>Multi-modal AI agent with integrated workflows and tools</p>
            </div>
            
            <div class="status success">
                <strong>System Status:</strong> Online and ready to process tasks
            </div>
            
            <h2>Available Capabilities</h2>
            <div class="capabilities">
                <div class="capability">
                    <h3>💻 Code Generation</h3>
                    <p>Generate, analyze, and modify code across multiple languages</p>
                    <ul>
                        <li>Multi-language support</li>
                        <li>Code analysis and optimization</li>
                        <li>Bug detection and fixes</li>
                    </ul>
                </div>
                
                <div class="capability">
                    <h3>⚙️ Workflow Automation</h3>
                    <p>Create and execute complex n8n workflows</p>
                    <ul>
                        <li>Data pipeline automation</li>
                        <li>API integration workflows</li>
                        <li>Notification systems</li>
                    </ul>
                </div>
                
                <div class="capability">
                    <h3>📊 AI Analysis</h3>
                    <p>Analyze data, generate insights, and create reports</p>
                    <ul>
                        <li>Data pattern recognition</li>
                        <li>Predictive analytics</li>
                        <li>Automated reporting</li>
                    </ul>
                </div>
                
                <div class="capability">
                    <h3>🎨 Multimodal Processing</h3>
                    <p>Process text, images, audio, and video content</p>
                    <ul>
                        <li>Image analysis and generation</li>
                        <li>Audio transcription</li>
                        <li>Video processing</li>
                    </ul>
                </div>
                
                <div class="capability">
                    <h3>🔗 System Integration</h3>
                    <p>Integrate with external APIs and services</p>
                    <ul>
                        <li>API orchestration</li>
                        <li>Webhook handling</li>
                        <li>Database operations</li>
                    </ul>
                </div>
            </div>
            
            <div class="status info">
                <strong>API Documentation:</strong> Available at <a href="/docs">/docs</a>
            </div>
        </div>
    </body>
    </html>
    """
    return HTMLResponse(content=html_content)

@app.post("/api/tasks", response_model=AgentResponse)
async def create_task(task: AgentTask, background_tasks: BackgroundTasks):
    """Create and execute a new task"""
    task.task_id = f"task_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{hash(task.description) % 10000}"
    
    # Execute task in background
    background_tasks.add_task(agent.execute_task, task)
    
    return AgentResponse(
        task_id=task.task_id,
        status="started",
        result={"message": f"Task {task.task_id} started successfully"}
    )

@app.get("/api/tasks/{task_id}", response_model=AgentResponse)
async def get_task_status(task_id: str):
    """Get task status and results"""
    # This would typically query a database
    # For now, return a mock response
    return AgentResponse(
        task_id=task_id,
        status="completed",
        result={"message": f"Task {task_id} completed successfully"}
    )

@app.get("/api/capabilities")
async def get_capabilities():
    """Get all available agent capabilities"""
    return {
        "capabilities": {
            name: {
                "name": cap.name,
                "description": cap.description,
                "category": cap.category,
                "tools": cap.tools,
                "models": cap.models,
                "workflows": cap.workflows
            }
            for name, cap in agent.capabilities.items()
        }
    }

@app.get("/api/tools")
async def get_tools():
    """Get all available tools"""
    return {"tools": agent.tool_registry}

@app.get("/api/models")
async def get_models():
    """Get all available AI models"""
    return {"models": agent.model_registry}

@app.post("/api/workflows/execute")
async def execute_workflow(execution: WorkflowExecution):
    """Execute a specific workflow"""
    result = await agent._execute_workflow(execution.workflow_name, execution.input_data)
    return {"result": result}

@app.get("/api/health")
async def health_check():
    """Health check endpoint"""
    return {
        "status": "healthy",
        "timestamp": datetime.now().isoformat(),
        "capabilities_count": len(agent.capabilities),
        "tools_count": len(agent.tool_registry),
        "models_count": len(agent.model_registry)
    }

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=8000)