sci-gui-agent-benchmark/mm_agents/maestro/cli_app_maestro.py

import argparse
from ast import arg
import datetime
import io
import logging
import os
import platform
import sys
import time
from pathlib import Path
from dotenv import load_dotenv
from PIL import Image
from gui_agents.maestro.controller.main_controller import MainController
# Import analyze_display functionality
from gui_agents.utils.analyze_display import analyze_display_json, aggregate_results, format_output_line
from gui_agents.utils.common_utils import show_task_completion_notification
from desktop_env.desktop_env import DesktopEnv
from gui_agents.utils.common_utils import ImageDataFilter, SafeLoggingFilter

env_path = Path(os.path.dirname(os.path.abspath(__file__))) / '.env'
if env_path.exists():
    load_dotenv(dotenv_path=env_path)
else:
    parent_env_path = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) / '.env'
    if parent_env_path.exists():
        load_dotenv(dotenv_path=parent_env_path)

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

datetime_str: str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")

log_dir = "runtime"
os.makedirs(os.path.join(log_dir, datetime_str), exist_ok=True)

file_handler = logging.FileHandler(
    os.path.join(log_dir, datetime_str, "agents3.log"), encoding="utf-8"
)
debug_handler = logging.FileHandler(
    os.path.join(log_dir, datetime_str, "agents3_debug.log"), encoding="utf-8"
)
stdout_handler = logging.StreamHandler(sys.stdout)

# Add dedicated doubao API log handler
doubao_handler = logging.FileHandler(
    os.path.join(log_dir, datetime_str, "doubao_api.log"), encoding="utf-8"
)

# Create dedicated doubao API logger
doubao_logger = logging.getLogger("doubao_api")
doubao_logger.setLevel(logging.DEBUG)
doubao_logger.addHandler(doubao_handler)

file_handler.setLevel(logging.INFO)
debug_handler.setLevel(logging.DEBUG)
stdout_handler.setLevel(logging.INFO)
doubao_handler.setLevel(logging.DEBUG)

# Add SafeLoggingFilter to prevent format errors from third-party libraries (like OpenAI)
safe_filter = SafeLoggingFilter()
debug_handler.addFilter(safe_filter)

# Also apply SafeLoggingFilter to OpenAI library loggers
try:
    import openai
    openai_logger = logging.getLogger('openai')
    openai_logger.addFilter(safe_filter)
    httpx_logger = logging.getLogger('httpx')
    httpx_logger.addFilter(safe_filter)
    logger.info("SafeLoggingFilter applied to third-party libraries (OpenAI, HTTPX)")
except ImportError:
    logger.info("SafeLoggingFilter applied to main handlers only (OpenAI not available)")
    pass

if os.getenv('KEEP_IMAGE_LOGS', 'false').lower() != 'true':
    image_filter = ImageDataFilter()
    debug_handler.addFilter(image_filter)
    logger.info("Image data filtering enabled - image data in debug logs will be filtered")
else:
    logger.info("Image data filtering disabled - debug logs will contain complete image data")

formatter = logging.Formatter(
    fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s"
)
file_handler.setFormatter(formatter)
debug_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
doubao_handler.setFormatter(formatter)

logger.addHandler(file_handler)
logger.addHandler(debug_handler)
logger.addHandler(stdout_handler)


def auto_analyze_execution(timestamp_dir: str):
    """
    Automatically analyze execution statistics from display.json files after task completion
    
    Args:
        timestamp_dir: Directory containing the execution logs and display.json
    """
    import time
    
    try:
        # Analyze the display.json file for this execution
        display_json_path = os.path.join(timestamp_dir, "display.json")
        
        # Wait for file to be fully written
        max_wait_time = 10  # Maximum wait time in seconds
        wait_interval = 0.5  # Check every 0.5 seconds
        waited_time = 0
        
        while waited_time < max_wait_time:
            if os.path.exists(display_json_path):
                # Check if file is still being written by monitoring its size
                try:
                    size1 = os.path.getsize(display_json_path)
                    time.sleep(wait_interval)
                    size2 = os.path.getsize(display_json_path)
                    
                    # If file size hasn't changed in the last 0.5 seconds, it's likely complete
                    if size1 == size2:
                        logger.info(f"Display.json file appears to be complete (size: {size1} bytes)")
                        break
                    else:
                        logger.info(f"Display.json file still being written (size changed from {size1} to {size2} bytes)")
                        waited_time += wait_interval
                        continue
                except OSError:
                    # File might be temporarily inaccessible
                    time.sleep(wait_interval)
                    waited_time += wait_interval
                    continue
            else:
                logger.info(f"Waiting for display.json file to be created... ({waited_time:.1f}s)")
                time.sleep(wait_interval)
                waited_time += wait_interval
        
        if os.path.exists(display_json_path):
            logger.info(f"Auto-analyzing execution statistics from: {display_json_path}")
            
            # Analyze the single display.json file
            result = analyze_display_json(display_json_path)
            
            if result:
                # Format and log the statistics
                output_line = format_output_line(result)
                logger.info("=" * 80)
                logger.info("EXECUTION STATISTICS:")
                logger.info("Steps, Duration (seconds), (Input Tokens, Output Tokens, Total Tokens), Cost")
                logger.info("=" * 80)
                logger.info(output_line)
                logger.info("=" * 80)
                
                # Also print to console for immediate visibility
                print("\n" + "=" * 80)
                print("EXECUTION STATISTICS:")
                print("Steps, Duration (seconds), (Input Tokens, Output Tokens, Total Tokens), Cost")
                print("=" * 80)
                print(output_line)
                print("=" * 80)
            else:
                logger.warning("No valid data found in display.json for analysis")
        else:
            logger.warning(f"Display.json file not found at: {display_json_path} after waiting {max_wait_time} seconds")
            
    except Exception as e:
        logger.error(f"Error during auto-analysis: {e}")


def run_agent_maestro(params: dict):
    """
    Run the maestro controller with the given instruction
    
    Args:
        controller: The NewController instance to run
        instruction: The instruction/task to execute
        max_steps: Maximum number of steps to execute
    """

    backend = params["backend"]
    user_query = params["query"]
    max_steps = params["max_steps"]
    current_platform = params["current_platform"]
    env = params["env"]
    env_password = params["env_password"]

    import time
    
    logger.info(f"Starting maestro execution with instruction: {user_query}")
    
    total_start_time = time.time()
    # Ensure necessary directory structure exists
    timestamp_dir = os.path.join(log_dir, datetime_str)
    cache_dir = os.path.join(timestamp_dir, "cache", "screens")
    state_dir = os.path.join(timestamp_dir, "state")

    os.makedirs(cache_dir, exist_ok=True)
    os.makedirs(state_dir, exist_ok=True)

    # registry = Registry(global_state)

    # Initialize NewController (which includes all other components)
    controller = MainController(
        platform=current_platform,
        backend=backend,
        user_query=user_query,
        max_steps=max_steps,
        env=env,
        env_password=env_password,
        log_dir=log_dir,
        datetime_str=datetime_str
    )
    
    try:
        # Set the user query in the controller
        controller.execute_main_loop()
        
        # Check task status after execution to determine if task was successful
        task = controller.global_state.get_task()
        if task and task.status == "fulfilled":
            # Task completed successfully
            logger.info("Task completed successfully")
            show_task_completion_notification("success")
        elif task and task.status == "rejected":
            # Task was rejected/failed
            logger.info("Task was rejected/failed")
            show_task_completion_notification("failed")
        else:
            # Task status unknown or incomplete
            logger.info("Task execution completed with unknown status")
            show_task_completion_notification("completed")
        
    except Exception as e:
        logger.error(f"Error during maestro execution: {e}")
        # Show error notification
        show_task_completion_notification("error", str(e))
        raise
    
    finally:
        total_end_time = time.time()
        total_duration = total_end_time - total_start_time
        logger.info(f"Total execution time: {total_duration:.2f} seconds")
        
        # Auto-analyze execution statistics after task completion
        auto_analyze_execution(timestamp_dir)


def main():
    parser = argparse.ArgumentParser(description='Maestro CLI Application')
    parser.add_argument(
        '--backend',
        type=str,
        default='lybic',
        help='Backend to use (e.g., lybic, pyautogui, pyautogui_vmware)')
    parser.add_argument('--query',
                        type=str,
                        default='',
                        help='Initial query to execute')
    parser.add_argument('--max-steps',
                        type=int,
                        default=50,
                        help='Maximum number of steps to execute (default: 50)')
    parser.add_argument(
        '--lybic-sid',
        type=str,
        default=None,
        help='Lybic precreated sandbox ID (if not provided, will use LYBIC_PRECREATE_SID environment variable)')
    args = parser.parse_args()

    env = None
    env_password = ""

    # Set platform to Windows if backend is lybic
    if args.backend == 'lybic':
        current_platform = 'Windows'
        # Initialize hardware interface
        backend_kwargs = {"platform": current_platform}
        if args.lybic_sid is not None:
            backend_kwargs["precreate_sid"] = args.lybic_sid
            logger.info(f"Using Lybic SID from command line: {args.lybic_sid}")
        else:
            logger.info("Using Lybic SID from environment variable LYBIC_PRECREATE_SID")

    elif args.backend == 'pyautogui_vmware':
        env_password = "password"
        current_platform = os.getenv("USE_PRECREATE_VM", "Windows")
        if current_platform == "Ubuntu":
            path_to_vm = os.path.join("vmware_vm_data", "Ubuntu0", "Ubuntu0.vmx")
        elif current_platform == "Windows":
            path_to_vm = os.path.join("vmware_vm_data", "Windows0", "Windows0.vmx")
        else:
            raise ValueError(f"USE_PRECREATE_VM={current_platform} is not supported. Please use Ubuntu or Windows.")

        env = DesktopEnv(
            path_to_vm=path_to_vm,
            provider_name="vmware", 
            os_type=current_platform, 
            action_space="pyautogui",
            require_a11y_tree=False
        )
        env.reset()

    else:
        current_platform = platform.system()

    logger.info(f"Running maestro on platform: {current_platform}")
    logger.info(f"Using backend: {args.backend}")

    logger.info("Maestro components initialized successfully")            

    params = {
        "backend": args.backend,
        "query": '',
        "max_steps": args.max_steps,
        "current_platform": current_platform,
        "env": env,
        "env_password": env_password
    }
    # if query is provided, run the agent on the query
    if args.query:
        logger.info(f"Executing query: {args.query}")
        params["query"] = args.query
        run_agent_maestro(params)

    else:
        while True:
            query = input("Query: ")
            params["query"] = query
            # Run the agent on your own device
            run_agent_maestro(params)

            response = input("Would you like to provide another query? (y/n): ")
            if response.lower() != "y":
                break


if __name__ == "__main__":
    """
    python gui_agents/cli_app_maestro.py --backend lybic
    python gui_agents/cli_app_maestro.py --backend pyautogui --max-steps 1
    python gui_agents/cli_app_maestro.py --backend pyautogui_vmware --max-steps 1
    python gui_agents/cli_app_maestro.py --backend lybic --max-steps 15
    python gui_agents/cli_app_maestro.py --backend lybic --lybic-sid SBX-01K1X6ZKAERXAN73KTJ1XXJXAF
    """
    main()