Merge remote-tracking branch 'upstream/feat/aws-provider-support'

.gitignore

@@ -195,4 +195,6 @@ vmware_vm_data
 # result
 **/result*/**/*
 
-.vscode
+.vscode
+
+dataimpulse_proxy_config.json

desktop_env/controllers/setup.py

@@ -21,11 +21,20 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from desktop_env.controllers.python import PythonController
 from desktop_env.evaluators.metrics.utils import compare_urls
+from desktop_env.providers.aws.proxy_pool import get_global_proxy_pool, init_proxy_pool, ProxyInfo
+
+import dotenv
+# Load environment variables from .env file
+dotenv.load_dotenv()
+
+CLIENT_PASSWORD = os.getenv("CLIENT_PASSWORD", "password")  # Default password for sudo operations
+PROXY_CONFIG_FILE = os.getenv("PROXY_CONFIG_FILE", "dataimpulse_proxy_config.json")  # Default proxy config file
 
 logger = logging.getLogger("desktopenv.setup")
 
 FILE_PATH = os.path.dirname(os.path.abspath(__file__))
 
+init_proxy_pool(PROXY_CONFIG_FILE)  # initialize the global proxy pool
 
 class SetupController:
     def __init__(self, vm_ip: str, server_port: int = 5000, chromium_port: int = 9222, vlc_port: int = 8080, cache_dir: str = "cache"):
@@ -360,6 +369,51 @@ class SetupController:
         except requests.exceptions.RequestException as e:
             logger.error("An error occurred while trying to send the request: %s", e)
 
+    def _proxy_setup(self, client_password: str = CLIENT_PASSWORD):
+        """Setup system-wide proxy configuration using proxy pool
+        
+        Args:
+            client_password (str): Password for sudo operations, defaults to "password"
+        """
+        # Get proxy from global proxy pool
+        proxy_pool = get_global_proxy_pool()
+        current_proxy = proxy_pool.get_next_proxy()
+        
+        if not current_proxy:
+            logger.error("No proxy available from proxy pool")
+            raise Exception("No proxy available from proxy pool")
+        
+        # Format proxy URL
+        proxy_url = proxy_pool._format_proxy_url(current_proxy)
+        logger.info(f"Setting up proxy: {current_proxy.host}:{current_proxy.port}")
+        
+        # Configure system proxy environment variables
+        proxy_commands = [
+            f"echo '{client_password}' | sudo -S bash -c \"echo 'export http_proxy={proxy_url}' >> /etc/environment\"",
+            f"echo '{client_password}' | sudo -S bash -c \"echo 'export https_proxy={proxy_url}' >> /etc/environment\"",
+            f"echo '{client_password}' | sudo -S bash -c \"echo 'export HTTP_PROXY={proxy_url}' >> /etc/environment\"",
+            f"echo '{client_password}' | sudo -S bash -c \"echo 'export HTTPS_PROXY={proxy_url}' >> /etc/environment\"",
+        ]
+        
+        # Execute all proxy configuration commands
+        for cmd in proxy_commands:
+            try:
+                self._execute_setup([cmd], shell=True)
+            except Exception as e:
+                logger.error(f"Failed to execute proxy setup command: {e}")
+                proxy_pool.mark_proxy_failed(current_proxy)
+                raise
+        
+        # Reload environment variables
+        reload_cmd = "source /etc/environment"
+        try:
+            logger.info(f"Proxy setup completed successfully for {current_proxy.host}:{current_proxy.port}")
+            proxy_pool.mark_proxy_success(current_proxy)
+        except Exception as e:
+            logger.error(f"Failed to reload environment variables: {e}")
+            proxy_pool.mark_proxy_failed(current_proxy)
+            raise
+
     # Chrome setup
     def _chrome_open_tabs_setup(self, urls_to_open: List[str]):
         host = self.vm_ip

desktop_env/desktop_env.py

@@ -191,6 +191,10 @@ class DesktopEnv(gym.Env):
             logger.info("Setting up environment...")
             self.setup_controller.setup(self.config)
             logger.info("Environment setup complete.")
+        
+            if task_config.get("proxy", False):
+                # If using proxy, set up the proxy configuration
+                self.setup_controller._proxy_setup()
 
         observation = self._get_obs()
         return observation

desktop_env/providers/aws/manager.py

@@ -33,7 +33,7 @@ DEFAULT_REGION = "us-east-1"
 # todo: public the AMI images
 # ami-05e7d7bd279ea4f14
 IMAGE_ID_MAP = {
-    "us-east-1": "ami-02fea2e5b77c79c17",
+    "us-east-1": "ami-00d1fe56632acbefd",
     "ap-east-1": "ami-0c092a5b8be4116f5",
 }
 

evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {
@@ -39,7 +40,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json

@@ -29,12 +29,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=10 /home/user/Desktop/Colorful-Flowers.mp3",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "--start-time=10",
-          "/home/user/Desktop/Colorful-Flowers.mp3"
-        ]
       }
     },
     {
@@ -95,7 +91,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json

@@ -18,7 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {

evaluation_examples/examples/vlc/5ac2891a-eacd-4954-b339-98abba077adb.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     }
   ],

evaluation_examples/examples/vlc/7882ed6e-bece-4bf0-bada-c32dc1ddae72.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     }
   ],

evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {

evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json

@@ -18,12 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=15 '/home/user/Desktop/Gen 2.mp4'",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "--start-time=15",
-          "/home/user/Desktop/Gen 2.mp4"
-        ]
       }
     },
     {

evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json

@@ -18,12 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=73 '/home/user/Desktop/Baby Justin Bieber.mp4'",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "--start-time=73",
-          "/home/user/Desktop/Baby Justin Bieber.mp4"
-        ]
       }
     },
     {

evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {
@@ -39,7 +40,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {
@@ -39,7 +40,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json

@@ -18,11 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show /home/user/Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "/home/user/Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4"
-        ]
       }
     },
     {

evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {

evaluation_examples/examples/vlc/cb130f0d-d36f-4302-9838-b3baf46139b6.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     }
   ],

evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {
@@ -39,7 +40,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json

@@ -18,14 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=120.5 --stop-time=121 --play-and-pause '/home/user/Desktop/Interstellar Movie - Official Trailer.mp4'",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "--start-time=120.5",
-          "--stop-time=121",
-          "--play-and-pause",
-          "/home/user/Desktop/Interstellar Movie - Official Trailer.mp4"
-        ]
       }
     },
     {

evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json

@@ -7,7 +7,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": "VLC_VERBOSE=-1 vlc"
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
+        "shell": true
       }
     },
     {
@@ -39,7 +40,8 @@
       {
         "type": "launch",
         "parameters": {
-          "command": "vlc"
+          "command": "vlc --no-audio --no-video-title-show",
+          "shell": true
         }
       }
     ],

evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json

@@ -18,14 +18,8 @@
     {
       "type": "launch",
       "parameters": {
-        "command": [
+        "command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=120.5 --stop-time=121 --play-and-pause '/home/user/Desktop/Interstellar Movie - Official Trailer.mp4'",
-          "VLC_VERBOSE=-1",
+        "shell": true
-          "vlc",
-          "--start-time=120.5",
-          "--stop-time=121",
-          "--play-and-pause",
-          "/home/user/Desktop/Interstellar Movie - Official Trailer.mp4"
-        ]
       }
     },
     {

monitor/README.md

@@ -19,10 +19,13 @@ The monitor can be configured by editing the `.env` file in the monitor director
 
 | Variable | Description | Default Value |
 |----------|-------------|---------------|
-| TASK_CONFIG_PATH | Path to the task configuration JSON file | ../evaluation_examples/test_small.json |
+| TASK_CONFIG_PATH | Path to the task configuration file | ../evaluation_examples/test.json |
-| EXAMPLES_BASE_PATH | Base path for task example files | ../evaluation_examples/examples |
+| EXAMPLES_BASE_PATH | Base path for example files | ../evaluation_examples/examples |
-| RESULTS_BASE_PATH | Base path for execution results | ../results_operator_aws/pyautogui/screenshot/computer-use-preview |
+| RESULTS_BASE_PATH | Base path for storing results | ../results |
-| MAX_STEPS | Maximum steps to display for a task | 50 |
+| ACTION_SPACE | Action space type (e.g., pyautogui, keyboard) | pyautogui |
+| OBSERVATION_TYPE | Type of observation (e.g., screenshot, video) | screenshot |
+| MODEL_NAME | Name of the model to use for task execution | computer-use-preview |
+| MAX_STEPS | Maximum steps to display for a task | 150 |
 | FLASK_PORT | Port for the web server | 80 |
 | FLASK_HOST | Host address for the web server | 0.0.0.0 |
 | FLASK_DEBUG | Enable debug mode (true/false) | false |
@@ -30,13 +33,16 @@ The monitor can be configured by editing the `.env` file in the monitor director
 For example:
 ```bash
 # .env
-TASK_CONFIG_PATH=../evaluation_examples/test_small.json
+TASK_CONFIG_PATH=../evaluation_examples/test.json
 EXAMPLES_BASE_PATH=../evaluation_examples/examples
-RESULTS_BASE_PATH=../results_operator_aws/pyautogui/screenshot/computer-use-preview
+RESULTS_BASE_PATH=../results
-MAX_STEPS=50
+ACTION_SPACE=pyautogui
+OBSERVATION_TYPE=screenshot
+MODEL_NAME=computer-use-preview
+MAX_STEPS=150
 FLASK_PORT=80
 FLASK_HOST=0.0.0.0
-FLASK_DEBUG=false
+FLASK_DEBUG=true
 ```
 
 ## Running with Docker

monitor/main.py

@@ -12,8 +12,11 @@ from dotenv import load_dotenv
 # Load environment variables from .env file
 load_dotenv()
 
-# {task_type}_{task_id}: status_dict
+# {task_type}_{task_id}: (status_dict, timestamp)
+# For "Done" status, we need to verify it for a period to ensure it doesn't change to "Error"
 TASK_STATUS_CACHE = {}
+# Time in seconds to consider "Done" status as stable (default: 30s)
+DONE_STABILITY_PERIOD = int(os.getenv("DONE_STABILITY_PERIOD", "30"))
 
 app = Flask(__name__)
 
@@ -26,14 +29,14 @@ if MONITOR_IN_DOCKER:
     RESULTS_BASE_PATH = "/app/results"
 else:
     # Load configuration from environment variables
-    TASK_CONFIG_PATH = os.getenv("TASK_CONFIG_PATH", "../evaluation_examples/test_small.json")
+    TASK_CONFIG_PATH = os.getenv("TASK_CONFIG_PATH", "../evaluation_examples/test.json")
     EXAMPLES_BASE_PATH = os.getenv("EXAMPLES_BASE_PATH", "../evaluation_examples/examples")
     RESULTS_BASE_PATH = os.getenv("RESULTS_BASE_PATH", "../results")
 
 ACTION_SPACE=os.getenv("ACTION_SPACE", "pyautogui")
 OBSERVATION_TYPE=os.getenv("OBSERVATION_TYPE", "screenshot")
 MODEL_NAME=os.getenv("MODEL_NAME", "computer-use-preview")
-MAX_STEPS = int(os.getenv("MAX_STEPS", "50"))
+MAX_STEPS = int(os.getenv("MAX_STEPS", "150"))
 
 RESULTS_PATH = os.path.join(RESULTS_BASE_PATH, ACTION_SPACE, OBSERVATION_TYPE, MODEL_NAME)
 
@@ -177,9 +180,24 @@ def get_task_status_brief(task_type, task_id):
     # Generate cache key based on task type and ID
     cache_key = f"{task_type}_{task_id}"
     
-    #  Check if the status is already cached
+    # Check if the status is already cached
+    current_time = time.time()
+    last_cache_time = None
     if cache_key in TASK_STATUS_CACHE:
-        return TASK_STATUS_CACHE[cache_key]
+        cached_status, cached_time = TASK_STATUS_CACHE[cache_key]
+        last_cache_time = cached_time
+        # If cached status is "Done", check if it's within the stability period
+        if cached_status["status"].startswith("Done"):
+            # If within stability period, recalculate status to ensure it's correct
+            if current_time - cached_time < DONE_STABILITY_PERIOD:
+                # Status is still in verification period, refresh it
+                pass
+            else:
+                # Status is stable, return from cache
+                return cached_status
+        else:
+            # For non-Done status (like Error), just return from cache
+            return cached_status
     
     result_dir = os.path.join(RESULTS_PATH, task_type, task_id)
     
@@ -293,7 +311,8 @@ def get_task_status_brief(task_type, task_id):
     
     # Cache the status if it is done or error
     if status.startswith("Done") or status == "Error":
-        TASK_STATUS_CACHE[cache_key] = status_dict
+        current_time = last_cache_time if last_cache_time else current_time
+        TASK_STATUS_CACHE[cache_key] = (status_dict, current_time)
     
     return status_dict