lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/feat/aws-provider-support'

Browse Source
This commit is contained in:
yuanmengqi
2025-06-07 13:22:53 +00:00
parent c57b1d4e7a bbd4401ff5
commit f48d80002f
23 changed files with 147 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -195,4 +195,6 @@ vmware_vm_data
# result
**/result*/**/*
.vscode
.vscode
dataimpulse_proxy_config.json

54
desktop_env/controllers/setup.py
View File

@@ -21,11 +21,20 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder
from desktop_env.controllers.python import PythonController
from desktop_env.evaluators.metrics.utils import compare_urls
from desktop_env.providers.aws.proxy_pool import get_global_proxy_pool, init_proxy_pool, ProxyInfo
import dotenv
# Load environment variables from .env file
dotenv.load_dotenv()
CLIENT_PASSWORD = os.getenv("CLIENT_PASSWORD", "password") # Default password for sudo operations
PROXY_CONFIG_FILE = os.getenv("PROXY_CONFIG_FILE", "dataimpulse_proxy_config.json") # Default proxy config file
logger = logging.getLogger("desktopenv.setup")
FILE_PATH = os.path.dirname(os.path.abspath(__file__))
init_proxy_pool(PROXY_CONFIG_FILE) # initialize the global proxy pool
class SetupController:
def __init__(self, vm_ip: str, server_port: int = 5000, chromium_port: int = 9222, vlc_port: int = 8080, cache_dir: str = "cache"):
@@ -360,6 +369,51 @@ class SetupController:
except requests.exceptions.RequestException as e:
logger.error("An error occurred while trying to send the request: %s", e)
def _proxy_setup(self, client_password: str = CLIENT_PASSWORD):
"""Setup system-wide proxy configuration using proxy pool
Args:
client_password (str): Password for sudo operations, defaults to "password"
"""
# Get proxy from global proxy pool
proxy_pool = get_global_proxy_pool()
current_proxy = proxy_pool.get_next_proxy()
if not current_proxy:
logger.error("No proxy available from proxy pool")
raise Exception("No proxy available from proxy pool")
# Format proxy URL
proxy_url = proxy_pool._format_proxy_url(current_proxy)
logger.info(f"Setting up proxy: {current_proxy.host}:{current_proxy.port}")
# Configure system proxy environment variables
proxy_commands = [
f"echo '{client_password}' | sudo -S bash -c \"echo 'export http_proxy={proxy_url}' >> /etc/environment\"",
f"echo '{client_password}' | sudo -S bash -c \"echo 'export https_proxy={proxy_url}' >> /etc/environment\"",
f"echo '{client_password}' | sudo -S bash -c \"echo 'export HTTP_PROXY={proxy_url}' >> /etc/environment\"",
f"echo '{client_password}' | sudo -S bash -c \"echo 'export HTTPS_PROXY={proxy_url}' >> /etc/environment\"",
]
# Execute all proxy configuration commands
for cmd in proxy_commands:
try:
self._execute_setup([cmd], shell=True)
except Exception as e:
logger.error(f"Failed to execute proxy setup command: {e}")
proxy_pool.mark_proxy_failed(current_proxy)
raise
# Reload environment variables
reload_cmd = "source /etc/environment"
try:
logger.info(f"Proxy setup completed successfully for {current_proxy.host}:{current_proxy.port}")
proxy_pool.mark_proxy_success(current_proxy)
except Exception as e:
logger.error(f"Failed to reload environment variables: {e}")
proxy_pool.mark_proxy_failed(current_proxy)
raise
# Chrome setup
def _chrome_open_tabs_setup(self, urls_to_open: List[str]):
host = self.vm_ip

4
desktop_env/desktop_env.py
View File

@@ -191,6 +191,10 @@ class DesktopEnv(gym.Env):
logger.info("Setting up environment...")
self.setup_controller.setup(self.config)
logger.info("Environment setup complete.")
if task_config.get("proxy", False):
# If using proxy, set up the proxy configuration
self.setup_controller._proxy_setup()
observation = self._get_obs()
return observation

2
desktop_env/providers/aws/manager.py
View File

@@ -33,7 +33,7 @@ DEFAULT_REGION = "us-east-1"
# todo: public the AMI images
# ami-05e7d7bd279ea4f14
IMAGE_ID_MAP = {
"us-east-1": "ami-02fea2e5b77c79c17",
"us-east-1": "ami-00d1fe56632acbefd",
"ap-east-1": "ami-0c092a5b8be4116f5",
}

6
evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{
@@ -39,7 +40,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

11
evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json
View File

@@ -29,12 +29,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"--start-time=10",
"/home/user/Desktop/Colorful-Flowers.mp3"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=10 /home/user/Desktop/Colorful-Flowers.mp3",
"shell": true
}
},
{
@@ -95,7 +91,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

3
evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json
View File

@@ -18,7 +18,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{

3
evaluation_examples/examples/vlc/5ac2891a-eacd-4954-b339-98abba077adb.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

3
evaluation_examples/examples/vlc/7882ed6e-bece-4bf0-bada-c32dc1ddae72.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

3
evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{

8
evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json
View File

@@ -18,12 +18,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"--start-time=15",
"/home/user/Desktop/Gen 2.mp4"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=15 '/home/user/Desktop/Gen 2.mp4'",
"shell": true
}
},
{

8
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -18,12 +18,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"--start-time=73",
"/home/user/Desktop/Baby Justin Bieber.mp4"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=73 '/home/user/Desktop/Baby Justin Bieber.mp4'",
"shell": true
}
},
{

6
evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{
@@ -39,7 +40,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

6
evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{
@@ -39,7 +40,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

7
evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json
View File

@@ -18,11 +18,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"/home/user/Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show /home/user/Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4",
"shell": true
}
},
{

3
evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{

3
evaluation_examples/examples/vlc/cb130f0d-d36f-4302-9838-b3baf46139b6.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

6
evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{
@@ -39,7 +40,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

10
evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json
View File

@@ -18,14 +18,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"/home/user/Desktop/Interstellar Movie - Official Trailer.mp4"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=120.5 --stop-time=121 --play-and-pause '/home/user/Desktop/Interstellar Movie - Official Trailer.mp4'",
"shell": true
}
},
{

6
evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json
View File

@@ -7,7 +7,8 @@
{
"type": "launch",
"parameters": {
"command": "VLC_VERBOSE=-1 vlc"
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show",
"shell": true
}
},
{
@@ -39,7 +40,8 @@
{
"type": "launch",
"parameters": {
"command": "vlc"
"command": "vlc --no-audio --no-video-title-show",
"shell": true
}
}
],

10
evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json
View File

@@ -18,14 +18,8 @@
{
"type": "launch",
"parameters": {
"command": [
"VLC_VERBOSE=-1",
"vlc",
"--start-time=120.5",
"--stop-time=121",
"--play-and-pause",
"/home/user/Desktop/Interstellar Movie - Official Trailer.mp4"
]
"command": "VLC_VERBOSE=-1 vlc --no-audio --no-video-title-show --start-time=120.5 --stop-time=121 --play-and-pause '/home/user/Desktop/Interstellar Movie - Official Trailer.mp4'",
"shell": true
}
},
{

22
monitor/README.md
View File

@@ -19,10 +19,13 @@ The monitor can be configured by editing the `.env` file in the monitor director
| Variable | Description | Default Value |
|----------|-------------|---------------|
| TASK_CONFIG_PATH | Path to the task configuration JSON file | ../evaluation_examples/test_small.json |
| EXAMPLES_BASE_PATH | Base path for task example files | ../evaluation_examples/examples |
| RESULTS_BASE_PATH | Base path for execution results | ../results_operator_aws/pyautogui/screenshot/computer-use-preview |
| MAX_STEPS | Maximum steps to display for a task | 50 |
| TASK_CONFIG_PATH | Path to the task configuration file | ../evaluation_examples/test.json |
| EXAMPLES_BASE_PATH | Base path for example files | ../evaluation_examples/examples |
| RESULTS_BASE_PATH | Base path for storing results | ../results |
| ACTION_SPACE | Action space type (e.g., pyautogui, keyboard) | pyautogui |
| OBSERVATION_TYPE | Type of observation (e.g., screenshot, video) | screenshot |
| MODEL_NAME | Name of the model to use for task execution | computer-use-preview |
| MAX_STEPS | Maximum steps to display for a task | 150 |
| FLASK_PORT | Port for the web server | 80 |
| FLASK_HOST | Host address for the web server | 0.0.0.0 |
| FLASK_DEBUG | Enable debug mode (true/false) | false |
@@ -30,13 +33,16 @@ The monitor can be configured by editing the `.env` file in the monitor director
For example:
```bash
# .env
TASK_CONFIG_PATH=../evaluation_examples/test_small.json
TASK_CONFIG_PATH=../evaluation_examples/test.json
EXAMPLES_BASE_PATH=../evaluation_examples/examples
RESULTS_BASE_PATH=../results_operator_aws/pyautogui/screenshot/computer-use-preview
MAX_STEPS=50
RESULTS_BASE_PATH=../results
ACTION_SPACE=pyautogui
OBSERVATION_TYPE=screenshot
MODEL_NAME=computer-use-preview
MAX_STEPS=150
FLASK_PORT=80
FLASK_HOST=0.0.0.0
FLASK_DEBUG=false
FLASK_DEBUG=true
```
## Running with Docker

31
monitor/main.py
View File

@@ -12,8 +12,11 @@ from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
# {task_type}_{task_id}: status_dict
# {task_type}_{task_id}: (status_dict, timestamp)
# For "Done" status, we need to verify it for a period to ensure it doesn't change to "Error"
TASK_STATUS_CACHE = {}
# Time in seconds to consider "Done" status as stable (default: 30s)
DONE_STABILITY_PERIOD = int(os.getenv("DONE_STABILITY_PERIOD", "30"))
app = Flask(__name__)
@@ -26,14 +29,14 @@ if MONITOR_IN_DOCKER:
RESULTS_BASE_PATH = "/app/results"
else:
# Load configuration from environment variables
TASK_CONFIG_PATH = os.getenv("TASK_CONFIG_PATH", "../evaluation_examples/test_small.json")
TASK_CONFIG_PATH = os.getenv("TASK_CONFIG_PATH", "../evaluation_examples/test.json")
EXAMPLES_BASE_PATH = os.getenv("EXAMPLES_BASE_PATH", "../evaluation_examples/examples")
RESULTS_BASE_PATH = os.getenv("RESULTS_BASE_PATH", "../results")
ACTION_SPACE=os.getenv("ACTION_SPACE", "pyautogui")
OBSERVATION_TYPE=os.getenv("OBSERVATION_TYPE", "screenshot")
MODEL_NAME=os.getenv("MODEL_NAME", "computer-use-preview")
MAX_STEPS = int(os.getenv("MAX_STEPS", "50"))
MAX_STEPS = int(os.getenv("MAX_STEPS", "150"))
RESULTS_PATH = os.path.join(RESULTS_BASE_PATH, ACTION_SPACE, OBSERVATION_TYPE, MODEL_NAME)
@@ -177,9 +180,24 @@ def get_task_status_brief(task_type, task_id):
# Generate cache key based on task type and ID
cache_key = f"{task_type}_{task_id}"
# Check if the status is already cached
# Check if the status is already cached
current_time = time.time()
last_cache_time = None
if cache_key in TASK_STATUS_CACHE:
return TASK_STATUS_CACHE[cache_key]
cached_status, cached_time = TASK_STATUS_CACHE[cache_key]
last_cache_time = cached_time
# If cached status is "Done", check if it's within the stability period
if cached_status["status"].startswith("Done"):
# If within stability period, recalculate status to ensure it's correct
if current_time - cached_time < DONE_STABILITY_PERIOD:
# Status is still in verification period, refresh it
pass
else:
# Status is stable, return from cache
return cached_status
else:
# For non-Done status (like Error), just return from cache
return cached_status
result_dir = os.path.join(RESULTS_PATH, task_type, task_id)
@@ -293,7 +311,8 @@ def get_task_status_brief(task_type, task_id):
# Cache the status if it is done or error
if status.startswith("Done") or status == "Error":
TASK_STATUS_CACHE[cache_key] = status_dict
current_time = last_cache_time if last_cache_time else current_time
TASK_STATUS_CACHE[cache_key] = (status_dict, current_time)
return status_dict