lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Feature] Initialize and Implement Aguvis Evaluation on OSWorld (#98)

Browse Source 
* Initialize Aguvis eval on OSWorld

* Debug

* Debug

* v1, internal version

* Add experiments script

* Fix minor bugs

* Update new endpoint

* Update ip

* Update

* Update

* Update

* Update

* Update

* Update

* Update

* Update

* Fix model name

* Fix docker close issues; update prompting

* Fix missed

* Fix the default port to avoid crashing on examples like '_update_browse_history_setup'

* Fix server and chromium ports in setup

* Revert and add missed dependency

* Add VLC port for docker

* Update

* Clean

---------

Co-authored-by: Tianbao Xie <tianbaoxie@U-492FC39R-0217.local>
Co-authored-by: FredWuCZ <fredwucz@outlook.com>
This commit is contained in:
Tianbao Xie
2024-11-11 12:36:16 +08:00
committed by GitHub
parent b35dc40ff4
commit 20442244fa
7 changed files with 910 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
desktop_env/controllers/setup.py
View File

@@ -28,10 +28,11 @@ FILE_PATH = os.path.dirname(os.path.abspath(__file__))
class SetupController:
def __init__(self, vm_ip: str, server_port: int, chromium_port: int, cache_dir: str):
def __init__(self, vm_ip: str, server_port: int = 5000, chromium_port: int = 9222, vlc_port: int = 8080, cache_dir: str = "cache"):
self.vm_ip: str = vm_ip
self.server_port: int = server_port
self.chromium_port: int = chromium_port
self.vlc_port: int = vlc_port
self.http_server: str = f"http://{vm_ip}:{server_port}"
self.http_server_setup_root: str = f"http://{vm_ip}:{server_port}/setup"
self.cache_dir: str = cache_dir
@@ -532,7 +533,7 @@ class SetupController:
"""
host = self.vm_ip
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
port = self.chromium_port
remote_debugging_url = f"http://{host}:{port}"
with sync_playwright() as p:
@@ -643,7 +644,7 @@ class SetupController:
logger.info('Fake browsing history added successfully.')
controller = PythonController(self.vm_ip)
controller = PythonController(self.vm_ip, self.server_port)
# get the path of the history file according to the platform
os_type = controller.get_vm_platform()

4
desktop_env/desktop_env.py
View File

@@ -59,6 +59,7 @@ class DesktopEnv(gym.Env):
self.server_port = 5000
self.chromium_port = 9222
self.vnc_port = 8006
self.vlc_port = 8080
self.manager, self.provider = create_vm_manager_and_provider(provider_name, region)
self.os_type = os_type
@@ -104,8 +105,9 @@ class DesktopEnv(gym.Env):
self.server_port = int(vm_ip_ports[1])
self.chromium_port = int(vm_ip_ports[2])
self.vnc_port = int(vm_ip_ports[3])
self.vlc_port = int(vm_ip_ports[4])
self.controller = PythonController(vm_ip=self.vm_ip, server_port=self.server_port)
self.setup_controller = SetupController(vm_ip=self.vm_ip, server_port=self.server_port, chromium_port=self.chromium_port, cache_dir=self.cache_dir_base)
self.setup_controller = SetupController(vm_ip=self.vm_ip, server_port=self.server_port, chromium_port=self.chromium_port, vlc_port=self.vlc_port, cache_dir=self.cache_dir_base)
def _revert_to_snapshot(self):
# Revert to certain snapshot of the virtual machine, and refresh the path to vm and ip of vm

2
desktop_env/evaluators/getters/vlc.py
View File

@@ -14,7 +14,7 @@ def get_vlc_playing_info(env, config: Dict[str, str]):
"""
host = env.vm_ip
port = 8080
port = env.vlc_port
password = 'password'
_path = os.path.join(env.cache_dir, config["dest"])

15
desktop_env/providers/docker/provider.py
View File

@@ -28,6 +28,8 @@ class DockerProvider(Provider):
self.server_port = None
self.vnc_port = None
self.chromium_port = None
self.vlc_port = None
self.container = None
self.environment = {"DISK_SIZE": "32G", "RAM_SIZE": "4G", "CPU_CORES": "4"} # Modify if needed
temp_dir = Path(os.getenv('TEMP') if platform.system() == 'Windows' else '/tmp')
@@ -92,6 +94,7 @@ class DockerProvider(Provider):
self.vnc_port = self._get_available_port(8006)
self.server_port = self._get_available_port(5000)
self.chromium_port = self._get_available_port(9222)
self.vlc_port = self._get_available_port(8080)
# Start container while still holding the lock
self.container = self.client.containers.run(
@@ -108,13 +111,14 @@ class DockerProvider(Provider):
ports={
8006: self.vnc_port,
5000: self.server_port,
9222: self.chromium_port
9222: self.chromium_port,
8080: self.vlc_port
},
detach=True
)
logger.info(f"Started container with ports - VNC: {self.vnc_port}, "
f"Server: {self.server_port}, Chrome: {self.chromium_port}")
f"Server: {self.server_port}, Chrome: {self.chromium_port}, VLC: {self.vlc_port}")
# Wait for VM to be ready
self._wait_for_vm_ready()
@@ -130,15 +134,15 @@ class DockerProvider(Provider):
raise e
def get_ip_address(self, path_to_vm: str) -> str:
if not all([self.server_port, self.chromium_port, self.vnc_port]):
if not all([self.server_port, self.chromium_port, self.vnc_port, self.vlc_port]):
raise RuntimeError("VM not started - ports not allocated")
return f"localhost:{self.server_port}:{self.chromium_port}:{self.vnc_port}"
return f"localhost:{self.server_port}:{self.chromium_port}:{self.vnc_port}:{self.vlc_port}"
def save_state(self, path_to_vm: str, snapshot_name: str):
raise NotImplementedError("Snapshots not available for Docker provider")
def revert_to_snapshot(self, path_to_vm: str, snapshot_name: str):
pass
self.stop_emulator(path_to_vm)
def stop_emulator(self, path_to_vm: str):
if self.container:
@@ -154,3 +158,4 @@ class DockerProvider(Provider):
self.server_port = None
self.vnc_port = None
self.chromium_port = None
self.vlc_port = None