lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/xlang-ai/DesktopEnv

Browse Source
This commit is contained in:
tsuky_chen
2024-02-02 14:50:16 +08:00
parent 9e8c1f8ce6 e07a3d52ce
commit c86868f034
14 changed files with 490 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -24,6 +24,8 @@ todo
- [x] Add accessibility tree from the OS into the observation space
- [ ] Add pre-process and post-process action support for benchmarking setup and evaluation
- [ ] Multiprocess support, this can enable the reinforcement learning to be more efficient
- [ ] Experiment logging and visualization system
- [ ] Add more tasks, maybe scale to 300 for v1.0.0, and create a dynamic leaderboard
## Road map of benchmark, tools and resources (Proposed)
- [ ] Improve the annotation tool base on DuckTrack, make it more robust which align on accessibility tree

2
desktop_env/controllers/python.py
View File

@@ -11,7 +11,7 @@ logger = logging.getLogger("desktopenv.pycontroller")
class PythonController:
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; {command}"):
def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; pyautogui.FAILSAFE = False; {command}"):
self.vm_ip = vm_ip
self.http_server = f"http://{vm_ip}:5000"
self.pkgs_prefix = pkgs_prefix # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages

54
desktop_env/controllers/setup.py
View File

@@ -7,6 +7,7 @@ import uuid
import tempfile
from typing import Any, Union, Optional
from typing import Dict, List
import os
import requests
from pydrive.auth import GoogleAuth
@@ -114,6 +115,7 @@ class SetupController:
if not os.path.exists(cache_path):
max_retries = 3
downloaded = False
e = None
for i in range(max_retries):
try:
response = requests.get(url, stream=True)
@@ -128,7 +130,7 @@ class SetupController:
break
except requests.RequestException as e:
logger.error(f"Failed to download {url}. Retrying... ({max_retries - i - 1} attempts left)")
logger.error(f"Failed to download {url} caused by {e}. Retrying... ({max_retries - i - 1} attempts left)")
if not downloaded:
raise requests.RequestException(f"Failed to download {url}. No retries left. Error: {e}")
@@ -344,39 +346,49 @@ class SetupController:
port = 9222 # fixme: this port is hard-coded, need to be changed from config file
remote_debugging_url = f"http://{host}:{port}"
with sync_playwright() as p:
logger.info("Connect to Chrome @: %s", remote_debugging_url)
logger.debug("PLAYWRIGHT ENV: %s", repr(os.environ))
for attempt in range(15):
if attempt>0:
time.sleep(5)
browser = None
for attempt in range(15):
with sync_playwright() as p:
try:
browser = p.chromium.connect_over_cdp(remote_debugging_url)
break
#break
except Exception as e:
if attempt < 14:
logger.error(f"Attempt {attempt + 1}: Failed to connect, retrying. Error: {e}")
time.sleep(1)
#time.sleep(10)
continue
else:
logger.error(f"Failed to connect after multiple attempts: {e}")
raise e
if not browser:
return
if not browser:
return
for i, url in enumerate(urls_to_open):
# Use the first context (which should be the only one if using default profile)
if i == 0:
context = browser.contexts[0]
logger.info("Opening %s...", urls_to_open)
for i, url in enumerate(urls_to_open):
# Use the first context (which should be the only one if using default profile)
if i == 0:
context = browser.contexts[0]
page = context.new_page() # Create a new page (tab) within the existing context
page.goto(url, timeout=60000)
logger.info(f"Opened tab {i + 1}: {url}")
page = context.new_page() # Create a new page (tab) within the existing context
try:
page.goto(url, timeout=60000)
except:
logger.warning("Opening %s exceeds time limit", url) # only for human test
logger.info(f"Opened tab {i + 1}: {url}")
if i == 0:
# clear the default tab
default_page = context.pages[0]
default_page.close()
if i == 0:
# clear the default tab
default_page = context.pages[0]
default_page.close()
# Do not close the context or browser; they will remain open after script ends
return browser, context
# Do not close the context or browser; they will remain open after script ends
return browser, context
def _chrome_close_tabs_setup(self, urls_to_close: List[str]):
time.sleep(5) # Wait for Chrome to finish launching
@@ -552,4 +564,4 @@ class SetupController:
else:
raise NotImplementedError
return browser, context
return browser, context

4
desktop_env/evaluators/README.md
View File

@@ -191,7 +191,7 @@ To enable and use the HTTP interface in VLC Media Player for remote control and
#### 4. Configure Lua HTTP
- Expand the `Main interfaces` node and select `Lua`.
- Under `Lua HTTP`, set a password in the `Lua HTTP` section. This password will be required to access the HTTP interface.
- Under `Lua HTTP`, set a password `password` in the `Lua HTTP` section. This password will be required to access the HTTP interface.
#### 5. Save and Restart VLC
@@ -217,4 +217,4 @@ pip install opencv-python-headless Pillow imagehash
- If the port is in use by another application, you may change the port number in VLC's settings.
## GIMP
Click on the "Keep" of the image loading pop-up.
Click on the "Keep" of the image loading pop-up.

8
desktop_env/server/main.py
View File

@@ -2,7 +2,7 @@ import ctypes
import os
import platform
import shlex
import subprocess
import subprocess, signal
from pathlib import Path
from typing import Any, Optional
from typing import List, Dict, Tuple
@@ -997,7 +997,7 @@ def start_recording():
start_command = f"ffmpeg -y -f x11grab -draw_mouse 1 -s {screen_width}x{screen_height} -i :0.0 -c:v libx264 -r 30 {recording_path}"
recording_process = subprocess.Popen(shlex.split(start_command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
recording_process = subprocess.Popen(shlex.split(start_command), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return jsonify({'status': 'success', 'message': 'Started recording.'})
@@ -1009,10 +1009,8 @@ def end_recording():
if not recording_process:
return jsonify({'status': 'error', 'message': 'No recording in progress to stop.'}), 400
recording_process.terminate()
recording_process.send_signal(signal.SIGINT)
recording_process.wait()
# return_code = recording_process.returncode
output, error = recording_process.communicate()
recording_process = None
# return recording video file

10
evaluation_examples/examples/chrome/bb5e4c0d-f964-439c-97b6-bdb9747de3f4.json
View File

@@ -12,6 +12,16 @@
"--remote-debugging-port=9222"
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
]
}
}
],
"trajectory": "trajectories/",

4
evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -1,7 +1,7 @@
{
"id": "66399b0d-8fda-4618-95c4-bfc6191617e9",
"snapshot": "libreoffice_writer",
"instruction": "Could you help me insert a 7*5 empty table at the point of cursor?",
"instruction": "Could you help me insert a 7(columns)*5(rows) empty table at the point of cursor?",
"source": "https://www.youtube.com/watch?v=l25Evu4ohKg",
"config": [
{
@@ -27,7 +27,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(5); pyautogui.press(\"down\", presses=40, interval=10); time.sleep(1); pyautogui.scroll(-2)"
"import pyautogui; import time; pyautogui.press(\"down\", presses=40, interval=0.01); time.sleep(1); pyautogui.scroll(-2)"
]
}
}

4
evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
View File

@@ -38,7 +38,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(5); pyautogui.press(\"down\", presses=8, interval=3); time.sleep(1); pyautogui.scroll(-2)"
"import pyautogui; import time; time.sleep(5); pyautogui.press(\"down\", presses=8, interval=0.01); time.sleep(1); pyautogui.scroll(-2)"
]
}
}
@@ -81,7 +81,7 @@
},
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=427765e0-3f97-4a72-92db-a1fe7cdde73b&at=APZUnTUhNLh2PDu4OGkCVQW-LPCd:1704173991269",
"path": "https://drive.usercontent.google.com/download?id=1xbhlfqGrPutHHi2aHg66jwXD-yaZpe9j&export=download&authuser=0&confirm=t&uuid=802d477e-d97b-4641-84fb-9eaf8805c35c&at=APZUnTWS0KOqHCPnufPJfDEfGE2u:1706822844322",
"dest": "Viewing_Your_Class_Schedule_and_Textbooks_Gold.docx"
}
}

2
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{

131
experiment_a11y_tree.py
View File

@@ -242,18 +242,18 @@ if __name__ == '__main__':
# main("libreoffice_calc", example_id)
impress_list = [
# "5d901039-a89c-4bfb-967b-bf66f4df075e",
# "550ce7e7-747b-495f-b122-acdc4d0b8e54",
# "455d3c66-7dc6-4537-a39a-36d3e9119df7",
# "af23762e-2bfd-4a1d-aada-20fa8de9ce07",
# "c59742c0-4323-4b9d-8a02-723c251deaa0",
# "ef9d12bd-bcee-4ba0-a40e-918400f43ddf",
# "9ec204e4-f0a3-42f8-8458-b772a6797cab",
# "0f84bef9-9790-432e-92b7-eece357603fb",
# "ce88f674-ab7a-43da-9201-468d38539e4a",
# "3b27600c-3668-4abd-8f84-7bcdebbccbdb",
# "a097acff-6266-4291-9fbd-137af7ecd439",
# "bf4e9888-f10f-47af-8dba-76413038b73c",
"5d901039-a89c-4bfb-967b-bf66f4df075e",
"550ce7e7-747b-495f-b122-acdc4d0b8e54",
"455d3c66-7dc6-4537-a39a-36d3e9119df7",
"af23762e-2bfd-4a1d-aada-20fa8de9ce07",
"c59742c0-4323-4b9d-8a02-723c251deaa0",
"ef9d12bd-bcee-4ba0-a40e-918400f43ddf",
"9ec204e4-f0a3-42f8-8458-b772a6797cab",
"0f84bef9-9790-432e-92b7-eece357603fb",
"ce88f674-ab7a-43da-9201-468d38539e4a",
"3b27600c-3668-4abd-8f84-7bcdebbccbdb",
"a097acff-6266-4291-9fbd-137af7ecd439",
"bf4e9888-f10f-47af-8dba-76413038b73c",
"21760ecb-8f62-40d2-8d85-0cee5725cb72"
]
# for example_id in impress_list:
@@ -326,27 +326,106 @@ if __name__ == '__main__':
# logger.error("An error occurred while running the example: %s", e)
# continue
from tqdm import tqdm
# for example_id in tqdm(vlc_list):
# try:
# main("vlc", example_id, gpt4_model="gpt-3.5-turbo-16k")
# except Exception as e:
# print(f"An error occurred while running the example: {e}")
# continue
chrome_list = [
"bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"06fe7178-4491-4589-810f-2e2bc9502122",
"e1e75309-3ddb-4d09-92ec-de869c928143",
"35253b65-1c19-4304-8aa4-6884b8218fc0",
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
"44ee5668-ecd5-4366-a6ce-c1c9b8d4e938",
"2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3",
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
"af630914-714e-4a24-a7bb-f9af687d3b91"
]
# for example_id in tqdm(chrome_list):
# try:
# main("chrome", example_id, gpt4_model="gpt-3.5-turbo-16k")
# except Exception as e:
# print(f"An error occurred while running the example: {e}")
# continue
vs_code_list = [
# "0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
# "53ad5833-3455-407b-bbc6-45b4c79ab8fb",
# "eabc805a-bfcf-4460-b250-ac92135819f6",
# "982d12a5-beab-424f-8d38-d2a48429e511",
# "4e60007a-f5be-4bfc-9723-c39affa0a6d3",
# "e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
# "9439a27b-18ae-42d8-9778-5f68f891805e",
# "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
# "930fdb3b-11a8-46fe-9bac-577332e2640e",
# "276cc624-87ea-4f08-ab93-f770e3790175",
# "9d425400-e9b2-4424-9a4b-d4c7abac4140"
]
for example_id in tqdm(vs_code_list):
try:
main("vs_code", example_id, gpt4_model="gpt-3.5-turbo-16k")
except Exception as e:
print(f"An error occurred while running the example: {e}")
continue
thunderbird_list = [
# "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
# "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"12086550-11c0-466b-b367-1d9e75b3910e",
"06fe7178-4491-4589-810f-2e2bc9502122",
"6766f2b8-8a72-417f-a9e5-56fcaa735837",
"e1e75309-3ddb-4d09-92ec-de869c928143",
"3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"35253b65-1c19-4304-8aa4-6884b8218fc0",
"d088f539-cab4-4f9a-ac92-9999fc3a656e",
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
"030eeff7-b492-4218-b312-701ec99ee0cc",
"94760984-3ff5-41ee-8347-cf1af709fea0",
"99146c54-4f37-4ab8-9327-5f3291665e1e",
"c9e7eaf2-b1a1-4efc-a982-721972fa9f02"
]
# for example_id in tqdm(thunderbird_list):
# try:
# main("thunderbird", example_id, gpt4_model="gpt-3.5-turbo-16k")
# except Exception as e:
# print(f"An error occurred while running the example: {e}")
# continue
multiple_list = [
"f8cfa149-d1c1-4215-8dac-4a0932bad3c2",
"897e3b53-5d4d-444b-85cb-2cdc8a97d903",
"4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc",
"b52b40a5-ad70-4c53-b5b0-5650a8387052",
"46407397-a7d5-4c6b-92c6-dbe038b1457b",
"2b9493d7-49b8-493a-a71b-56cd1f4d6908",
"51f5801c-18b3-4f25-b0c3-02f85507a078",
"2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
"510f64c8-9bcc-4be1-8d30-638705850618",
"937087b6-f668-4ba6-9110-60682ee33441",
"ee9a3c83-f437-4879-8918-be5efbb9fac7",
# "f8cfa149-d1c1-4215-8dac-4a0932bad3c2",
# "897e3b53-5d4d-444b-85cb-2cdc8a97d903",
"2fe4b718-3bd7-46ec-bdce-b184f5653624",
"3680a5ee-6870-426a-a997-eba929a0d25c",
"e135df7c-7687-4ac0-a5f0-76b74438b53e",
# "4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc",
# "b52b40a5-ad70-4c53-b5b0-5650a8387052",
# "46407397-a7d5-4c6b-92c6-dbe038b1457b",
# "2b9493d7-49b8-493a-a71b-56cd1f4d6908",
# "51f5801c-18b3-4f25-b0c3-02f85507a078",
"58565672-7bfe-48ab-b828-db349231de6b",
"2fe4b718-3bd7-46ec-bdce-b184f5653624"
# "2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
# "510f64c8-9bcc-4be1-8d30-638705850618",
# "937087b6-f668-4ba6-9110-60682ee33441",
# "ee9a3c83-f437-4879-8918-be5efbb9fac7",
# "3680a5ee-6870-426a-a997-eba929a0d25c",
# "e135df7c-7687-4ac0-a5f0-76b74438b53e",
"ee9a3c83-f437-4879-8918-be5efbb9fac7",
# "58565672-7bfe-48ab-b828-db349231de6b",
# "2fe4b718-3bd7-46ec-bdce-b184f5653624"
]
for example_id in multiple_list:
try:
main("multi_apps", example_id)
main("multi_apps", example_id, gpt4_model="gpt-3.5-turbo-16k")
except Exception as e:
logger.error("An error occurred while running the example: %s", e)
continue

139
experiment_screenshot.py
View File

@@ -123,9 +123,8 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr
logger.info("Environment closed.")
def main(example_class, example_id):
def main(example_class, example_id, gpt4_model = "gpt-4-vision-preview"):
action_space = "pyautogui"
gpt4_model = "gpt-4-vision-preview"
gemini_model = "gemini-pro-vision"
logger.info("Running example %s/%s", example_class, example_id)
@@ -137,7 +136,7 @@ def main(example_class, example_id):
example["snapshot"] = "exp_v5"
api_key = os.environ.get("OPENAI_API_KEY")
agent = GPT4v_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space,
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'], action_space=action_space,
exp="screenshot")
#
# api_key = os.environ.get("GENAI_API_KEY")
@@ -168,35 +167,49 @@ if __name__ == '__main__':
"af630914-714e-4a24-a7bb-f9af687d3b91"
]
calc_list = [
# "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
# "0bf05a7d-b28b-44d2-955a-50b41e24012a",
# "7a4e4bc8-922c-4c84-865c-25ba34136be1",
# "2bd59342-0664-4ccb-ba87-79379096cc08",
# "ecb0df7a-4e8d-4a03-b162-053391d3afaf",
# "7efeb4b1-3d19-4762-b163-63328d66303b",
# "4e6fcf72-daf3-439f-a232-c434ce416af6",
# "6054afcb-5bab-4702-90a0-b259b5d3217c",
# "abed40dc-063f-4598-8ba5-9fe749c0615d",
# "01b269ae-2111-4a07-81fd-3fcd711993b0",
# "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"51b11269-2ca8-4b2a-9163-f21758420e78",
"7e429b8d-a3f0-4ed0-9b58-08957d00b127",
"347ef137-7eeb-4c80-a3bb-0951f26a8aff",
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"3aaa4e37-dc91-482e-99af-132a612d40f3",
"37608790-6147-45d0-9f20-1137bb35703d",
"f9584479-3d0d-4c79-affa-9ad7afdd8850",
"d681960f-7bc3-4286-9913-a8812ba3261a",
"21df9241-f8d7-4509-b7f1-37e501a823f7",
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
"a01fbce3-2793-461f-ab86-43680ccbae25",
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c",
]
"a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"7efeb4b1-3d19-4762-b163-63328d66303b",
"4e6fcf72-daf3-439f-a232-c434ce416af6",
"6054afcb-5bab-4702-90a0-b259b5d3217c",
"abed40dc-063f-4598-8ba5-9fe749c0615d",
"01b269ae-2111-4a07-81fd-3fcd711993b0",
"8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"af2b02f7-acee-4be4-8b66-499fab394915",
"da1d63b8-fa12-417b-ba18-f748e5f770f3",
"636380ea-d5f6-4474-b6ca-b2ed578a20f1",
"5ba77536-05c5-4aae-a9ff-6e298d094c3e",
"4bc4eaf4-ca5e-4db2-8138-8d4e65af7c0b",
"672a1b02-c62f-4ae2-acf0-37f5fb3052b0",
"648fe544-16ba-44af-a587-12ccbe280ea6",
"8985d1e4-5b99-4711-add4-88949ebb2308",
"9e606842-2e27-43bf-b1d1-b43289c9589b",
"fcb6e45b-25c4-4087-9483-03d714f473a9",
"68c0c5b7-96f3-4e87-92a7-6c1b967fd2d2",
"fff629ea-046e-4793-8eec-1a5a15c3eb35",
"5c9a206c-bb00-4fb6-bb46-ee675c187df5",
"e975ae74-79bd-4672-8d1c-dc841a85781d",
"34a6938a-58da-4897-8639-9b90d6db5391",
"b5a22759-b4eb-4bf2-aeed-ad14e8615f19",
"2f9913a1-51ed-4db6-bfe0-7e1c95b3139e",
"2558031e-401d-4579-8e00-3ecf540fb492",
"0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"51b11269-2ca8-4b2a-9163-f21758420e78",
"7e429b8d-a3f0-4ed0-9b58-08957d00b127",
"347ef137-7eeb-4c80-a3bb-0951f26a8aff",
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"3aaa4e37-dc91-482e-99af-132a612d40f3",
"37608790-6147-45d0-9f20-1137bb35703d",
"f9584479-3d0d-4c79-affa-9ad7afdd8850",
"d681960f-7bc3-4286-9913-a8812ba3261a",
"21df9241-f8d7-4509-b7f1-37e501a823f7",
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
"a01fbce3-2793-461f-ab86-43680ccbae25",
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c"
]
# for example_id in calc_list:
# main("libreoffice_calc", example_id)
@@ -246,13 +259,13 @@ if __name__ == '__main__':
#
vs_code_list = [
"0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
"53ad5833-3455-407b-bbc6-45b4c79ab8fb",
"eabc805a-bfcf-4460-b250-ac92135819f6",
"982d12a5-beab-424f-8d38-d2a48429e511",
"4e60007a-f5be-4bfc-9723-c39affa0a6d3",
"e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
"9439a27b-18ae-42d8-9778-5f68f891805e",
# "0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
# "53ad5833-3455-407b-bbc6-45b4c79ab8fb",
# "eabc805a-bfcf-4460-b250-ac92135819f6",
# "982d12a5-beab-424f-8d38-d2a48429e511",
# "4e60007a-f5be-4bfc-9723-c39affa0a6d3",
# "e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
# "9439a27b-18ae-42d8-9778-5f68f891805e",
"ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
"930fdb3b-11a8-46fe-9bac-577332e2640e",
"276cc624-87ea-4f08-ab93-f770e3790175",
@@ -266,28 +279,28 @@ if __name__ == '__main__':
# logger.error("An error occurred while running the example: %s", e)
# continue
multiple_list = [
"f8cfa149-d1c1-4215-8dac-4a0932bad3c2",
"897e3b53-5d4d-444b-85cb-2cdc8a97d903",
"4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc",
"b52b40a5-ad70-4c53-b5b0-5650a8387052",
"46407397-a7d5-4c6b-92c6-dbe038b1457b",
"2b9493d7-49b8-493a-a71b-56cd1f4d6908",
"51f5801c-18b3-4f25-b0c3-02f85507a078",
"2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
"510f64c8-9bcc-4be1-8d30-638705850618",
"937087b6-f668-4ba6-9110-60682ee33441",
"ee9a3c83-f437-4879-8918-be5efbb9fac7",
"3680a5ee-6870-426a-a997-eba929a0d25c",
"e135df7c-7687-4ac0-a5f0-76b74438b53e",
"58565672-7bfe-48ab-b828-db349231de6b",
"2fe4b718-3bd7-46ec-bdce-b184f5653624"
]
for example_id in multiple_list:
try:
main("multi_apps", example_id)
except Exception as e:
logger.error("An error occurred while running the example: %s", e)
continue
# multiple_list = [
# "f8cfa149-d1c1-4215-8dac-4a0932bad3c2",
# "897e3b53-5d4d-444b-85cb-2cdc8a97d903",
# "4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc",
# "b52b40a5-ad70-4c53-b5b0-5650a8387052",
# "46407397-a7d5-4c6b-92c6-dbe038b1457b",
# "2b9493d7-49b8-493a-a71b-56cd1f4d6908",
# "51f5801c-18b3-4f25-b0c3-02f85507a078",
# "2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
# "510f64c8-9bcc-4be1-8d30-638705850618",
# "937087b6-f668-4ba6-9110-60682ee33441",
# "ee9a3c83-f437-4879-8918-be5efbb9fac7",
# "3680a5ee-6870-426a-a997-eba929a0d25c",
# "e135df7c-7687-4ac0-a5f0-76b74438b53e",
# "58565672-7bfe-48ab-b828-db349231de6b",
# "2fe4b718-3bd7-46ec-bdce-b184f5653624"
# ]
#
# for example_id in multiple_list:
# try:
# main("multi_apps", example_id)
# except Exception as e:
# logger.error("An error occurred while running the example: %s", e)
# continue

132
experiment_screenshot_a11y_tree.py
View File

@@ -43,7 +43,7 @@ logger.addHandler(sdebug_handler)
logger = logging.getLogger("desktopenv.experiment")
PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu2\Ubuntu2.vmx"
# PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
@@ -185,34 +185,49 @@ if __name__ == '__main__':
# continue
calc_list = [
# "eb03d19a-b88d-4de4-8a64-ca0ac66f426b",
# "0bf05a7d-b28b-44d2-955a-50b41e24012a",
# "7a4e4bc8-922c-4c84-865c-25ba34136be1",
# "2bd59342-0664-4ccb-ba87-79379096cc08",
# "ecb0df7a-4e8d-4a03-b162-053391d3afaf",
# "7efeb4b1-3d19-4762-b163-63328d66303b",
# "4e6fcf72-daf3-439f-a232-c434ce416af6",
# "6054afcb-5bab-4702-90a0-b259b5d3217c",
# "abed40dc-063f-4598-8ba5-9fe749c0615d",
# "01b269ae-2111-4a07-81fd-3fcd711993b0",
# "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
# "0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
# "4188d3a4-077d-46b7-9c86-23e1a036f6c1",
# "51b11269-2ca8-4b2a-9163-f21758420e78",
# "7e429b8d-a3f0-4ed0-9b58-08957d00b127",
# "347ef137-7eeb-4c80-a3bb-0951f26a8aff",
# "6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
# "3aaa4e37-dc91-482e-99af-132a612d40f3",
# "37608790-6147-45d0-9f20-1137bb35703d",
# "f9584479-3d0d-4c79-affa-9ad7afdd8850",
"d681960f-7bc3-4286-9913-a8812ba3261a",
"21df9241-f8d7-4509-b7f1-37e501a823f7",
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
"a01fbce3-2793-461f-ab86-43680ccbae25",
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c",
]
"a9f325aa-8c05-4e4f-8341-9e4358565f4f",
"ecb0df7a-4e8d-4a03-b162-053391d3afaf",
"7efeb4b1-3d19-4762-b163-63328d66303b",
"4e6fcf72-daf3-439f-a232-c434ce416af6",
"6054afcb-5bab-4702-90a0-b259b5d3217c",
"abed40dc-063f-4598-8ba5-9fe749c0615d",
"01b269ae-2111-4a07-81fd-3fcd711993b0",
"8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
"af2b02f7-acee-4be4-8b66-499fab394915",
"da1d63b8-fa12-417b-ba18-f748e5f770f3",
"636380ea-d5f6-4474-b6ca-b2ed578a20f1",
"5ba77536-05c5-4aae-a9ff-6e298d094c3e",
"4bc4eaf4-ca5e-4db2-8138-8d4e65af7c0b",
"672a1b02-c62f-4ae2-acf0-37f5fb3052b0",
"648fe544-16ba-44af-a587-12ccbe280ea6",
"8985d1e4-5b99-4711-add4-88949ebb2308",
"9e606842-2e27-43bf-b1d1-b43289c9589b",
"fcb6e45b-25c4-4087-9483-03d714f473a9",
"68c0c5b7-96f3-4e87-92a7-6c1b967fd2d2",
"fff629ea-046e-4793-8eec-1a5a15c3eb35",
"5c9a206c-bb00-4fb6-bb46-ee675c187df5",
"e975ae74-79bd-4672-8d1c-dc841a85781d",
"34a6938a-58da-4897-8639-9b90d6db5391",
"b5a22759-b4eb-4bf2-aeed-ad14e8615f19",
"2f9913a1-51ed-4db6-bfe0-7e1c95b3139e",
"2558031e-401d-4579-8e00-3ecf540fb492",
"0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
"4188d3a4-077d-46b7-9c86-23e1a036f6c1",
"51b11269-2ca8-4b2a-9163-f21758420e78",
"7e429b8d-a3f0-4ed0-9b58-08957d00b127",
"347ef137-7eeb-4c80-a3bb-0951f26a8aff",
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
"3aaa4e37-dc91-482e-99af-132a612d40f3",
"37608790-6147-45d0-9f20-1137bb35703d",
"f9584479-3d0d-4c79-affa-9ad7afdd8850",
"d681960f-7bc3-4286-9913-a8812ba3261a",
"21df9241-f8d7-4509-b7f1-37e501a823f7",
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
"a01fbce3-2793-461f-ab86-43680ccbae25",
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c"
]
# for example_id in calc_list:
# try:
@@ -283,9 +298,64 @@ if __name__ == '__main__':
"2fe4b718-3bd7-46ec-bdce-b184f5653624"
]
for example_id in multiple_list:
# for example_id in multiple_list:
# try:
# main("multi_apps", example_id)
# except Exception as e:
# logger.error("An error occurred while running the example: %s", e)
# continue
chrome_list = [
# "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"06fe7178-4491-4589-810f-2e2bc9502122",
"e1e75309-3ddb-4d09-92ec-de869c928143",
"35253b65-1c19-4304-8aa4-6884b8218fc0",
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
"44ee5668-ecd5-4366-a6ce-c1c9b8d4e938",
"2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3",
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
"af630914-714e-4a24-a7bb-f9af687d3b91"
]
# for example_id in chrome_list:
# try:
# main("chrome", example_id)
# except Exception as e:
# logger.error("An error occurred while running the example: %s", e)
# continue
writer_list = [
"6ada715d-3aae-4a32-a6a7-429b2e43fb93",
"ecc2413d-8a48-416e-a3a2-d30106ca36cb",
"0e47de2a-32e0-456c-a366-8c607ef7a9d2",
"4bcb1253-a636-4df4-8cb0-a35c04dfef31",
"0810415c-bde4-4443-9047-d5f70165a697",
"e528b65e-1107-4b8c-8988-490e4fece599",
"66399b0d-8fda-4618-95c4-bfc6191617e9",
"936321ce-5236-426a-9a20-e0e3c5dc536f",
"3ef2b351-8a84-4ff2-8724-d86eae9b842e",
"0b17a146-2934-46c7-8727-73ff6b6483e8",
"0e763496-b6bb-4508-a427-fad0b6c3e195",
"f178a4a9-d090-4b56-bc4c-4b72a61a035d",
"adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
"0a0faba3-5580-44df-965d-f562a99b291c",
"e246f6d8-78d7-44ac-b668-fcf47946cb50",
"8472fece-c7dd-4241-8d65-9b3cd1a0b568",
"88fe4b2d-3040-4c70-9a70-546a47764b48",
"d53ff5ee-3b1a-431e-b2be-30ed2673079b",
"72b810ef-4156-4d09-8f08-a0cf57e7cefe",
"6f81754e-285d-4ce0-b59e-af7edb02d108",
"b21acd93-60fd-4127-8a43-2f5178f4a830"
]
for example_id in writer_list:
try:
main("multi_apps", example_id)
main("libreoffice_writer", example_id)
except Exception as e:
logger.error("An error occurred while running the example: %s", e)
continue

110
experiment_screenshot_som.py
View File

@@ -151,7 +151,111 @@ def main(example_class, example_id):
if __name__ == '__main__':
xx_list = [
from tqdm import tqdm
# impress_list = [
# # "5d901039-a89c-4bfb-967b-bf66f4df075e",
# "550ce7e7-747b-495f-b122-acdc4d0b8e54",
# "455d3c66-7dc6-4537-a39a-36d3e9119df7",
# "af23762e-2bfd-4a1d-aada-20fa8de9ce07",
# "c59742c0-4323-4b9d-8a02-723c251deaa0",
# "ef9d12bd-bcee-4ba0-a40e-918400f43ddf",
# "9ec204e4-f0a3-42f8-8458-b772a6797cab",
# "0f84bef9-9790-432e-92b7-eece357603fb",
# "ce88f674-ab7a-43da-9201-468d38539e4a",
# "3b27600c-3668-4abd-8f84-7bcdebbccbdb",
# "a097acff-6266-4291-9fbd-137af7ecd439",
# "bf4e9888-f10f-47af-8dba-76413038b73c",
# "21760ecb-8f62-40d2-8d85-0cee5725cb72"
# ]
# for example_id in impress_list:
# main("libreoffice_impress", example_id)
vlc_list = [
"8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89",
"8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89",
"8f080098-ddb1-424c-b438-4e96e5e4786e",
"bba3381f-b5eb-4439-bd9e-80c22218d5a7",
"fba2c100-79e8-42df-ae74-b592418d54f4",
"efcf0d81-0835-4880-b2fd-d866e8bc2294",
"8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f",
"aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6",
"386dbd0e-0241-4a0a-b6a2-6704fba26b1c",
"9195653c-f4aa-453d-aa95-787f6ccfaae9",
"d06f0d4d-2cd5-4ede-8de9-598629438c6e",
"a5bbbcd5-b398-4c91-83d4-55e1e31bbb81",
"f3977615-2b45-4ac5-8bba-80c17dbe2a37",
"215dfd39-f493-4bc3-a027-8a97d72c61bf"
]
for example_id in xx_list:
main("xx", example_id)
# for example_id in tqdm(vlc_list):
# try:
# main("vlc", example_id)
# except Exception as e:
# print(f"An error occurred while running the example: {e}")
# continue
chrome_list = [
"bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"06fe7178-4491-4589-810f-2e2bc9502122",
"e1e75309-3ddb-4d09-92ec-de869c928143",
"35253b65-1c19-4304-8aa4-6884b8218fc0",
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
"44ee5668-ecd5-4366-a6ce-c1c9b8d4e938",
"2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3",
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
"af630914-714e-4a24-a7bb-f9af687d3b91"
]
for example_id in tqdm(chrome_list):
try:
main("chrome", example_id)
except Exception as e:
print(f"An error occurred while running the example: {e}")
continue
vs_code_list = [
"0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
"53ad5833-3455-407b-bbc6-45b4c79ab8fb",
"eabc805a-bfcf-4460-b250-ac92135819f6",
"982d12a5-beab-424f-8d38-d2a48429e511",
"4e60007a-f5be-4bfc-9723-c39affa0a6d3",
"e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
"9439a27b-18ae-42d8-9778-5f68f891805e",
"ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
"930fdb3b-11a8-46fe-9bac-577332e2640e",
"276cc624-87ea-4f08-ab93-f770e3790175",
"9d425400-e9b2-4424-9a4b-d4c7abac4140"
]
for example_id in tqdm(vs_code_list):
try:
main("vs_code", example_id)
except Exception as e:
print(f"An error occurred while running the example: {e}")
continue
thunderbird_list = [
"bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
"7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
"12086550-11c0-466b-b367-1d9e75b3910e",
"06fe7178-4491-4589-810f-2e2bc9502122",
"6766f2b8-8a72-417f-a9e5-56fcaa735837",
"e1e75309-3ddb-4d09-92ec-de869c928143",
"3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5",
"35253b65-1c19-4304-8aa4-6884b8218fc0",
"d088f539-cab4-4f9a-ac92-9999fc3a656e",
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
"030eeff7-b492-4218-b312-701ec99ee0cc",
"94760984-3ff5-41ee-8347-cf1af709fea0",
"99146c54-4f37-4ab8-9327-5f3291665e1e",
"c9e7eaf2-b1a1-4efc-a982-721972fa9f02"
]
for example_id in tqdm(thunderbird_list):
try:
main("thunderbird", example_id)
except Exception as e:
print(f"An error occurred while running the example: {e}")
continue

70
mm_agents/gpt_4v_agent.py
View File

@@ -15,12 +15,10 @@ import google.generativeai as genai
import openai
import requests
from PIL import Image
from openai.error import (
from openai import (
APIConnectionError,
APIError,
RateLimitError,
ServiceUnavailableError,
InvalidRequestError
RateLimitError
)
from mm_agents.accessibility_tree_wrap.heuristic_retrieve import find_leaf_nodes, filter_nodes, draw_bounding_boxes
@@ -115,6 +113,7 @@ def parse_actions_from_string(input_string):
def parse_code_from_string(input_string):
input_string = input_string.replace(";", "\n")
if input_string.strip() in ['WAIT', 'DONE', 'FAIL']:
return [input_string.strip()]
@@ -475,19 +474,15 @@ class GPT4v_Agent:
with open("messages.json", "w") as f:
f.write(json.dumps(messages, indent=4))
try:
response = self.call_llm({
"model": self.model,
"messages": messages,
"max_tokens": self.max_tokens
})
except Exception as e:
logger.warning("LLM INVOCATION ERROR: %s", str(e))
response = ""
response = self.call_llm({
"model": self.model,
"messages": messages,
"max_tokens": self.max_tokens
})
logger.debug("RESPONSE: %s", response)
# {{{
if self.exp == "seeact":
messages.append({
"role": "assistant",
@@ -523,13 +518,13 @@ class GPT4v_Agent:
except Exception as e:
print("Failed to parse action from response", e)
actions = None
self.thoughts.append("") # }}}
self.thoughts.append("")
return actions
@backoff.on_exception(
backoff.expo,
(APIError, RateLimitError, APIConnectionError, ServiceUnavailableError, InvalidRequestError),
(APIError, RateLimitError, APIConnectionError),
max_tries=10
)
def call_llm(self, payload):
@@ -582,23 +577,34 @@ class GPT4v_Agent:
misrtal_messages[1]['content'] = misrtal_messages[0]['content'] + "\n" + misrtal_messages[1]['content']
misrtal_messages.pop(0)
openai.api_base = "http://localhost:8000/v1"
openai.api_key = "test"
response = openai.ChatCompletion.create(
# openai.api_base = "http://localhost:8000/v1"
# openai.api_key = "test"
# response = openai.ChatCompletion.create(
# messages=misrtal_messages,
# model="Mixtral-8x7B-Instruct-v0.1"
# )
from openai import OpenAI
TOGETHER_API_KEY = "d011650e7537797148fb6170ec1e0be7ae75160375686fae02277136078e90d2"
client = OpenAI(api_key=TOGETHER_API_KEY,
base_url='https://api.together.xyz',
)
response = client.chat.completions.create(
messages=misrtal_messages,
model="Mixtral-8x7B-Instruct-v0.1"
model="mistralai/Mixtral-8x7B-Instruct-v0.1",
max_tokens=1024
)
try:
return response['choices'][0]['message']['content']
# return response['choices'][0]['message']['content']
return response.choices[0].message.content
except Exception as e:
print("Failed to call LLM: " + str(e))
return ""
elif self.model.startswith("gemini"):
api_key = os.environ.get("GENAI_API_KEY")
genai.api_key = api_key
def encoded_img_to_pil_img(data_str):
base64_str = data_str.replace("data:image/png;base64,", "")
image_data = base64.b64decode(base64_str)
@@ -611,8 +617,13 @@ class GPT4v_Agent:
gemini_messages = []
for i, message in enumerate(messages):
role_mapping = {
"assistant": "model",
"user": "user",
"system": "system"
}
gemini_message = {
"role": message["role"],
"role": role_mapping[message["role"]],
"parts": []
}
assert len(message["content"]) in [1, 2], "One text, or one text with one image"
@@ -628,6 +639,15 @@ class GPT4v_Agent:
gemini_messages.append(gemini_message)
# the mistral not support system message in our endpoint, so we concatenate it at the first user message
if gemini_messages[0]['role'] == "system":
gemini_messages[1]['parts'][0] = gemini_messages[0]['parts'][0] + "\n" + gemini_messages[1]['parts'][0]
gemini_messages.pop(0)
print(gemini_messages)
api_key = os.environ.get("GENAI_API_KEY")
assert api_key is not None, "Please set the GENAI_API_KEY environment variable"
genai.configure(api_key=api_key)
response = genai.GenerativeModel(self.model).generate_content(
gemini_messages,
generation_config={