lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan10th

Browse Source 
new Thunderbird task config
This commit is contained in:
David Chang
2024-01-10 17:36:59 +08:00
parent ec3bc3079f
commit cf5d480f44
10 changed files with 325 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
desktop_env/controllers/python.py
View File

@@ -1,5 +1,5 @@
import json
from typing import Any, Dict
from typing import Any, Dict, Optional
import requests
from desktop_env.envs.actions import KEYBOARD_KEYS
@@ -22,6 +22,15 @@ class PythonController:
logger.error("Failed to get screenshot. Status code: %d", response.status_code)
return None
def get_accessibility_tree(self) -> Optional[str]:
response: requests.Response = requests.get(self.http_server + "/accessibility")
if response.status_code == 200:
return response.json()["AT"]
else:
logger.error("Failed to get accessibility tree. Status code: %d", response.status_code)
return None
def get_file(self, file_path: str):
"""
Gets a file from the server.
@@ -65,7 +74,7 @@ class PythonController:
if action_type == "MOVE_TO":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.moveTo()")
self.execute_python_command("pyautogui.moveTo()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -75,7 +84,7 @@ class PythonController:
elif action_type == "CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.click()")
self.execute_python_command("pyautogui.click()")
elif "button" in parameters and "x" in parameters and "y" in parameters:
button = parameters["button"]
x = parameters["x"]
@@ -106,7 +115,7 @@ class PythonController:
elif action_type == "MOUSE_DOWN":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.mouseDown()")
self.execute_python_command("pyautogui.mouseDown()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseDown(button='{button}')")
@@ -115,7 +124,7 @@ class PythonController:
elif action_type == "MOUSE_UP":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.mouseUp()")
self.execute_python_command("pyautogui.mouseUp()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseUp(button='{button}')")
@@ -124,7 +133,7 @@ class PythonController:
elif action_type == "RIGHT_CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.rightClick()")
self.execute_python_command("pyautogui.rightClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -134,7 +143,7 @@ class PythonController:
elif action_type == "DOUBLE_CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.doubleClick()")
self.execute_python_command("pyautogui.doubleClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -200,7 +209,7 @@ class PythonController:
raise Exception(f"Unknown parameters: {parameters}")
keys = parameters["keys"]
if not isinstance(keys, list):
raise Exception(f"Keys must be a list of keys")
raise Exception("Keys must be a list of keys")
for key in keys:
if key.lower() not in KEYBOARD_KEYS:
raise Exception(f"Key must be one of {KEYBOARD_KEYS}")