lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-20 22:25:45 +08:00
parent 354741d19c 15e01e7ccc
commit aa3411f7e4
8 changed files with 104 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/controllers/python.py
View File

@@ -64,7 +64,7 @@ class PythonController:
It can be used to execute the pyautogui commands, or... any other python command. who knows?
"""
# command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
command_list = ["python3", "-c", self.pkgs_prefix.format(command=command)]
command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
payload = json.dumps({"command": command_list, "shell": False})
headers = {
'Content-Type': 'application/json'
@@ -344,4 +344,4 @@ class PythonController:
return response.json()["directory_tree"]
else:
logger.error("Failed to get directory tree. Status code: %d", response.status_code)
return None
return None

26
desktop_env/server/main.py
View File

@@ -414,9 +414,18 @@ def _create_pywinauto_node(node: BaseWrapper, depth: int = 0, flag: Optional[str
attribute_dict: Dict[str, Any] = {"name": node.element_info.name}
# States {{{ #
attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower()
attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower()
attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower()
try:
attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower()
except:
pass
try:
attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower()
except:
pass
try:
attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower()
except:
pass
if hasattr(node, "is_minimized"):
try:
@@ -603,9 +612,14 @@ def get_accessibility_tree():
@app.route('/screen_size', methods=['POST'])
def get_screen_size():
d = display.Display()
screen_width = d.screen().width_in_pixels
screen_height = d.screen().height_in_pixels
if platform_name=="Linux":
d = display.Display()
screen_width = d.screen().width_in_pixels
screen_height = d.screen().height_in_pixels
elif platform_name=="Windows":
user32 = ctypes.windll.user32
screen_width: int = user32.GetSystemMetrics(0)
screen_height: int = user32.GetSystemMetrics(1)
return jsonify(
{
"width": screen_width,

39
evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -21,16 +21,18 @@
"type": "launch",
"parameters": {
"command": [
"C:\Program Files\Google\Chrome\Application\chrome.exe",
"--remote-debugging-port=1337"
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337"
]
}
},
{
"type": "launch",
"parameters": {
"command": "nc -l -p 9222 |nc 127.0.0.1 1337",
"shell": true
"command": [
"ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
}
},
{
@@ -54,8 +56,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571",
"path": "/home/user/thunderbird-profile.tar.gz"
"url": "https://drive.google.com/uc?id=1Yy-ZrkMq4pIQq1Y75bD2WVJXxHMTaMqE&export=download",
"path": "C:\\Users\\chenj\\thunderbird-profile.7z"
}
]
}
@@ -64,21 +66,30 @@
"type": "execute",
"parameters": {
"command": [
"tar",
"-xz",
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
"C:\\Program Files\\7-Zip\\7z.exe",
"x", "C:\\Users\\chenj\\thunderbird-profile.7z"
]
}
},
{
"type": "execute",
"parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
"C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe"
]
}
}

11
evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json
View File

@@ -21,7 +21,7 @@
"type": "launch",
"parameters": {
"command": [
"google-chrome",
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337"
]
}
@@ -30,9 +30,8 @@
"type": "launch",
"parameters": {
"command": [
"socat",
"tcp-listen:9222,fork",
"tcp:localhost:1337"
"ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
}
},
@@ -60,7 +59,7 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=18TvzE8jnULU2g9XJsT-TaPEKcLGNVfu0&export=download&authuser=0&confirm=t&uuid=d914e031-9aa6-431b-81c0-73fcb87af027&at=APZUnTUx56WM_I3gnhHo-eZX__kx:1706158167271",
"path": "/home/user/Desktop/form.docx"
"path": "C:\\Users\\chenj\\Desktop\\form.docx"
}
]
}
@@ -68,7 +67,7 @@
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/form.docx"
"path": "C:\\Users\\chenj\\Desktop\\form.docx"
}
}
],

36
evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json
View File

@@ -8,8 +8,7 @@
"type": "launch",
"parameters": {
"command": [
"libreoffice",
"--calc"
"C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE"
]
}
},
@@ -18,8 +17,8 @@
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1wKXmJ14dnxSzdy9ZF_ePWU7zpevY6Dry&export=download&authuser=0&confirm=t&uuid=9b476c95-8eee-4a9a-8cee-c3620d5ce250&at=APZUnTUzDeeeMNr34DB1vEnBK6N7:1706719624132",
"path": "/home/user/thunderbird-profile.tar.gz"
"url": "https://drive.google.com/uc?id=1njAaNiujlh1DZzGK7nL5iZsppsNAMkH7&export=download",
"path": "C:\\Users\\chenj\\thunderbird-profile.7z"
}
]
}
@@ -28,21 +27,30 @@
"type": "execute",
"parameters": {
"command": [
"tar",
"--recursive-unlink",
"-xz",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
"C:\\Program Files\\7-Zip\\7z.exe",
"x", "C:\\Users\\chenj\\thunderbird-profile.7z"
]
}
},
{
"type": "execute",
"parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "launch",
"parameters": {
"command": [
"/usr/bin/thunderbird"
"C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe"
]
}
}
@@ -61,12 +69,12 @@
"result": [
{
"type": "vm_file",
"path": "/home/user/Desktop/contacts.csv",
"path": "C:\\Users\\chenj\\Desktop\\contacts.csv",
"dest": "contacts.csv"
},
{
"type": "vm_file",
"path": "/home/user/Desktop/contacts.xlsx",
"path": "C:\\Users\\chenj\\Desktop\\contacts.xlsx",
"dest": "contacts.xlsx"
}
],

20
evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json
View File

@@ -10,11 +10,11 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1JGZNCShtmpu7A8Z8lkjc8hdFEAMXZVvh&export=download&authuser=0&confirm=t&uuid=67063da6-2a72-4ed2-92b2-ade508439ce4&at=APZUnTUgS17YjX-D0oSvALwnPosB:1709368886960",
"path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx"
"path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx"
},
{
"url": "https://drive.usercontent.google.com/download?id=1iySmK8zvTzgmERH7KQuESP05NBsMunhV&export=download&authuser=0&confirm=t&uuid=130f6cee-0f9a-4f2e-a84d-89a3b302f350&at=APZUnTXugQOTOApe1_zxUbafo2Sp:1709369519349",
"path": "/home/user/Desktop/book_list_result.docx"
"path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx"
}
]
}
@@ -22,7 +22,7 @@
{
"type": "open",
"parameters": {
"path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx"
"path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx"
}
}
],
@@ -38,7 +38,7 @@
{
"type": "activate_window",
"parameters": {
"window_name": "book_list_result.docx - LibreOffice Writer",
"window_name": "book_list_result - Word",
"strict": true
}
},
@@ -54,10 +54,16 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"expected": {
"type": "cloud_file",
@@ -66,7 +72,7 @@
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/book_list_result.docx",
"path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx",
"dest": "book_list_result.docx"
}
}

28
evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -10,21 +10,27 @@
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1l09TnSiXo-qOK2UazcIdrT_M6JwTfzq7&export=download&authuser=0&confirm=t&uuid=80bd550f-f3a6-4b69-ae0f-221c12b11fd9&at=APZUnTWgUlKuIDJZmkr0Q9Bze3w_:1709784652645",
"path": "/home/user/Desktop/calculator.zip"
"path": "C:\\Users\\chenj\\Desktop\\calculator.zip"
}
]
}
},
{
"type": "execute",
"parameters": {
"command": [
"/bin/bash",
"-c",
"unzip /home/user/Desktop/calculator.zip -d /home/user/Desktop/ && rm -rf /home/user/Desktop/calculator.zip"
]
}
}
{
"type": "execute",
"parameters": {
"command": [
"C:\\Program Files\\7-Zip\\7z.exe",
"C:\\Users\\chenj\\Desktop\\calculator.zip"
]
}
},
{
"type": "execute",
"parameters": {
"command": "rm C:\\Users\\chenj\\Desktop\\calculator.zip",
"shell": true
}
}
],
"trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef",
"related_apps": [

32
main.py
View File

@@ -70,38 +70,6 @@ def human_agent():
done = False
logger.info('\x1b[32m[TASK INSTRUCTION]: \x1b[32;3m%s\x1b[0m', example["instruction"])
trajectory = [
{
"action_type": "MOVE_TO", #
"parameters": {
"x": 754,
"y": 1057
}
},
{"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
]
for i in range(len(trajectory)):
# action = get_human_action()
# action = {
# "action_type": 0,
# "click_type": 3,
# }
logger.info(trajectory[i])
observation, reward, done, info = env.step(trajectory[i])
observation.pop("accessibility_tree")
logger.info("Observation: %s", observation)
logger.info("Reward: %.2f", reward)
logger.info("Info: %s", info)
logger.info("================================\n")
if done:
logger.info("The episode is done.")
break
input("Press Enter to start human operation...")
human_start_time = time.time()
input("Press Enter to finish human operation.")