lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'zdy'

Browse Source
This commit is contained in:
David Chang
2024-03-20 22:25:45 +08:00
parent 354741d19c 15e01e7ccc
commit aa3411f7e4
8 changed files with 104 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/controllers/python.py
View File

@@ -64,7 +64,7 @@ class PythonController:
It can be used to execute the pyautogui commands, or... any other python command. who knows? It can be used to execute the pyautogui commands, or... any other python command. who knows?
""" """
# command_list = ["python", "-c", self.pkgs_prefix.format(command=command)] # command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
command_list = ["python3", "-c", self.pkgs_prefix.format(command=command)] command_list = ["python", "-c", self.pkgs_prefix.format(command=command)]
payload = json.dumps({"command": command_list, "shell": False}) payload = json.dumps({"command": command_list, "shell": False})
headers = { headers = {
'Content-Type': 'application/json' 'Content-Type': 'application/json'
@@ -344,4 +344,4 @@ class PythonController:
return response.json()["directory_tree"] return response.json()["directory_tree"]
else: else:
logger.error("Failed to get directory tree. Status code: %d", response.status_code) logger.error("Failed to get directory tree. Status code: %d", response.status_code)
return None return None

26
desktop_env/server/main.py
View File

@@ -414,9 +414,18 @@ def _create_pywinauto_node(node: BaseWrapper, depth: int = 0, flag: Optional[str
attribute_dict: Dict[str, Any] = {"name": node.element_info.name} attribute_dict: Dict[str, Any] = {"name": node.element_info.name}
# States {{{ # # States {{{ #
attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower() try:
attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower() attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower()
attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower() except:
pass
try:
attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower()
except:
pass
try:
attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower()
except:
pass
if hasattr(node, "is_minimized"): if hasattr(node, "is_minimized"):
try: try:
@@ -603,9 +612,14 @@ def get_accessibility_tree():
@app.route('/screen_size', methods=['POST']) @app.route('/screen_size', methods=['POST'])
def get_screen_size(): def get_screen_size():
d = display.Display() if platform_name=="Linux":
screen_width = d.screen().width_in_pixels d = display.Display()
screen_height = d.screen().height_in_pixels screen_width = d.screen().width_in_pixels
screen_height = d.screen().height_in_pixels
elif platform_name=="Windows":
user32 = ctypes.windll.user32
screen_width: int = user32.GetSystemMetrics(0)
screen_height: int = user32.GetSystemMetrics(1)
return jsonify( return jsonify(
{ {
"width": screen_width, "width": screen_width,

39
evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json
View File

@@ -21,16 +21,18 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"C:\Program Files\Google\Chrome\Application\chrome.exe", "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337" "--remote-debugging-port=1337"
] ]
} }
}, },
{ {
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": "nc -l -p 9222 |nc 127.0.0.1 1337", "command": [
"shell": true "ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"--sh-exec", "ncat.exe 127.0.0.1 1337"
]
} }
}, },
{ {
@@ -54,8 +56,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571", "url": "https://drive.google.com/uc?id=1Yy-ZrkMq4pIQq1Y75bD2WVJXxHMTaMqE&export=download",
"path": "/home/user/thunderbird-profile.tar.gz" "path": "C:\\Users\\chenj\\thunderbird-profile.7z"
} }
] ]
} }
@@ -64,21 +66,30 @@
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": [ "command": [
"tar", "C:\\Program Files\\7-Zip\\7z.exe",
"-xz", "x", "C:\\Users\\chenj\\thunderbird-profile.7z"
"--recursive-unlink",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
] ]
} }
}, },
{
"type": "execute",
"parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{ {
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"/usr/bin/thunderbird" "C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe"
] ]
} }
} }

11
evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json
View File

@@ -21,7 +21,7 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"google-chrome", "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"--remote-debugging-port=1337" "--remote-debugging-port=1337"
] ]
} }
@@ -30,9 +30,8 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"socat", "ncat.exe", "-k", "-l", "0.0.0.0", "9222",
"tcp-listen:9222,fork", "--sh-exec", "ncat.exe 127.0.0.1 1337"
"tcp:localhost:1337"
] ]
} }
}, },
@@ -60,7 +59,7 @@
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=18TvzE8jnULU2g9XJsT-TaPEKcLGNVfu0&export=download&authuser=0&confirm=t&uuid=d914e031-9aa6-431b-81c0-73fcb87af027&at=APZUnTUx56WM_I3gnhHo-eZX__kx:1706158167271", "url": "https://drive.usercontent.google.com/download?id=18TvzE8jnULU2g9XJsT-TaPEKcLGNVfu0&export=download&authuser=0&confirm=t&uuid=d914e031-9aa6-431b-81c0-73fcb87af027&at=APZUnTUx56WM_I3gnhHo-eZX__kx:1706158167271",
"path": "/home/user/Desktop/form.docx" "path": "C:\\Users\\chenj\\Desktop\\form.docx"
} }
] ]
} }
@@ -68,7 +67,7 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "/home/user/Desktop/form.docx" "path": "C:\\Users\\chenj\\Desktop\\form.docx"
} }
} }
], ],

36
evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json
View File

@@ -8,8 +8,7 @@
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"libreoffice", "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE"
"--calc"
] ]
} }
}, },
@@ -18,8 +17,8 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1wKXmJ14dnxSzdy9ZF_ePWU7zpevY6Dry&export=download&authuser=0&confirm=t&uuid=9b476c95-8eee-4a9a-8cee-c3620d5ce250&at=APZUnTUzDeeeMNr34DB1vEnBK6N7:1706719624132", "url": "https://drive.google.com/uc?id=1njAaNiujlh1DZzGK7nL5iZsppsNAMkH7&export=download",
"path": "/home/user/thunderbird-profile.tar.gz" "path": "C:\\Users\\chenj\\thunderbird-profile.7z"
} }
] ]
} }
@@ -28,21 +27,30 @@
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": [ "command": [
"tar", "C:\\Program Files\\7-Zip\\7z.exe",
"--recursive-unlink", "x", "C:\\Users\\chenj\\thunderbird-profile.7z"
"-xz",
"-f",
"/home/user/thunderbird-profile.tar.gz",
"-C",
"/home/user/"
] ]
} }
}, },
{
"type": "execute",
"parameters": {
"command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{
"type": "execute",
"parameters": {
"command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird",
"shell": true
}
},
{ {
"type": "launch", "type": "launch",
"parameters": { "parameters": {
"command": [ "command": [
"/usr/bin/thunderbird" "C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe"
] ]
} }
} }
@@ -61,12 +69,12 @@
"result": [ "result": [
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Desktop/contacts.csv", "path": "C:\\Users\\chenj\\Desktop\\contacts.csv",
"dest": "contacts.csv" "dest": "contacts.csv"
}, },
{ {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Desktop/contacts.xlsx", "path": "C:\\Users\\chenj\\Desktop\\contacts.xlsx",
"dest": "contacts.xlsx" "dest": "contacts.xlsx"
} }
], ],

20
evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json
View File

@@ -10,11 +10,11 @@
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1JGZNCShtmpu7A8Z8lkjc8hdFEAMXZVvh&export=download&authuser=0&confirm=t&uuid=67063da6-2a72-4ed2-92b2-ade508439ce4&at=APZUnTUgS17YjX-D0oSvALwnPosB:1709368886960", "url": "https://drive.usercontent.google.com/download?id=1JGZNCShtmpu7A8Z8lkjc8hdFEAMXZVvh&export=download&authuser=0&confirm=t&uuid=67063da6-2a72-4ed2-92b2-ade508439ce4&at=APZUnTUgS17YjX-D0oSvALwnPosB:1709368886960",
"path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" "path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx"
}, },
{ {
"url": "https://drive.usercontent.google.com/download?id=1iySmK8zvTzgmERH7KQuESP05NBsMunhV&export=download&authuser=0&confirm=t&uuid=130f6cee-0f9a-4f2e-a84d-89a3b302f350&at=APZUnTXugQOTOApe1_zxUbafo2Sp:1709369519349", "url": "https://drive.usercontent.google.com/download?id=1iySmK8zvTzgmERH7KQuESP05NBsMunhV&export=download&authuser=0&confirm=t&uuid=130f6cee-0f9a-4f2e-a84d-89a3b302f350&at=APZUnTXugQOTOApe1_zxUbafo2Sp:1709369519349",
"path": "/home/user/Desktop/book_list_result.docx" "path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx"
} }
] ]
} }
@@ -22,7 +22,7 @@
{ {
"type": "open", "type": "open",
"parameters": { "parameters": {
"path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" "path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx"
} }
} }
], ],
@@ -38,7 +38,7 @@
{ {
"type": "activate_window", "type": "activate_window",
"parameters": { "parameters": {
"window_name": "book_list_result.docx - LibreOffice Writer", "window_name": "book_list_result - Word",
"strict": true "strict": true
} }
}, },
@@ -54,10 +54,16 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
] ]
} }
} },
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
], ],
"expected": { "expected": {
"type": "cloud_file", "type": "cloud_file",
@@ -66,7 +72,7 @@
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "/home/user/Desktop/book_list_result.docx", "path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx",
"dest": "book_list_result.docx" "dest": "book_list_result.docx"
} }
} }

28
evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json
View File

@@ -10,21 +10,27 @@
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1l09TnSiXo-qOK2UazcIdrT_M6JwTfzq7&export=download&authuser=0&confirm=t&uuid=80bd550f-f3a6-4b69-ae0f-221c12b11fd9&at=APZUnTWgUlKuIDJZmkr0Q9Bze3w_:1709784652645", "url": "https://drive.usercontent.google.com/download?id=1l09TnSiXo-qOK2UazcIdrT_M6JwTfzq7&export=download&authuser=0&confirm=t&uuid=80bd550f-f3a6-4b69-ae0f-221c12b11fd9&at=APZUnTWgUlKuIDJZmkr0Q9Bze3w_:1709784652645",
"path": "/home/user/Desktop/calculator.zip" "path": "C:\\Users\\chenj\\Desktop\\calculator.zip"
} }
] ]
} }
}, },
{ {
"type": "execute", "type": "execute",
"parameters": { "parameters": {
"command": [ "command": [
"/bin/bash", "C:\\Program Files\\7-Zip\\7z.exe",
"-c", "C:\\Users\\chenj\\Desktop\\calculator.zip"
"unzip /home/user/Desktop/calculator.zip -d /home/user/Desktop/ && rm -rf /home/user/Desktop/calculator.zip" ]
] }
} },
} {
"type": "execute",
"parameters": {
"command": "rm C:\\Users\\chenj\\Desktop\\calculator.zip",
"shell": true
}
}
], ],
"trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef", "trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef",
"related_apps": [ "related_apps": [

32
main.py
View File

@@ -70,38 +70,6 @@ def human_agent():
done = False done = False
logger.info('\x1b[32m[TASK INSTRUCTION]: \x1b[32;3m%s\x1b[0m', example["instruction"]) logger.info('\x1b[32m[TASK INSTRUCTION]: \x1b[32;3m%s\x1b[0m', example["instruction"])
trajectory = [
{
"action_type": "MOVE_TO", #
"parameters": {
"x": 754,
"y": 1057
}
},
{"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
]
for i in range(len(trajectory)):
# action = get_human_action()
# action = {
# "action_type": 0,
# "click_type": 3,
# }
logger.info(trajectory[i])
observation, reward, done, info = env.step(trajectory[i])
observation.pop("accessibility_tree")
logger.info("Observation: %s", observation)
logger.info("Reward: %.2f", reward)
logger.info("Info: %s", info)
logger.info("================================\n")
if done:
logger.info("The episode is done.")
break
input("Press Enter to start human operation...") input("Press Enter to start human operation...")
human_start_time = time.time() human_start_time = time.time()
input("Press Enter to finish human operation.") input("Press Enter to finish human operation.")