From 15e01e7ccc218f65b1f21a3b61c55e25e0e0c229 Mon Sep 17 00:00:00 2001 From: David Chang Date: Wed, 20 Mar 2024 22:22:57 +0800 Subject: [PATCH] ver Mar20thv2 fixed bugs in server/main.py (_create_pywinauto_node and get_screen_size) finished migration of a few task configs to Windows fixed bug in python.py --- branch_flag | 2 +- desktop_env/controllers/python.py | 4 +- desktop_env/server/main.py | 26 ++++++++++--- .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 39 ++++++++++++------- .../897e3b53-5d4d-444b-85cb-2cdc8a97d903.json | 11 +++--- .../c867c42d-a52d-4a24-8ae3-f75d256b5618.json | 36 ++++++++++------- .../da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json | 20 ++++++---- .../f918266a-b3e0-4914-865d-4faa564f1aef.json | 28 +++++++------ main.py | 32 --------------- 9 files changed, 105 insertions(+), 93 deletions(-) diff --git a/branch_flag b/branch_flag index cb4898e..9daeafb 100644 --- a/branch_flag +++ b/branch_flag @@ -1 +1 @@ -claude +test diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index 4159cde..ea11644 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -64,7 +64,7 @@ class PythonController: It can be used to execute the pyautogui commands, or... any other python command. who knows? """ # command_list = ["python", "-c", self.pkgs_prefix.format(command=command)] - command_list = ["python3", "-c", self.pkgs_prefix.format(command=command)] + command_list = ["python", "-c", self.pkgs_prefix.format(command=command)] payload = json.dumps({"command": command_list, "shell": False}) headers = { 'Content-Type': 'application/json' @@ -344,4 +344,4 @@ class PythonController: return response.json()["directory_tree"] else: logger.error("Failed to get directory tree. Status code: %d", response.status_code) - return None \ No newline at end of file + return None diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index e9ab99f..8e900a3 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -414,9 +414,18 @@ def _create_pywinauto_node(node: BaseWrapper, depth: int = 0, flag: Optional[str attribute_dict: Dict[str, Any] = {"name": node.element_info.name} # States {{{ # - attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower() - attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower() - attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower() + try: + attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower() + except: + pass + try: + attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower() + except: + pass + try: + attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower() + except: + pass if hasattr(node, "is_minimized"): try: @@ -603,9 +612,14 @@ def get_accessibility_tree(): @app.route('/screen_size', methods=['POST']) def get_screen_size(): - d = display.Display() - screen_width = d.screen().width_in_pixels - screen_height = d.screen().height_in_pixels + if platform_name=="Linux": + d = display.Display() + screen_width = d.screen().width_in_pixels + screen_height = d.screen().height_in_pixels + elif platform_name=="Windows": + user32 = ctypes.windll.user32 + screen_width: int = user32.GetSystemMetrics(0) + screen_height: int = user32.GetSystemMetrics(1) return jsonify( { "width": screen_width, diff --git a/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json index 6e179ca..2106e9e 100644 --- a/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json +++ b/evaluation_examples/examples/Windows/multi_app/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -21,16 +21,18 @@ "type": "launch", "parameters": { "command": [ - "C:\Program Files\Google\Chrome\Application\chrome.exe", - "--remote-debugging-port=1337" + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "--remote-debugging-port=1337" ] } }, { "type": "launch", "parameters": { - "command": "nc -l -p 9222 |nc 127.0.0.1 1337", - "shell": true + "command": [ + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" + ] } }, { @@ -54,8 +56,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571", - "path": "/home/user/thunderbird-profile.tar.gz" + "url": "https://drive.google.com/uc?id=1Yy-ZrkMq4pIQq1Y75bD2WVJXxHMTaMqE&export=download", + "path": "C:\\Users\\chenj\\thunderbird-profile.7z" } ] } @@ -64,21 +66,30 @@ "type": "execute", "parameters": { "command": [ - "tar", - "-xz", - "--recursive-unlink", - "-f", - "/home/user/thunderbird-profile.tar.gz", - "-C", - "/home/user/" + "C:\\Program Files\\7-Zip\\7z.exe", + "x", "C:\\Users\\chenj\\thunderbird-profile.7z" ] } }, + { + "type": "execute", + "parameters": { + "command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "shell": true + } + }, { "type": "launch", "parameters": { "command": [ - "/usr/bin/thunderbird" + "C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe" ] } } diff --git a/evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json b/evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json index 184049d..948d662 100644 --- a/evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json +++ b/evaluation_examples/examples/Windows/multi_app/897e3b53-5d4d-444b-85cb-2cdc8a97d903.json @@ -21,7 +21,7 @@ "type": "launch", "parameters": { "command": [ - "google-chrome", + "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", "--remote-debugging-port=1337" ] } @@ -30,9 +30,8 @@ "type": "launch", "parameters": { "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" + "ncat.exe", "-k", "-l", "0.0.0.0", "9222", + "--sh-exec", "ncat.exe 127.0.0.1 1337" ] } }, @@ -60,7 +59,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=18TvzE8jnULU2g9XJsT-TaPEKcLGNVfu0&export=download&authuser=0&confirm=t&uuid=d914e031-9aa6-431b-81c0-73fcb87af027&at=APZUnTUx56WM_I3gnhHo-eZX__kx:1706158167271", - "path": "/home/user/Desktop/form.docx" + "path": "C:\\Users\\chenj\\Desktop\\form.docx" } ] } @@ -68,7 +67,7 @@ { "type": "open", "parameters": { - "path": "/home/user/Desktop/form.docx" + "path": "C:\\Users\\chenj\\Desktop\\form.docx" } } ], diff --git a/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json b/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json index 2a8162d..72d43a0 100644 --- a/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json +++ b/evaluation_examples/examples/Windows/multi_app/c867c42d-a52d-4a24-8ae3-f75d256b5618.json @@ -8,8 +8,7 @@ "type": "launch", "parameters": { "command": [ - "libreoffice", - "--calc" + "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE" ] } }, @@ -18,8 +17,8 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1wKXmJ14dnxSzdy9ZF_ePWU7zpevY6Dry&export=download&authuser=0&confirm=t&uuid=9b476c95-8eee-4a9a-8cee-c3620d5ce250&at=APZUnTUzDeeeMNr34DB1vEnBK6N7:1706719624132", - "path": "/home/user/thunderbird-profile.tar.gz" + "url": "https://drive.google.com/uc?id=1njAaNiujlh1DZzGK7nL5iZsppsNAMkH7&export=download", + "path": "C:\\Users\\chenj\\thunderbird-profile.7z" } ] } @@ -28,21 +27,30 @@ "type": "execute", "parameters": { "command": [ - "tar", - "--recursive-unlink", - "-xz", - "-f", - "/home/user/thunderbird-profile.tar.gz", - "-C", - "/home/user/" + "C:\\Program Files\\7-Zip\\7z.exe", + "x", "C:\\Users\\chenj\\thunderbird-profile.7z" ] } }, + { + "type": "execute", + "parameters": { + "command": "rm -r C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mv C:\\Users\\chenj\\Thunderbird C:\\Users\\chenj\\AppData\\Roaming\\Thunderbird", + "shell": true + } + }, { "type": "launch", "parameters": { "command": [ - "/usr/bin/thunderbird" + "C:\\Program Files\\Mozilla Thunderbird\\thunderbird.exe" ] } } @@ -61,12 +69,12 @@ "result": [ { "type": "vm_file", - "path": "/home/user/Desktop/contacts.csv", + "path": "C:\\Users\\chenj\\Desktop\\contacts.csv", "dest": "contacts.csv" }, { "type": "vm_file", - "path": "/home/user/Desktop/contacts.xlsx", + "path": "C:\\Users\\chenj\\Desktop\\contacts.xlsx", "dest": "contacts.xlsx" } ], diff --git a/evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json b/evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json index ee3fa37..9138805 100644 --- a/evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json +++ b/evaluation_examples/examples/Windows/multi_app/da52d699-e8d2-4dc5-9191-a2199e0b6a9b.json @@ -10,11 +10,11 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1JGZNCShtmpu7A8Z8lkjc8hdFEAMXZVvh&export=download&authuser=0&confirm=t&uuid=67063da6-2a72-4ed2-92b2-ade508439ce4&at=APZUnTUgS17YjX-D0oSvALwnPosB:1709368886960", - "path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" + "path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx" }, { "url": "https://drive.usercontent.google.com/download?id=1iySmK8zvTzgmERH7KQuESP05NBsMunhV&export=download&authuser=0&confirm=t&uuid=130f6cee-0f9a-4f2e-a84d-89a3b302f350&at=APZUnTXugQOTOApe1_zxUbafo2Sp:1709369519349", - "path": "/home/user/Desktop/book_list_result.docx" + "path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx" } ] } @@ -22,7 +22,7 @@ { "type": "open", "parameters": { - "path": "/home/user/Desktop/2023_validation_Book_Reading_Rate.xlsx" + "path": "C:\\Users\\chenj\\Desktop\\2023_validation_Book_Reading_Rate.xlsx" } } ], @@ -38,7 +38,7 @@ { "type": "activate_window", "parameters": { - "window_name": "book_list_result.docx - LibreOffice Writer", + "window_name": "book_list_result - Word", "strict": true } }, @@ -54,10 +54,16 @@ "command": [ "python", "-c", - "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); " + "import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");" ] } - } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + } ], "expected": { "type": "cloud_file", @@ -66,7 +72,7 @@ }, "result": { "type": "vm_file", - "path": "/home/user/Desktop/book_list_result.docx", + "path": "C:\\Users\\chenj\\Desktop\\book_list_result.docx", "dest": "book_list_result.docx" } } diff --git a/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json b/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json index 27adcf8..09c8983 100644 --- a/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json +++ b/evaluation_examples/examples/Windows/multi_app/f918266a-b3e0-4914-865d-4faa564f1aef.json @@ -10,21 +10,27 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=1l09TnSiXo-qOK2UazcIdrT_M6JwTfzq7&export=download&authuser=0&confirm=t&uuid=80bd550f-f3a6-4b69-ae0f-221c12b11fd9&at=APZUnTWgUlKuIDJZmkr0Q9Bze3w_:1709784652645", - "path": "/home/user/Desktop/calculator.zip" + "path": "C:\\Users\\chenj\\Desktop\\calculator.zip" } ] } }, - { - "type": "execute", - "parameters": { - "command": [ - "/bin/bash", - "-c", - "unzip /home/user/Desktop/calculator.zip -d /home/user/Desktop/ && rm -rf /home/user/Desktop/calculator.zip" - ] - } - } + { + "type": "execute", + "parameters": { + "command": [ + "C:\\Program Files\\7-Zip\\7z.exe", + "C:\\Users\\chenj\\Desktop\\calculator.zip" + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "rm C:\\Users\\chenj\\Desktop\\calculator.zip", + "shell": true + } + } ], "trajectory": "trajectories/f918266a-b3e0-4914-865d-4faa564f1aef", "related_apps": [ diff --git a/main.py b/main.py index 06debec..b6ae310 100644 --- a/main.py +++ b/main.py @@ -70,38 +70,6 @@ def human_agent(): done = False logger.info('\x1b[32m[TASK INSTRUCTION]: \x1b[32;3m%s\x1b[0m', example["instruction"]) - trajectory = [ - { - "action_type": "MOVE_TO", # - "parameters": { - "x": 754, - "y": 1057 - } - }, - {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}} - ] - - for i in range(len(trajectory)): - # action = get_human_action() - - # action = { - # "action_type": 0, - # "click_type": 3, - # } - logger.info(trajectory[i]) - - observation, reward, done, info = env.step(trajectory[i]) - observation.pop("accessibility_tree") - logger.info("Observation: %s", observation) - logger.info("Reward: %.2f", reward) - logger.info("Info: %s", info) - - logger.info("================================\n") - - if done: - logger.info("The episode is done.") - break - input("Press Enter to start human operation...") human_start_time = time.time() input("Press Enter to finish human operation.")