From c875cad3e519cf14f346539fb4bf23d27ee34bf3 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sun, 28 Jan 2024 15:32:14 +0800 Subject: [PATCH] Fix some errors found in thunderbird examples --- .../0bf05a7d-b28b-44d2-955a-50b41e24012a.json | 86 ++++++------ .../a01fbce3-2793-461f-ab86-43680ccbae25.json | 109 ---------------- ...bce3-2793-461f-ab86-43680ccbae25.json.json | 85 ------------ experiment_a11y_tree.py | 123 +++++++++++------- experiment_screenshot.py | 53 ++++---- experiment_screenshot_a11y_tree.py | 45 ++----- experiment_screenshot_seeact.py | 32 +---- experiment_screenshot_som.py | 32 +---- 8 files changed, 172 insertions(+), 393 deletions(-) delete mode 100644 evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json delete mode 100644 evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json diff --git a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json index 4340397..48e5e61 100644 --- a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json +++ b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json @@ -21,55 +21,55 @@ "path": "/home/user/Customers_New_7digit_Id.xlsx" } } - }, + ], "trajectory": "trajectories/0bf05a7d-b28b-44d2-955a-50b41e24012a", "related_apps": [ "libreoffice calc" ], "evaluator": { "postconfig": [ - { - "type": "activate_window", - "parameters": { - "window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc", - "strict": true - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" - ] - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "libreoffice", - "--convert-to", - "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", - "--outdir", - "/home/user", - "/home/user/Customers_New_7digit_Id.xlsx" - ] - } - } + { + "type": "activate_window", + "parameters": { + "window_name": "Customers_New_7digit_Id.xlsx - LibreOffice Calc", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "libreoffice", + "--convert-to", + "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", + "--outdir", + "/home/user", + "/home/user/Customers_New_7digit_Id.xlsx" + ] + } + } ], "func": "compare_table", "result": { diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json deleted file mode 100644 index b19b14a..0000000 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "id": "a01fbce3-2793-461f-ab86-43680ccbae25", - "snapshot": "libreoffice_calc", - "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", - "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", - "config": [ - { - "type": "download", - "parameters": { - "file": [ - { - "url": "https://drive.usercontent.google.com/download?id=1uT0axjo9lwkKu6hYVnsAL2FCrdH0DLUv&export=download&authuser=0&confirm=t&uuid=e7da6304-9c7a-4862-8a30-9f2284b843da&at=APZUnTVNHThpAZJmF6IuPckFvslw:1704187618838", - "path": "/home/user/Set_Decimal_Separator_Dot.xlsx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "/home/user/Set_Decimal_Separator_Dot.xlsx" - } - } - ], - "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", - "related_apps": [ - "libreoffice_calc" - ], - "evaluator": { - "postconfig": [ - { - "type": "activate_window", - "parameters": { - "window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc", - "strict": true - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" - ] - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "libreoffice", - "--convert-to", - "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true,false,false,1", - "--outdir", - "/home/user", - "/home/user/Set_Decimal_Separator_Dot.xlsx" - ] - } - } - ], - "func": "compare_table", - "result": { - "type": "vm_file", - "path": [ - "/home/user/Set_Decimal_Separator_Dot.xlsx", - "/home/user/Set_Decimal_Separator_Dot-Sheet1.csv" - ], - "dest": [ - "Set_Decimal_Separator_Dot.xlsx", - "Set_Decimal_Separator_Dot-Sheet1.csv" - ], - "multi": true - }, - "expected": { - "type": "cloud_file", - "path": [ - "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", - "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" - ], - "dest": [ - "Set_Decimal_Separator_Dot_gold.xlsx", - "Set_Decimal_Separator_Dot_gold-Sheet1.csv" - ], - "multi": true - }, - "options": { - "rules": [ - { - "type": "sheet_print", - "sheet_idx0": "RNSheet1", - "sheet_idx1": "ENSheet1" - } - ] - } - } -} diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json deleted file mode 100644 index e7bc03b..0000000 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "id": "a01fbce3-2793-461f-ab86-43680ccbae25", - "snapshot": "libreoffice_calc", - "instruction": "I need to set the decimal separator as a comma (,) for localized data representation and clarity in visualization. Can you assist with this?", - "source": "https://superuser.com/questions/1250677/how-to-set-decimal-separator-in-libre-office-calc", - "config": [], - "trajectory": "trajectories/a01fbce3-2793-461f-ab86-43680ccbae25", - "related_apps": [ - "libreoffice_calc" - ], - "evaluator": { - "postconfig": [ - { - "type": "activate_window", - "parameters": { - "window_name": "Set_Decimal_Separator_Dot.xlsx - LibreOffice Calc", - "strict": true - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" - ] - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "libreoffice", - "--convert-to", - "csv:Text - txt - csv (StarCalc):44,34,UTF-8,,,,false,true,true", - "--outdir", - "/home/user", - "/home/user/Set_Decimal_Separator_Dot.xlsx" - ] - } - } - ], - "func": "compare_table", - "result": { - "type": "vm_file", - "path": [ - "/home/user/Set_Decimal_Separator_Dot.xlsx", - "/home/user/Set_Decimal_Separator_Dot.csv" - ], - "dest": [ - "Set_Decimal_Separator_Dot.xlsx", - "Set_Decimal_Separator_Dot.csv" - ], - "multi": true - }, - "expected": { - "type": "cloud_file", - "path": [ - "https://drive.usercontent.google.com/download?id=15O0l5fxVi1JX_12KOLfbxWPHjXPZPon5&export=download&authuser=0&confirm=t&uuid=395e6c57-11a7-4b33-af4c-98ff2390742b&at=APZUnTVKcrUGrjRfBEwT_AD53Cmn:1705497822975", - "https://drive.usercontent.google.com/download?id=1rKDWcovxw4Qtd3RHs7M5p_QqryI0SQO3&export=download&authuser=0&confirm=t&uuid=eb6ffb6d-f7c2-44d8-ad77-db6c0aaf5cc7&at=APZUnTWr2VxrJPiiKVMdFd0IykrR:1705497846507" - ], - "dest": [ - "Set_Decimal_Separator_Dot_gold.xlsx", - "Set_Decimal_Separator_Dot_gold.csv" - ], - "multi": true - }, - "options": { - "as_shown": true - } - } -} diff --git a/experiment_a11y_tree.py b/experiment_a11y_tree.py index 86e6a72..f03d2a6 100644 --- a/experiment_a11y_tree.py +++ b/experiment_a11y_tree.py @@ -1,10 +1,11 @@ +import ctypes import datetime import json import logging import os import sys -import threading -import time +import func_timeout + from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent @@ -45,6 +46,7 @@ logger = logging.getLogger("desktopenv.experiment") PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx" + def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True): trajectory_recording_path = os.path.join(example_trajectory_dir, "trajectory.json") env = DesktopEnv( @@ -103,28 +105,10 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr except Exception as e: print(f"An error occurred while stopping the recording: {e}") - # Run the `record` function in a separate thread - recording_thread = threading.Thread(target=stop_recording()) - recording_thread.start() - - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 60 # seconds - start_time = time.time() - - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Stopping recording.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + try: + func_timeout.func_timeout(30, stop_recording) + except func_timeout.exceptions.FunctionTimedOut: + logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) @@ -150,7 +134,7 @@ def main(example_class, example_id): with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) - example["snapshot"] = "exp_chrome" + example["snapshot"] = "exp_v1" api_key = os.environ.get("OPENAI_API_KEY") agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'], max_tokens=1000, @@ -186,26 +170,75 @@ if __name__ == '__main__': "f3977615-2b45-4ac5-8bba-80c17dbe2a37", "215dfd39-f493-4bc3-a027-8a97d72c61bf" ] - for example_id in vlc_list: - recording_thread = threading.Thread(target=main, args=("vlc", example_id)) - recording_thread.start() - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 600 # seconds - start_time = time.time() + chrome_list = [ + # "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", + "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", + "06fe7178-4491-4589-810f-2e2bc9502122", + "e1e75309-3ddb-4d09-92ec-de869c928143", + "35253b65-1c19-4304-8aa4-6884b8218fc0", + "2ad9387a-65d8-4e33-ad5b-7580065a27ca", + "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263", + "44ee5668-ecd5-4366-a6ce-c1c9b8d4e938", + "2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3", + "480bcfea-d68f-4aaa-a0a9-2589ef319381", + "af630914-714e-4a24-a7bb-f9af687d3b91" + ] - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Kill this example.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + calc_list = [ + "eb03d19a-b88d-4de4-8a64-ca0ac66f426b", + "0bf05a7d-b28b-44d2-955a-50b41e24012a", + "7b802dad-6e0f-4204-9815-d4e3f57627d8", + "7a4e4bc8-922c-4c84-865c-25ba34136be1", + "2bd59342-0664-4ccb-ba87-79379096cc08", + "a9f325aa-8c05-4e4f-8341-9e4358565f4f", + "ecb0df7a-4e8d-4a03-b162-053391d3afaf", + "7efeb4b1-3d19-4762-b163-63328d66303b", + "4e6fcf72-daf3-439f-a232-c434ce416af6", + "6054afcb-5bab-4702-90a0-b259b5d3217c", + "abed40dc-063f-4598-8ba5-9fe749c0615d", + "01b269ae-2111-4a07-81fd-3fcd711993b0", + "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14", + "af2b02f7-acee-4be4-8b66-499fab394915", + "da1d63b8-fa12-417b-ba18-f748e5f770f3", + "636380ea-d5f6-4474-b6ca-b2ed578a20f1", + "5ba77536-05c5-4aae-a9ff-6e298d094c3e", + "4bc4eaf4-ca5e-4db2-8138-8d4e65af7c0b", + "672a1b02-c62f-4ae2-acf0-37f5fb3052b0", + "648fe544-16ba-44af-a587-12ccbe280ea6", + "8985d1e4-5b99-4711-add4-88949ebb2308", + "9e606842-2e27-43bf-b1d1-b43289c9589b", + "fcb6e45b-25c4-4087-9483-03d714f473a9", + "68c0c5b7-96f3-4e87-92a7-6c1b967fd2d2", + "fff629ea-046e-4793-8eec-1a5a15c3eb35", + "5c9a206c-bb00-4fb6-bb46-ee675c187df5", + "e975ae74-79bd-4672-8d1c-dc841a85781d", + "34a6938a-58da-4897-8639-9b90d6db5391", + "b5a22759-b4eb-4bf2-aeed-ad14e8615f19", + "2f9913a1-51ed-4db6-bfe0-7e1c95b3139e", + "2558031e-401d-4579-8e00-3ecf540fb492", + "39aa4e37-dc91-482e-99af-132a612d40f3", + "0cecd4f3-74de-457b-ba94-29ad6b5dafb6", + "4188d3a4-077d-46b7-9c86-23e1a036f6c1", + "51b11269-2ca8-4b2a-9163-f21758420e78", + "7e429b8d-a3f0-4ed0-9b58-08957d00b127", + "f5a90742-3fa2-40fc-a564-f29b054e0337", + "22df9241-f8d7-4509-b7f1-37e501a823f7", + "1434ca3e-f9e3-4db8-9ca7-b4c653be7d17", + "347ef137-7eeb-4c80-a3bb-0951f26a8aff", + "6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5", + "3aaa4e37-dc91-482e-99af-132a612d40f3", + "37608790-6147-45d0-9f20-1137bb35703d", + "f9584479-3d0d-4c79-affa-9ad7afdd8850", + "d681960f-7bc3-4286-9913-a8812ba3261a", + "21df9241-f8d7-4509-b7f1-37e501a823f7", + "f6a90742-3fa2-40fc-a564-f29b054e0337", + "1334ca3e-f9e3-4db8-9ca7-b4c653be7d17", + "357ef137-7eeb-4c80-a3bb-0951f26a8aff", + "aa3a8974-2e85-438b-b29e-a64df44deb4b", + "a01fbce3-2793-461f-ab86-43680ccbae25", + "4f07fbe9-70de-4927-a4d5-bb28bc12c52c", + ] + for example_id in calc_list: + main("libreoffice_calc", example_id) diff --git a/experiment_screenshot.py b/experiment_screenshot.py index 943a8ec..28b0f1e 100644 --- a/experiment_screenshot.py +++ b/experiment_screenshot.py @@ -3,11 +3,11 @@ import json import logging import os import sys -import threading import time - +import func_timeout from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent + # from mm_agents.gemini_pro_agent import GeminiPro_Agent # Logger Configs {{{ # @@ -77,7 +77,6 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr logger.info("Done: %s", done) logger.info("Info: %s", info) - # Save screenshot and trajectory information with open(os.path.join(example_trajectory_dir, f"step_{step_num}_{action_timestamp}.png"), "wb") as _f: with open(observation['screenshot'], "rb") as __f: @@ -106,28 +105,10 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr except Exception as e: print(f"An error occurred while stopping the recording: {e}") - # Run the `record` function in a separate thread - recording_thread = threading.Thread(target=stop_recording()) - recording_thread.start() - - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 60 # seconds - start_time = time.time() - - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Stopping recording.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + try: + func_timeout.func_timeout(30, stop_recording) + except func_timeout.exceptions.FunctionTimedOut: + logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) @@ -153,10 +134,11 @@ def main(example_class, example_id): with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) - example["snapshot"] = "exp_chrome" + example["snapshot"] = "exp_v1" api_key = os.environ.get("OPENAI_API_KEY") - agent = GPT4v_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space, exp="screenshot") + agent = GPT4v_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space, + exp="screenshot") # # api_key = os.environ.get("GENAI_API_KEY") # agent = GeminiPro_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space, exp="screenshot") @@ -172,7 +154,18 @@ def main(example_class, example_id): if __name__ == '__main__': - xx_list = [ + chrome_list = [ + # "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", + # "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", + # "06fe7178-4491-4589-810f-2e2bc9502122", + # "e1e75309-3ddb-4d09-92ec-de869c928143", + # "35253b65-1c19-4304-8aa4-6884b8218fc0", + # "2ad9387a-65d8-4e33-ad5b-7580065a27ca", + # "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263", + # "44ee5668-ecd5-4366-a6ce-c1c9b8d4e938", + # "2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3", + "480bcfea-d68f-4aaa-a0a9-2589ef319381", + "af630914-714e-4a24-a7bb-f9af687d3b91" ] - for example_id in xx_list: - main("xx", example_id) + for example_id in chrome_list: + main("chrome", example_id) diff --git a/experiment_screenshot_a11y_tree.py b/experiment_screenshot_a11y_tree.py index 4d3fe21..042bbfb 100644 --- a/experiment_screenshot_a11y_tree.py +++ b/experiment_screenshot_a11y_tree.py @@ -1,10 +1,10 @@ +import ctypes import datetime import json import logging import os import sys -import threading -import time +import func_timeout from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent @@ -44,9 +44,10 @@ logger.addHandler(sdebug_handler) logger = logging.getLogger("desktopenv.experiment") PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx" -#PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" +# PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" + def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True): trajectory_recording_path = os.path.join(example_trajectory_dir, "trajectory.json") env = DesktopEnv( @@ -105,28 +106,10 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr except Exception as e: print(f"An error occurred while stopping the recording: {e}") - # Run the `record` function in a separate thread - recording_thread = threading.Thread(target=stop_recording()) - recording_thread.start() - - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 60 # seconds - start_time = time.time() - - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Stopping recording.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + try: + func_timeout.func_timeout(30, stop_recording) + except func_timeout.exceptions.FunctionTimedOut: + logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) @@ -143,9 +126,9 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr def main(example_class, example_id): action_space = "pyautogui" - #example_class = "libreoffice_calc" - #example_id = "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3" - #example_id = "01b269ae-2111-4a07-81fd-3fcd711993b0" + # example_class = "libreoffice_calc" + # example_id = "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3" + # example_id = "01b269ae-2111-4a07-81fd-3fcd711993b0" gpt4_model = "gpt-4-vision-preview" gemini_model = "gemini-pro-vision" @@ -155,9 +138,9 @@ def main(example_class, example_id): with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) - example["snapshot"] = "exp_chrome" - #example["snapshot"] = "exp_setup4" - #example["snapshot"] = "Snapshot 30" + example["snapshot"] = "exp_v1" + # example["snapshot"] = "exp_setup4" + # example["snapshot"] = "Snapshot 30" api_key = os.environ.get("OPENAI_API_KEY") agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'], diff --git a/experiment_screenshot_seeact.py b/experiment_screenshot_seeact.py index 3f72375..541d549 100644 --- a/experiment_screenshot_seeact.py +++ b/experiment_screenshot_seeact.py @@ -1,10 +1,10 @@ +import ctypes import datetime import json import logging import os import sys -import threading -import time +import func_timeout from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent @@ -104,28 +104,10 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr except Exception as e: print(f"An error occurred while stopping the recording: {e}") - # Run the `record` function in a separate thread - recording_thread = threading.Thread(target=stop_recording()) - recording_thread.start() - - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 60 # seconds - start_time = time.time() - - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Stopping recording.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + try: + func_timeout.func_timeout(30, stop_recording) + except func_timeout.exceptions.FunctionTimedOut: + logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) @@ -147,7 +129,7 @@ def main(example_class, example_id): with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) - example["snapshot"] = "exp_chrome" + example["snapshot"] = "exp_v1" api_key = os.environ.get("OPENAI_API_KEY") agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'], diff --git a/experiment_screenshot_som.py b/experiment_screenshot_som.py index abd77d9..2ecdafe 100644 --- a/experiment_screenshot_som.py +++ b/experiment_screenshot_som.py @@ -1,10 +1,10 @@ +import ctypes import datetime import json import logging import os import sys -import threading -import time +import func_timeout from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent @@ -104,28 +104,10 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr except Exception as e: print(f"An error occurred while stopping the recording: {e}") - # Run the `record` function in a separate thread - recording_thread = threading.Thread(target=stop_recording()) - recording_thread.start() - - # Start a timer for your timeout length (in this case, 60 seconds) - timeout = 60 # seconds - start_time = time.time() - - # The main thread will wait for the set timeout period or until the recording is done - while recording_thread.is_alive(): - elapsed_time = time.time() - start_time - if elapsed_time >= timeout: - print("Timeout reached. Stopping recording.") - break - time.sleep(0.1) # Sleep for a short time to prevent this loop from using too much CPU - - # kill the recording thread if it is still alive - if recording_thread.is_alive(): - recording_thread.kill() - - # Wait for the recording thread to finish before exiting - recording_thread.join() + try: + func_timeout.func_timeout(30, stop_recording) + except func_timeout.exceptions.FunctionTimedOut: + logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) @@ -147,7 +129,7 @@ def main(example_class, example_id): with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) - example["snapshot"] = "exp_chrome" + example["snapshot"] = "exp_v1" api_key = os.environ.get("OPENAI_API_KEY") agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'],