#import ctypes import datetime import json import logging import os import sys import func_timeout from desktop_env.envs.desktop_env import DesktopEnv from mm_agents.gpt_4v_agent import GPT4v_Agent # Logger Configs {{{ # logger = logging.getLogger() logger.setLevel(logging.DEBUG) datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S") file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)), encoding="utf-8") debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)), encoding="utf-8") stdout_handler = logging.StreamHandler(sys.stdout) sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)), encoding="utf-8") file_handler.setLevel(logging.INFO) debug_handler.setLevel(logging.DEBUG) stdout_handler.setLevel(logging.INFO) sdebug_handler.setLevel(logging.DEBUG) formatter = logging.Formatter( fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s") file_handler.setFormatter(formatter) debug_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter) sdebug_handler.setFormatter(formatter) stdout_handler.addFilter(logging.Filter("desktopenv")) sdebug_handler.addFilter(logging.Filter("desktopenv")) logger.addHandler(file_handler) logger.addHandler(debug_handler) logger.addHandler(stdout_handler) logger.addHandler(sdebug_handler) # }}} Logger Configs # logger = logging.getLogger("desktopenv.experiment") PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx" def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True): trajectory_recording_path = os.path.join(example_trajectory_dir, "trajectory.json") env = DesktopEnv( path_to_vm=PATH_TO_VM, action_space=agent.action_space, task_config=example ) # reset the environment to certain snapshot observation = env.reset() done = False step_num = 0 if recording: # send a request to the server to start recording env.controller.start_recording() while not done and step_num < max_steps: actions = agent.predict(observation) step_num += 1 for action in actions: # Capture the timestamp before executing the action action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S") logger.info("Step %d: %s", step_num, action) observation, reward, done, info = env.step(action) logger.info("Reward: %.2f", reward) logger.info("Done: %s", done) logger.info("Info: %s", info) # Save screenshot and trajectory information with open(os.path.join(example_trajectory_dir, f"step_{step_num}_{action_timestamp}.png"), "wb") as _f: with open(observation['screenshot'], "rb") as __f: screenshot = __f.read() _f.write(screenshot) with open(trajectory_recording_path, "a") as f: f.write(json.dumps({ "step_num": step_num, "action_timestamp": action_timestamp, "action": action, "reward": reward, "done": done, "info": info, "screenshot_file": f"step_{step_num}_{action_timestamp}.png" })) f.write("\n") if done: logger.info("The episode is done.") break def stop_recording(): try: env.controller.end_recording(os.path.join(example_trajectory_dir, "recording.mp4")) except Exception as e: print(f"An error occurred while stopping the recording: {e}") try: func_timeout.func_timeout(30, stop_recording) except func_timeout.exceptions.FunctionTimedOut: logger.info("Recording timed out.") result = env.evaluate() logger.info("Result: %.2f", result) with open(trajectory_recording_path, "a") as f: f.write(json.dumps({ "result": result })) f.write("\n") # env.close() logger.info("Environment closed.") def main(example_class, example_id): action_space = "pyautogui" gpt4_model = "gpt-4-vision-preview" gemini_model = "gemini-pro-vision" with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f: example = json.load(f) example["snapshot"] = "exp_v5" logger.info("TASK: %s/%s", example_class, example_id) api_key = os.environ.get("OPENAI_API_KEY") agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, max_tokens=1000, instruction=example['instruction'], action_space=action_space, exp="som") # api_key = os.environ.get("GENAI_API_KEY") # agent = GeminiPro_Agent(api_key=api_key, model=gemini_model, instruction=example['instruction'], action_space=action_space) root_trajectory_dir = "exp_trajectory" example_trajectory_dir = os.path.join(root_trajectory_dir, "som", example_class, gpt4_model, example_id) # example_trajectory_dir = os.path.join(root_trajectory_dir, "som", example_class, gemini_model, example_id) os.makedirs(example_trajectory_dir, exist_ok=True) run_one_example(example, agent, 15, example_trajectory_dir) if __name__ == '__main__': from tqdm import tqdm # impress_list = [ # # "5d901039-a89c-4bfb-967b-bf66f4df075e", # "550ce7e7-747b-495f-b122-acdc4d0b8e54", # "455d3c66-7dc6-4537-a39a-36d3e9119df7", # "af23762e-2bfd-4a1d-aada-20fa8de9ce07", # "c59742c0-4323-4b9d-8a02-723c251deaa0", # "ef9d12bd-bcee-4ba0-a40e-918400f43ddf", # "9ec204e4-f0a3-42f8-8458-b772a6797cab", # "0f84bef9-9790-432e-92b7-eece357603fb", # "ce88f674-ab7a-43da-9201-468d38539e4a", # "3b27600c-3668-4abd-8f84-7bcdebbccbdb", # "a097acff-6266-4291-9fbd-137af7ecd439", # "bf4e9888-f10f-47af-8dba-76413038b73c", # "21760ecb-8f62-40d2-8d85-0cee5725cb72" # ] # for example_id in impress_list: # main("libreoffice_impress", example_id) vlc_list = [ "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", "8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89", "8f080098-ddb1-424c-b438-4e96e5e4786e", "bba3381f-b5eb-4439-bd9e-80c22218d5a7", "fba2c100-79e8-42df-ae74-b592418d54f4", "efcf0d81-0835-4880-b2fd-d866e8bc2294", "8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f", "aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6", "386dbd0e-0241-4a0a-b6a2-6704fba26b1c", "9195653c-f4aa-453d-aa95-787f6ccfaae9", "d06f0d4d-2cd5-4ede-8de9-598629438c6e", "a5bbbcd5-b398-4c91-83d4-55e1e31bbb81", "f3977615-2b45-4ac5-8bba-80c17dbe2a37", "215dfd39-f493-4bc3-a027-8a97d72c61bf" ] # for example_id in tqdm(vlc_list): # try: # main("vlc", example_id) # except Exception as e: # print(f"An error occurred while running the example: {e}") # continue chrome_list = [ "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", "06fe7178-4491-4589-810f-2e2bc9502122", "e1e75309-3ddb-4d09-92ec-de869c928143", "35253b65-1c19-4304-8aa4-6884b8218fc0", "2ad9387a-65d8-4e33-ad5b-7580065a27ca", "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263", "44ee5668-ecd5-4366-a6ce-c1c9b8d4e938", "2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3", "480bcfea-d68f-4aaa-a0a9-2589ef319381", "af630914-714e-4a24-a7bb-f9af687d3b91" ] for example_id in tqdm(chrome_list): try: main("chrome", example_id) except Exception as e: print(f"An error occurred while running the example: {e}") continue vs_code_list = [ "0ed39f63-6049-43d4-ba4d-5fa2fe04a951", "53ad5833-3455-407b-bbc6-45b4c79ab8fb", "eabc805a-bfcf-4460-b250-ac92135819f6", "982d12a5-beab-424f-8d38-d2a48429e511", "4e60007a-f5be-4bfc-9723-c39affa0a6d3", "e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2", "9439a27b-18ae-42d8-9778-5f68f891805e", "ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae", "930fdb3b-11a8-46fe-9bac-577332e2640e", "276cc624-87ea-4f08-ab93-f770e3790175", "9d425400-e9b2-4424-9a4b-d4c7abac4140" ] for example_id in tqdm(vs_code_list): try: main("vs_code", example_id) except Exception as e: print(f"An error occurred while running the example: {e}") continue thunderbird_list = [ "bb5e4c0d-f964-439c-97b6-bdb9747de3f4", "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3", "12086550-11c0-466b-b367-1d9e75b3910e", "06fe7178-4491-4589-810f-2e2bc9502122", "6766f2b8-8a72-417f-a9e5-56fcaa735837", "e1e75309-3ddb-4d09-92ec-de869c928143", "3d1682a7-0fb0-49ae-a4dc-a73afd2d06d5", "35253b65-1c19-4304-8aa4-6884b8218fc0", "d088f539-cab4-4f9a-ac92-9999fc3a656e", "2ad9387a-65d8-4e33-ad5b-7580065a27ca", "480bcfea-d68f-4aaa-a0a9-2589ef319381", "030eeff7-b492-4218-b312-701ec99ee0cc", "94760984-3ff5-41ee-8347-cf1af709fea0", "99146c54-4f37-4ab8-9327-5f3291665e1e", "c9e7eaf2-b1a1-4efc-a982-721972fa9f02" ] for example_id in tqdm(thunderbird_list): try: main("thunderbird", example_id) except Exception as e: print(f"An error occurred while running the example: {e}") continue