204 lines
7.9 KiB
Python
204 lines
7.9 KiB
Python
#import ctypes
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import func_timeout
|
|
|
|
from desktop_env.envs.desktop_env import DesktopEnv
|
|
from mm_agents.gpt_4v_agent import GPT4v_Agent
|
|
|
|
# Logger Configs {{{ #
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
|
|
|
|
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
stdout_handler = logging.StreamHandler(sys.stdout)
|
|
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
|
|
file_handler.setLevel(logging.INFO)
|
|
debug_handler.setLevel(logging.DEBUG)
|
|
stdout_handler.setLevel(logging.INFO)
|
|
sdebug_handler.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter(
|
|
fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
|
|
file_handler.setFormatter(formatter)
|
|
debug_handler.setFormatter(formatter)
|
|
stdout_handler.setFormatter(formatter)
|
|
sdebug_handler.setFormatter(formatter)
|
|
|
|
stdout_handler.addFilter(logging.Filter("desktopenv"))
|
|
sdebug_handler.addFilter(logging.Filter("desktopenv"))
|
|
|
|
logger.addHandler(file_handler)
|
|
logger.addHandler(debug_handler)
|
|
logger.addHandler(stdout_handler)
|
|
logger.addHandler(sdebug_handler)
|
|
# }}} Logger Configs #
|
|
|
|
logger = logging.getLogger("desktopenv.experiment")
|
|
|
|
#PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
|
|
|
|
|
|
PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
|
|
|
|
def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True):
|
|
trajectory_recording_path = os.path.join(example_trajectory_dir, "trajectory.json")
|
|
env = DesktopEnv(
|
|
path_to_vm=PATH_TO_VM,
|
|
action_space=agent.action_space,
|
|
task_config=example
|
|
)
|
|
# reset the environment to certain snapshot
|
|
observation = env.reset()
|
|
done = False
|
|
step_num = 0
|
|
|
|
if recording:
|
|
# send a request to the server to start recording
|
|
env.controller.start_recording()
|
|
|
|
while not done and step_num < max_steps:
|
|
actions = agent.predict(observation)
|
|
step_num += 1
|
|
for action in actions:
|
|
# Capture the timestamp before executing the action
|
|
action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
|
|
logger.info("Step %d: %s", step_num, action)
|
|
|
|
observation, reward, done, info = env.step(action)
|
|
|
|
logger.info("Reward: %.2f", reward)
|
|
logger.info("Done: %s", done)
|
|
logger.info("Info: %s", info)
|
|
|
|
# Save screenshot and trajectory information
|
|
with open(os.path.join(example_trajectory_dir, f"step_{step_num}_{action_timestamp}.png"), "wb") as _f:
|
|
with open(observation['screenshot'], "rb") as __f:
|
|
screenshot = __f.read()
|
|
_f.write(screenshot)
|
|
|
|
with open(trajectory_recording_path, "a") as f:
|
|
f.write(json.dumps({
|
|
"step_num": step_num,
|
|
"action_timestamp": action_timestamp,
|
|
"action": action,
|
|
"reward": reward,
|
|
"done": done,
|
|
"info": info,
|
|
"screenshot_file": f"step_{step_num}_{action_timestamp}.png"
|
|
}))
|
|
f.write("\n")
|
|
|
|
if done:
|
|
logger.info("The episode is done.")
|
|
break
|
|
|
|
def stop_recording():
|
|
try:
|
|
env.controller.end_recording(os.path.join(example_trajectory_dir, "recording.mp4"))
|
|
except Exception as e:
|
|
print(f"An error occurred while stopping the recording: {e}")
|
|
|
|
try:
|
|
func_timeout.func_timeout(30, stop_recording)
|
|
except func_timeout.exceptions.FunctionTimedOut:
|
|
logger.info("Recording timed out.")
|
|
|
|
result = env.evaluate()
|
|
logger.info("Result: %.2f", result)
|
|
|
|
with open(trajectory_recording_path, "a") as f:
|
|
f.write(json.dumps({
|
|
"result": result
|
|
}))
|
|
f.write("\n")
|
|
|
|
# env.close()
|
|
logger.info("Environment closed.")
|
|
|
|
|
|
def main(example_class, example_id):
|
|
action_space = "pyautogui"
|
|
gpt4_model = "gpt-4-vision-preview"
|
|
#gemini_model = "gemini-pro-vision"
|
|
|
|
with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f:
|
|
example = json.load(f)
|
|
#example["snapshot"] = "exp_v1"
|
|
# example["snapshot"] = "exp_setup4"
|
|
example["snapshot"] = "Snapshot 30"
|
|
|
|
logger.info("TASK: %s/%s", example_class, example_id)
|
|
|
|
api_key = os.environ.get("OPENAI_API_KEY")
|
|
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, max_tokens=1000, instruction=example['instruction'],
|
|
action_space=action_space, exp="som")
|
|
|
|
# api_key = os.environ.get("GENAI_API_KEY")
|
|
# agent = GeminiPro_Agent(api_key=api_key, model=gemini_model, instruction=example['instruction'], action_space=action_space)
|
|
|
|
root_trajectory_dir = "exp_trajectory"
|
|
|
|
example_trajectory_dir = os.path.join(root_trajectory_dir, "som", example_class, gpt4_model, example_id)
|
|
# example_trajectory_dir = os.path.join(root_trajectory_dir, "som", example_class, gemini_model, example_id)
|
|
|
|
os.makedirs(example_trajectory_dir, exist_ok=True)
|
|
|
|
run_one_example(example, agent, 15, example_trajectory_dir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
xx_list = [ "94d95f96-9699-4208-98ba-3c3119edf9c2"
|
|
, "bedcedc4-4d72-425e-ad62-21960b11fe0d"
|
|
, "43c2d64c-bab5-4dcb-a30c-b888321c319a"
|
|
, "7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82"
|
|
, "ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3"
|
|
, "f9be0997-4b7c-45c5-b05c-4612b44a6118"
|
|
, "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2"
|
|
, "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57"
|
|
, "e0df059f-28a6-4169-924f-b9623e7184cc"
|
|
, "ddc75b62-7311-4af8-bfb3-859558542b36"
|
|
, "b6781586-6346-41cd-935a-a6b1487918fc"
|
|
, "3ce045a0-877b-42aa-8d2c-b4a863336ab8"
|
|
, "a4d98375-215b-4a4d-aee9-3d4370fccc41"
|
|
, "13584542-872b-42d8-b299-866967b5c3ef"
|
|
, "23393935-50c7-4a86-aeea-2b78fd089c5c"
|
|
# 15, ^ os, v calc
|
|
, "eb03d19a-b88d-4de4-8a64-ca0ac66f426b"
|
|
, "0bf05a7d-b28b-44d2-955a-50b41e24012a"
|
|
, "7a4e4bc8-922c-4c84-865c-25ba34136be1"
|
|
, "a9f325aa-8c05-4e4f-8341-9e4358565f4f"
|
|
, "ecb0df7a-4e8d-4a03-b162-053391d3afaf"
|
|
, "7efeb4b1-3d19-4762-b163-63328d66303b"
|
|
, "4e6fcf72-daf3-439f-a232-c434ce416af6"
|
|
, "6054afcb-5bab-4702-90a0-b259b5d3217c"
|
|
, "abed40dc-063f-4598-8ba5-9fe749c0615d"
|
|
, "01b269ae-2111-4a07-81fd-3fcd711993b0"
|
|
, "8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14"
|
|
, "0cecd4f3-74de-457b-ba94-29ad6b5dafb6"
|
|
, "4188d3a4-077d-46b7-9c86-23e1a036f6c1"
|
|
, "51b11269-2ca8-4b2a-9163-f21758420e78"
|
|
, "7e429b8d-a3f0-4ed0-9b58-08957d00b127"
|
|
, "347ef137-7eeb-4c80-a3bb-0951f26a8aff"
|
|
, "6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5"
|
|
, "3aaa4e37-dc91-482e-99af-132a612d40f3"
|
|
, "37608790-6147-45d0-9f20-1137bb35703d"
|
|
, "f9584479-3d0d-4c79-affa-9ad7afdd8850"
|
|
, "d681960f-7bc3-4286-9913-a8812ba3261a"
|
|
, "21df9241-f8d7-4509-b7f1-37e501a823f7"
|
|
, "1334ca3e-f9e3-4db8-9ca7-b4c653be7d17"
|
|
, "357ef137-7eeb-4c80-a3bb-0951f26a8aff"
|
|
, "aa3a8974-2e85-438b-b29e-a64df44deb4b"
|
|
, "a01fbce3-2793-461f-ab86-43680ccbae25"
|
|
, "4f07fbe9-70de-4927-a4d5-bb28bc12c52c"
|
|
]
|
|
for example_id in xx_list[18:]:
|
|
main("libreoffice_calc", example_id)
|