362 lines
13 KiB
Python
362 lines
13 KiB
Python
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import func_timeout
|
|
|
|
from desktop_env.envs.desktop_env import DesktopEnv
|
|
from mm_agents.gpt_4v_agent import GPT4v_Agent
|
|
|
|
# Logger Configs {{{ #
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
datetime_str: str = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
|
|
|
|
file_handler = logging.FileHandler(os.path.join("logs", "normal-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
debug_handler = logging.FileHandler(os.path.join("logs", "debug-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
stdout_handler = logging.StreamHandler(sys.stdout)
|
|
sdebug_handler = logging.FileHandler(os.path.join("logs", "sdebug-{:}.log".format(datetime_str)), encoding="utf-8")
|
|
|
|
file_handler.setLevel(logging.INFO)
|
|
debug_handler.setLevel(logging.DEBUG)
|
|
stdout_handler.setLevel(logging.INFO)
|
|
sdebug_handler.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter(
|
|
fmt="\x1b[1;33m[%(asctime)s \x1b[31m%(levelname)s \x1b[32m%(module)s/%(lineno)d-%(processName)s\x1b[1;33m] \x1b[0m%(message)s")
|
|
file_handler.setFormatter(formatter)
|
|
debug_handler.setFormatter(formatter)
|
|
stdout_handler.setFormatter(formatter)
|
|
sdebug_handler.setFormatter(formatter)
|
|
|
|
stdout_handler.addFilter(logging.Filter("desktopenv"))
|
|
sdebug_handler.addFilter(logging.Filter("desktopenv"))
|
|
|
|
logger.addHandler(file_handler)
|
|
logger.addHandler(debug_handler)
|
|
logger.addHandler(stdout_handler)
|
|
logger.addHandler(sdebug_handler)
|
|
# }}} Logger Configs #
|
|
|
|
logger = logging.getLogger("desktopenv.experiment")
|
|
|
|
PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu2\Ubuntu2.vmx"
|
|
|
|
|
|
# PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
|
|
|
|
def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True):
|
|
trajectory_recording_path = os.path.join(example_trajectory_dir, "trajectory.json")
|
|
env = DesktopEnv(
|
|
path_to_vm=PATH_TO_VM,
|
|
action_space=agent.action_space,
|
|
task_config=example
|
|
)
|
|
# reset the environment to certain snapshot
|
|
observation = env.reset()
|
|
done = False
|
|
step_num = 0
|
|
|
|
if recording:
|
|
# send a request to the server to start recording
|
|
env.controller.start_recording()
|
|
|
|
while not done and step_num < max_steps:
|
|
actions = agent.predict(observation)
|
|
step_num += 1
|
|
for action in actions:
|
|
# Capture the timestamp before executing the action
|
|
action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
|
|
logger.info("Step %d: %s", step_num, action)
|
|
|
|
observation, reward, done, info = env.step(action)
|
|
|
|
logger.info("Reward: %.2f", reward)
|
|
logger.info("Done: %s", done)
|
|
logger.info("Info: %s", info)
|
|
|
|
# Save screenshot and trajectory information
|
|
with open(os.path.join(example_trajectory_dir, f"step_{step_num}_{action_timestamp}.png"), "wb") as _f:
|
|
with open(observation['screenshot'], "rb") as __f:
|
|
screenshot = __f.read()
|
|
_f.write(screenshot)
|
|
|
|
with open(trajectory_recording_path, "a") as f:
|
|
f.write(json.dumps({
|
|
"step_num": step_num,
|
|
"action_timestamp": action_timestamp,
|
|
"action": action,
|
|
"reward": reward,
|
|
"done": done,
|
|
"info": info,
|
|
"screenshot_file": f"step_{step_num}_{action_timestamp}.png"
|
|
}))
|
|
f.write("\n")
|
|
|
|
if done:
|
|
logger.info("The episode is done.")
|
|
break
|
|
|
|
def stop_recording():
|
|
try:
|
|
env.controller.end_recording(os.path.join(example_trajectory_dir, "recording.mp4"))
|
|
except Exception as e:
|
|
print(f"An error occurred while stopping the recording: {e}")
|
|
|
|
try:
|
|
func_timeout.func_timeout(30, stop_recording)
|
|
except func_timeout.exceptions.FunctionTimedOut:
|
|
logger.info("Recording timed out.")
|
|
|
|
result = env.evaluate()
|
|
logger.info("Result: %.2f", result)
|
|
|
|
with open(trajectory_recording_path, "a") as f:
|
|
f.write(json.dumps({
|
|
"result": result
|
|
}))
|
|
f.write("\n")
|
|
|
|
# env.close()
|
|
logger.info("Environment closed.")
|
|
|
|
|
|
def main(example_class, example_id, gpt4_model="gpt-4-vision-preview"):
|
|
action_space = "pyautogui"
|
|
# example_class = "libreoffice_calc"
|
|
# example_id = "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3"
|
|
# example_id = "01b269ae-2111-4a07-81fd-3fcd711993b0"
|
|
gemini_model = "gemini-pro-vision"
|
|
|
|
logger.info("Running example %s/%s", example_class, example_id)
|
|
logger.info("Using model %s", gpt4_model)
|
|
# logger.info("Using model %s", gemini_model)
|
|
|
|
with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r", encoding="utf-8") as f:
|
|
example = json.load(f)
|
|
example["snapshot"] = "exp_v5"
|
|
# example["snapshot"] = "exp_setup4"
|
|
# example["snapshot"] = "Snapshot 30"
|
|
|
|
api_key = os.environ.get("OPENAI_API_KEY")
|
|
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'],
|
|
action_space=action_space, exp="both")
|
|
|
|
# api_key = os.environ.get("GENAI_API_KEY")
|
|
# agent = GeminiPro_Agent(api_key=api_key, model=gemini_model, instruction=example['instruction'], action_space=action_space, exp="both")
|
|
|
|
root_trajectory_dir = "exp_trajectory"
|
|
|
|
example_trajectory_dir = os.path.join(root_trajectory_dir, "both", example_class, gpt4_model, example_id)
|
|
# example_trajectory_dir = os.path.join(root_trajectory_dir, "both", example_class, gemini_model, example_id)
|
|
|
|
os.makedirs(example_trajectory_dir, exist_ok=True)
|
|
|
|
run_one_example(example, agent, 15, example_trajectory_dir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
os_list = [
|
|
"94d95f96-9699-4208-98ba-3c3119edf9c2",
|
|
"bedcedc4-4d72-425e-ad62-21960b11fe0d",
|
|
"43c2d64c-bab5-4dcb-a30c-b888321c319a",
|
|
"7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82",
|
|
"ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3",
|
|
"f9be0997-4b7c-45c5-b05c-4612b44a6118",
|
|
"28cc3b7e-b194-4bc9-8353-d04c0f4d56d2",
|
|
"5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57",
|
|
"e0df059f-28a6-4169-924f-b9623e7184cc",
|
|
"ddc75b62-7311-4af8-bfb3-859558542b36",
|
|
"b6781586-6346-41cd-935a-a6b1487918fc",
|
|
"3ce045a0-877b-42aa-8d2c-b4a863336ab8",
|
|
"a4d98375-215b-4a4d-aee9-3d4370fccc41",
|
|
"13584542-872b-42d8-b299-866967b5c3ef",
|
|
"23393935-50c7-4a86-aeea-2b78fd089c5c"
|
|
]
|
|
|
|
# for example_id in os_list:
|
|
# try:
|
|
# main("os", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
calc_list = [
|
|
"a9f325aa-8c05-4e4f-8341-9e4358565f4f",
|
|
"ecb0df7a-4e8d-4a03-b162-053391d3afaf",
|
|
"7efeb4b1-3d19-4762-b163-63328d66303b",
|
|
"4e6fcf72-daf3-439f-a232-c434ce416af6",
|
|
"6054afcb-5bab-4702-90a0-b259b5d3217c",
|
|
"abed40dc-063f-4598-8ba5-9fe749c0615d",
|
|
"01b269ae-2111-4a07-81fd-3fcd711993b0",
|
|
"8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14",
|
|
"af2b02f7-acee-4be4-8b66-499fab394915",
|
|
"da1d63b8-fa12-417b-ba18-f748e5f770f3",
|
|
"636380ea-d5f6-4474-b6ca-b2ed578a20f1",
|
|
"5ba77536-05c5-4aae-a9ff-6e298d094c3e",
|
|
"4bc4eaf4-ca5e-4db2-8138-8d4e65af7c0b",
|
|
"672a1b02-c62f-4ae2-acf0-37f5fb3052b0",
|
|
"648fe544-16ba-44af-a587-12ccbe280ea6",
|
|
"8985d1e4-5b99-4711-add4-88949ebb2308",
|
|
"9e606842-2e27-43bf-b1d1-b43289c9589b",
|
|
"fcb6e45b-25c4-4087-9483-03d714f473a9",
|
|
"68c0c5b7-96f3-4e87-92a7-6c1b967fd2d2",
|
|
"fff629ea-046e-4793-8eec-1a5a15c3eb35",
|
|
"5c9a206c-bb00-4fb6-bb46-ee675c187df5",
|
|
"e975ae74-79bd-4672-8d1c-dc841a85781d",
|
|
"34a6938a-58da-4897-8639-9b90d6db5391",
|
|
"b5a22759-b4eb-4bf2-aeed-ad14e8615f19",
|
|
"2f9913a1-51ed-4db6-bfe0-7e1c95b3139e",
|
|
"2558031e-401d-4579-8e00-3ecf540fb492",
|
|
"0cecd4f3-74de-457b-ba94-29ad6b5dafb6",
|
|
"4188d3a4-077d-46b7-9c86-23e1a036f6c1",
|
|
"51b11269-2ca8-4b2a-9163-f21758420e78",
|
|
"7e429b8d-a3f0-4ed0-9b58-08957d00b127",
|
|
"347ef137-7eeb-4c80-a3bb-0951f26a8aff",
|
|
"6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5",
|
|
"3aaa4e37-dc91-482e-99af-132a612d40f3",
|
|
"37608790-6147-45d0-9f20-1137bb35703d",
|
|
"f9584479-3d0d-4c79-affa-9ad7afdd8850",
|
|
"d681960f-7bc3-4286-9913-a8812ba3261a",
|
|
"21df9241-f8d7-4509-b7f1-37e501a823f7",
|
|
"1334ca3e-f9e3-4db8-9ca7-b4c653be7d17",
|
|
"357ef137-7eeb-4c80-a3bb-0951f26a8aff",
|
|
"aa3a8974-2e85-438b-b29e-a64df44deb4b",
|
|
"a01fbce3-2793-461f-ab86-43680ccbae25",
|
|
"4f07fbe9-70de-4927-a4d5-bb28bc12c52c"
|
|
]
|
|
|
|
# for example_id in calc_list:
|
|
# try:
|
|
# main("libreoffice_calc", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
impress_list = [
|
|
"5d901039-a89c-4bfb-967b-bf66f4df075e",
|
|
"550ce7e7-747b-495f-b122-acdc4d0b8e54",
|
|
"455d3c66-7dc6-4537-a39a-36d3e9119df7",
|
|
"af23762e-2bfd-4a1d-aada-20fa8de9ce07",
|
|
"c59742c0-4323-4b9d-8a02-723c251deaa0",
|
|
"ef9d12bd-bcee-4ba0-a40e-918400f43ddf",
|
|
"9ec204e4-f0a3-42f8-8458-b772a6797cab",
|
|
"0f84bef9-9790-432e-92b7-eece357603fb",
|
|
"ce88f674-ab7a-43da-9201-468d38539e4a",
|
|
"3b27600c-3668-4abd-8f84-7bcdebbccbdb",
|
|
"a097acff-6266-4291-9fbd-137af7ecd439",
|
|
"bf4e9888-f10f-47af-8dba-76413038b73c",
|
|
"21760ecb-8f62-40d2-8d85-0cee5725cb72"
|
|
]
|
|
|
|
# for example_id in impress_list:
|
|
# try:
|
|
# main("libreoffice_impress", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
vs_code_list = [
|
|
"0ed39f63-6049-43d4-ba4d-5fa2fe04a951",
|
|
"53ad5833-3455-407b-bbc6-45b4c79ab8fb",
|
|
"eabc805a-bfcf-4460-b250-ac92135819f6",
|
|
"982d12a5-beab-424f-8d38-d2a48429e511",
|
|
"4e60007a-f5be-4bfc-9723-c39affa0a6d3",
|
|
"e2b5e914-ffe1-44d2-8e92-58f8c5d92bb2",
|
|
"9439a27b-18ae-42d8-9778-5f68f891805e",
|
|
"ea98c5d7-3cf9-4f9b-8ad3-366b58e0fcae",
|
|
"930fdb3b-11a8-46fe-9bac-577332e2640e",
|
|
"276cc624-87ea-4f08-ab93-f770e3790175",
|
|
"9d425400-e9b2-4424-9a4b-d4c7abac4140"
|
|
]
|
|
|
|
# for example_id in vs_code_list:
|
|
# try:
|
|
# main("vs_code", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
multiple_list = [
|
|
"f8cfa149-d1c1-4215-8dac-4a0932bad3c2",
|
|
"897e3b53-5d4d-444b-85cb-2cdc8a97d903",
|
|
"4e9f0faf-2ecc-4ae8-a804-28c9a75d1ddc",
|
|
"b52b40a5-ad70-4c53-b5b0-5650a8387052",
|
|
"46407397-a7d5-4c6b-92c6-dbe038b1457b",
|
|
"2b9493d7-49b8-493a-a71b-56cd1f4d6908",
|
|
"51f5801c-18b3-4f25-b0c3-02f85507a078",
|
|
"2c9fc0de-3ee7-45e1-a5df-c86206ad78b5",
|
|
"510f64c8-9bcc-4be1-8d30-638705850618",
|
|
"937087b6-f668-4ba6-9110-60682ee33441",
|
|
"ee9a3c83-f437-4879-8918-be5efbb9fac7",
|
|
"3680a5ee-6870-426a-a997-eba929a0d25c",
|
|
"e135df7c-7687-4ac0-a5f0-76b74438b53e",
|
|
"58565672-7bfe-48ab-b828-db349231de6b",
|
|
"2fe4b718-3bd7-46ec-bdce-b184f5653624"
|
|
]
|
|
|
|
# for example_id in multiple_list:
|
|
# try:
|
|
# main("multi_apps", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
chrome_list = [
|
|
# "bb5e4c0d-f964-439c-97b6-bdb9747de3f4",
|
|
"7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3",
|
|
"06fe7178-4491-4589-810f-2e2bc9502122",
|
|
"e1e75309-3ddb-4d09-92ec-de869c928143",
|
|
"35253b65-1c19-4304-8aa4-6884b8218fc0",
|
|
"2ad9387a-65d8-4e33-ad5b-7580065a27ca",
|
|
"7a5a7856-f1b6-42a4-ade9-1ca81ca0f263",
|
|
"44ee5668-ecd5-4366-a6ce-c1c9b8d4e938",
|
|
"2ae9ba84-3a0d-4d4c-8338-3a1478dc5fe3",
|
|
"480bcfea-d68f-4aaa-a0a9-2589ef319381",
|
|
"af630914-714e-4a24-a7bb-f9af687d3b91"
|
|
]
|
|
|
|
# for example_id in chrome_list:
|
|
# try:
|
|
# main("chrome", example_id)
|
|
# except Exception as e:
|
|
# logger.error("An error occurred while running the example: %s", e)
|
|
# continue
|
|
|
|
|
|
writer_list = [
|
|
"6ada715d-3aae-4a32-a6a7-429b2e43fb93",
|
|
"ecc2413d-8a48-416e-a3a2-d30106ca36cb",
|
|
"0e47de2a-32e0-456c-a366-8c607ef7a9d2",
|
|
"4bcb1253-a636-4df4-8cb0-a35c04dfef31",
|
|
"0810415c-bde4-4443-9047-d5f70165a697",
|
|
"e528b65e-1107-4b8c-8988-490e4fece599",
|
|
"66399b0d-8fda-4618-95c4-bfc6191617e9",
|
|
"936321ce-5236-426a-9a20-e0e3c5dc536f",
|
|
"3ef2b351-8a84-4ff2-8724-d86eae9b842e",
|
|
"0b17a146-2934-46c7-8727-73ff6b6483e8",
|
|
"0e763496-b6bb-4508-a427-fad0b6c3e195",
|
|
"f178a4a9-d090-4b56-bc4c-4b72a61a035d",
|
|
"adf5e2c3-64c7-4644-b7b6-d2f0167927e7",
|
|
"0a0faba3-5580-44df-965d-f562a99b291c",
|
|
"e246f6d8-78d7-44ac-b668-fcf47946cb50",
|
|
"8472fece-c7dd-4241-8d65-9b3cd1a0b568",
|
|
"88fe4b2d-3040-4c70-9a70-546a47764b48",
|
|
"d53ff5ee-3b1a-431e-b2be-30ed2673079b",
|
|
"72b810ef-4156-4d09-8f08-a0cf57e7cefe",
|
|
"6f81754e-285d-4ce0-b59e-af7edb02d108",
|
|
"b21acd93-60fd-4127-8a43-2f5178f4a830"
|
|
]
|
|
|
|
for example_id in writer_list:
|
|
try:
|
|
main("libreoffice_writer", example_id)
|
|
except Exception as e:
|
|
logger.error("An error occurred while running the example: %s", e)
|
|
continue
|
|
|
|
|