lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Disable a11y tree temporarily

Browse Source
This commit is contained in:
Timothyxxx
2024-03-18 21:43:35 +08:00
parent 1c9c5fd2ad
commit f992d1f694
2 changed files with 22 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
lib_run_single.py
View File

@@ -2,7 +2,7 @@ import datetime
import json import json
import logging import logging
import os import os
import wandb # import wandb
from wrapt_timeout_decorator import * from wrapt_timeout_decorator import *
@@ -15,13 +15,13 @@ with open("./settings.json", "r") as file:
time_limit = data["time_limit"] time_limit = data["time_limit"]
@timeout(time_limit, use_signals=False) @timeout(time_limit, use_signals=False)
def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores, run): def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
agent.reset() agent.reset()
obs = env.reset(task_config=example) obs = env.reset(task_config=example)
done = False done = False
step_idx = 0 step_idx = 0
env.controller.start_recording() env.controller.start_recording()
str_table = wandb.Table(columns=["Screenshot", "A11T", "Modle Response", "Action", "Action timestamp", "Done"]) # str_table = wandb.Table(columns=["Screenshot", "A11T", "Modle Response", "Action", "Action timestamp", "Done"])
while not done and step_idx < max_steps: while not done and step_idx < max_steps:
response, actions = agent.predict( response, actions = agent.predict(
instruction, instruction,
@@ -43,10 +43,10 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
_f.write(screenshot) _f.write(screenshot)
# get a11tree and save to wandb # get a11tree and save to wandb
thisrun_a11tree = env.controller.get_accessibility_tree() thisrun_a11tree = env.controller.get_accessibility_tree()
str_table.add_data(wandb.Image(data_or_path=os.path.join(example_result_dir, f"step_{step_idx + 1}_{action_timestamp}.png"), caption=f"step_{step_idx + 1}_{action_timestamp}"), # str_table.add_data(wandb.Image(data_or_path=os.path.join(example_result_dir, f"step_{step_idx + 1}_{action_timestamp}.png"), caption=f"step_{step_idx + 1}_{action_timestamp}"),
thisrun_a11tree, # thisrun_a11tree,
response, action, action_timestamp, done) # response, action, action_timestamp, done)
run.log({"Reward": reward}) # run.log({"Reward": reward})
with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f: with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
f.write(json.dumps({ f.write(json.dumps({
"step_num": step_idx + 1, "step_num": step_idx + 1,
@@ -62,11 +62,11 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
logger.info("The episode is done.") logger.info("The episode is done.")
break break
step_idx += 1 step_idx += 1
run.log({"str_trajectory": str_table}) # run.log({"str_trajectory": str_table})
result = env.evaluate() result = env.evaluate()
logger.info("Result: %.2f", result) logger.info("Result: %.2f", result)
scores.append(result) scores.append(result)
with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f: with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
f.write(f"{result}\n") f.write(f"{result}\n")
env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4")) env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
run.log({"Result": result}) # run.log({"Result": result})

25
run.py
View File

@@ -8,7 +8,7 @@ import logging
import os import os
import random import random
import sys import sys
import wandb # import wandb
from tqdm import tqdm from tqdm import tqdm
@@ -52,7 +52,8 @@ logger = logging.getLogger("desktopenv.experiment")
# wandb config # wandb config
### set your wandb api key here ### set your wandb api key here
wandb.login(key=os.environ.get("WANDB_API_KEY", None)) # os.environ["WANDB_API_KEY"] = "48ec18fb4da7087238c6d6833eab9907565adbf3"
# wandb.login(key=os.environ.get("WANDB_API_KEY", None))
def config() -> argparse.Namespace: def config() -> argparse.Namespace:
@@ -147,8 +148,8 @@ def test(
for domain in tqdm(test_all_meta, desc="Domain"): for domain in tqdm(test_all_meta, desc="Domain"):
for example_id in tqdm(test_all_meta[domain], desc="Example", leave=False): for example_id in tqdm(test_all_meta[domain], desc="Example", leave=False):
run = wandb.init(project=f"OSworld-{args.action_space}-{args.observation_type}-{args.model}", group=f"{domain}", # run = wandb.init(project=f"OSworld-{args.action_space}-{args.observation_type}-{args.model}", group=f"{domain}",
name=f"{example_id}") # name=f"{example_id}")
# example setting # example setting
config_file = os.path.join(args.test_config_base_dir, f"examples/{domain}/{example_id}.json") config_file = os.path.join(args.test_config_base_dir, f"examples/{domain}/{example_id}.json")
with open(config_file, "r", encoding="utf-8") as f: with open(config_file, "r", encoding="utf-8") as f:
@@ -163,7 +164,7 @@ def test(
# wandb each example config settings # wandb each example config settings
cfg_args["instruction"] = instruction cfg_args["instruction"] = instruction
cfg_args["start_time"] = datetime.datetime.now().strftime("%Y:%m:%d-%H:%M:%S") cfg_args["start_time"] = datetime.datetime.now().strftime("%Y:%m:%d-%H:%M:%S")
run.config.update(cfg_args) # run.config.update(cfg_args)
example_result_dir = os.path.join( example_result_dir = os.path.join(
args.result_dir, args.result_dir,
@@ -177,10 +178,10 @@ def test(
# example start running # example start running
try: try:
lib_run_single.run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, lib_run_single.run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir,
scores, run) scores)
except Exception as e: except Exception as e:
logger.error(f"Exception in {domain}/{example_id}: {e}") logger.error(f"Exception in {domain}/{example_id}: {e}")
wandb.log({"Exception": wandb.Table(data=[[f"Exception in {domain}/{example_id}: {e}"]], columns=["Error"])}) # wandb.log({"Exception": wandb.Table(data=[[f"Exception in {domain}/{example_id}: {e}"]], columns=["Error"])})
env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4")) env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f: with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
f.write(json.dumps({ f.write(json.dumps({
@@ -188,11 +189,11 @@ def test(
})) }))
f.write("\n") f.write("\n")
# wandb settings # wandb settings
os.mkdir(os.path.join(wandb.run.dir, "results/")) # os.mkdir(os.path.join(wandb.run.dir, "results/"))
for file in os.listdir(example_result_dir): # for file in os.listdir(example_result_dir):
# move file to just under the root dir # # move file to just under the root dir
os.rename(os.path.join(example_result_dir, file), os.path.join(wandb.run.dir, f"./results/{file}")) # os.rename(os.path.join(example_result_dir, file), os.path.join(wandb.run.dir, f"./results/{file}"))
wandb.finish() # wandb.finish()
env.close() env.close()
logger.info(f"Average score: {sum(scores) / len(scores)}") logger.info(f"Average score: {sum(scores) / len(scores)}")