lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update new func

Browse Source
This commit is contained in:
Timothyxxx
2024-03-17 22:25:13 +08:00
parent 7feeab8f6b
commit e156a20e3d
2 changed files with 41 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
run.py
View File

@@ -6,6 +6,7 @@ import datetime
import json
import logging
import os
import random
import sys
from tqdm import tqdm
@@ -69,7 +70,7 @@ def config() -> argparse.Namespace:
"screenshot_a11y_tree",
"som"
],
default="som",
default="a11y_tree",
help="Observation type",
)
parser.add_argument("--screen_width", type=int, default=1920)
@@ -82,7 +83,7 @@ def config() -> argparse.Namespace:
parser.add_argument("--test_config_base_dir", type=str, default="evaluation_examples")
# lm config
parser.add_argument("--model", type=str, default="gpt-4-vision-preview")
parser.add_argument("--model", type=str, default="gpt-4-0125-preview")
parser.add_argument("--temperature", type=float, default=1.0)
parser.add_argument("--top_p", type=float, default=0.9)
parser.add_argument("--max_tokens", type=int, default=1500)
@@ -147,7 +148,7 @@ def test(
try:
lib_run_single.run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir,
scores)
except Exception as e:
except TimeoutError as e:
env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
logger.error(f"Time limit exceeded in {domain}/{example_id}")
with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
@@ -155,6 +156,14 @@ def test(
"Error": f"Time limit exceeded in {domain}/{example_id}"
}))
f.write("\n")
except Exception as e:
env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
logger.error(f"Exception in {domain}/{example_id}" + str(e))
with open(os.path.join(example_result_dir, "traj.jsonl"), "a") as f:
f.write(json.dumps({
"Error": f"Exception in {domain}/{example_id}" + str(e)
}))
f.write("\n")
env.close()
logger.info(f"Average score: {sum(scores) / len(scores)}")
@@ -193,15 +202,13 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
def get_result(action_space, use_model, observation_type, result_dir, total_file_json):
target_dir = os.path.join(result_dir, action_space, observation_type, use_model)
if not os.path.exists(target_dir):
print("New experiment, no result yet.")
return None
all_result = []
if not os.path.exists(target_dir):
return total_file_json
finished = {}
for domain in os.listdir(target_dir):
finished[domain] = []
domain_path = os.path.join(target_dir, domain)
if os.path.isdir(domain_path):
for example_id in os.listdir(domain_path):
@@ -209,10 +216,17 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file
if os.path.isdir(example_path):
if "result.txt" in os.listdir(example_path):
# empty all files under example_id
all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read()))
try:
all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read()))
except:
all_result.append(0.0)
print("Success Rate:", sum(all_result) / len(all_result) * 100, "%")
return all_result
if not all_result:
print("New experiment, no result yet.")
return None
else:
print("Current Success Rate:", sum(all_result) / len(all_result) * 100, "%")
return all_result
if __name__ == '__main__':
@@ -242,4 +256,8 @@ if __name__ == '__main__':
test_all_meta
)
# test(args, test_all_meta)
# make the order of key random in test_all_meta
for domain in test_all_meta:
random.shuffle(test_all_meta[domain])
test(args, test_all_meta)