lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update unfinished function and error catching

Browse Source
This commit is contained in:
Timothyxxx
2024-03-15 23:12:18 +08:00
parent 57f2257254
commit 99e86a2cd4
1 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
run.py
View File

@@ -6,9 +6,10 @@ import datetime
import json import json
import logging import logging
import os import os
import signal
import sys import sys
from tqdm import tqdm
from desktop_env.envs.desktop_env import DesktopEnv from desktop_env.envs.desktop_env import DesktopEnv
from mm_agents.agent import PromptAgent from mm_agents.agent import PromptAgent
@@ -52,7 +53,8 @@ def handler(signo, frame):
raise RuntimeError("Time limit exceeded!") raise RuntimeError("Time limit exceeded!")
signal.signal(signal.SIGALRM, handler) # fixme: windows doesn't support signal
# signal.signal(signal.SIGALRM, handler)
def config() -> argparse.Namespace: def config() -> argparse.Namespace:
@@ -128,8 +130,8 @@ def test(
headless=args.headless, headless=args.headless,
) )
for domain in test_all_meta: for domain in tqdm(test_all_meta, desc="Domain"):
for example_id in test_all_meta[domain]: for example_id in tqdm(test_all_meta[domain], desc="Example", leave=False):
# example setting # example setting
config_file = os.path.join(args.test_config_base_dir, f"examples/{domain}/{example_id}.json") config_file = os.path.join(args.test_config_base_dir, f"examples/{domain}/{example_id}.json")
with open(config_file, "r", encoding="utf-8") as f: with open(config_file, "r", encoding="utf-8") as f:
@@ -154,7 +156,7 @@ def test(
# example start running # example start running
try: try:
signal.alarm(time_limit) # signal.alarm(time_limit) fixme: windows doesn't support signal
agent.reset() agent.reset()
obs = env.reset(task_config=example) obs = env.reset(task_config=example)
done = False done = False
@@ -204,6 +206,8 @@ def test(
result = env.evaluate() result = env.evaluate()
logger.info("Result: %.2f", result) logger.info("Result: %.2f", result)
scores.append(result) scores.append(result)
with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
f.write(f"{result}\n")
env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4")) env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
except RuntimeError as e: except RuntimeError as e:
logger.error(f"Error in example {domain}/{example_id}: {e}") logger.error(f"Error in example {domain}/{example_id}: {e}")
@@ -224,6 +228,10 @@ def test(
})) }))
f.write("\n") f.write("\n")
continue continue
except Exception as e:
logger.error(f"Error in example {domain}/{example_id}: {e}")
continue
env.close() env.close()
logger.info(f"Average score: {sum(scores) / len(scores)}") logger.info(f"Average score: {sum(scores) / len(scores)}")
@@ -236,9 +244,13 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
finished = {} finished = {}
for domain in os.listdir(target_dir): for domain in os.listdir(target_dir):
finished[domain] = []
domain_path = os.path.join(target_dir, domain) domain_path = os.path.join(target_dir, domain)
if os.path.isdir(domain_path): if os.path.isdir(domain_path):
finished[domain] = os.listdir(domain_path) for example_id in os.listdir(domain_path):
example_path = os.path.join(domain_path, example_id)
if os.path.isdir(example_path) and "result.txt" in os.listdir(example_path):
finished[domain].append(example_id)
if not finished: if not finished:
return total_file_json return total_file_json