Fix small bugs in max time limit setting

lib_run_single.py

@@ -1,8 +1,10 @@
-import os
 import datetime
 import json
 import logging
+import os
+
 from wrapt_timeout_decorator import *
+
 logger = logging.getLogger("desktopenv.experiment")
 
 # Open the JSON file
@@ -11,6 +13,7 @@ with open("./settings.json", "r") as file:
     data = json.load(file)
 time_limit = data["time_limit"]
 
+
 @timeout(time_limit, use_signals=False)
 def run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores):
     agent.reset()
@@ -37,7 +40,7 @@ def run_single_example(agent, env, example, max_steps, instruction, args, exampl
 
             # Save screenshot and trajectory information
             with open(os.path.join(example_result_dir, f"step_{step_idx + 1}_{action_timestamp}.png"),
-                        "wb") as _f:
+                      "wb") as _f:
                 with open(obs['screenshot'], "rb") as __f:
                     screenshot = __f.read()
                 _f.write(screenshot)

run.py

@@ -9,11 +9,11 @@ import os
 import sys
 
 from tqdm import tqdm
-import time
 
+import lib_run_single
 from desktop_env.envs.desktop_env import DesktopEnv
 from mm_agents.agent import PromptAgent
-import lib_run_single
+
 #  Logger Configs {{{ #
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
@@ -48,6 +48,7 @@ logger.addHandler(sdebug_handler)
 
 logger = logging.getLogger("desktopenv.experiment")
 
+
 def config() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Run end-to-end evaluation on the benchmark"
@@ -79,7 +80,6 @@ def config() -> argparse.Namespace:
     # agent config
     parser.add_argument("--max_trajectory_length", type=int, default=3)
     parser.add_argument("--test_config_base_dir", type=str, default="evaluation_examples")
-    parser.add_argument("--example_time_limit", type=int, default=1200)
 
     # lm config
     parser.add_argument("--model", type=str, default="gpt-4-vision-preview")
@@ -101,7 +101,6 @@ def test(
 ) -> None:
     scores = []
     max_steps = args.max_steps
-    time_limit = args.example_time_limit
 
     # log args
     logger.info("Args: %s", args)
@@ -146,7 +145,8 @@ def test(
             os.makedirs(example_result_dir, exist_ok=True)
             # example start running
             try:
-                lib_run_single.run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir, scores)
+                lib_run_single.run_single_example(agent, env, example, max_steps, instruction, args, example_result_dir,
+                                                  scores)
             except Exception as e:
                 env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
                 logger.error(f"Time limit exceeded in {domain}/{example_id}")
@@ -168,11 +168,10 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
 
     finished = {}
     for domain in os.listdir(target_dir):
-        finished[domain] = []
         finished[domain] = []
         domain_path = os.path.join(target_dir, domain)
         if os.path.isdir(domain_path):
-           for example_id in os.listdir(domain_path):
+            for example_id in os.listdir(domain_path):
                 example_path = os.path.join(domain_path, example_id)
                 if os.path.isdir(example_path):
                     if "result.txt" not in os.listdir(example_path):
@@ -192,6 +191,30 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
     return total_file_json
 
 
+def get_result(action_space, use_model, observation_type, result_dir, total_file_json):
+    target_dir = os.path.join(result_dir, action_space, observation_type, use_model)
+
+    all_result = []
+
+    if not os.path.exists(target_dir):
+        return total_file_json
+
+    finished = {}
+    for domain in os.listdir(target_dir):
+        finished[domain] = []
+        domain_path = os.path.join(target_dir, domain)
+        if os.path.isdir(domain_path):
+            for example_id in os.listdir(domain_path):
+                example_path = os.path.join(domain_path, example_id)
+                if os.path.isdir(example_path):
+                    if "result.txt" in os.listdir(example_path):
+                        # empty all files under example_id
+                        all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read()))
+
+    print("Success Rate:", sum(all_result) / len(all_result) * 100, "%")
+    return all_result
+
+
 if __name__ == '__main__':
     ####### The complete version of the list of examples #######
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -211,4 +234,12 @@ if __name__ == '__main__':
     for domain in test_file_list:
         left_info += f"{domain}: {len(test_file_list[domain])}\n"
     logger.info(f"Left tasks:\n{left_info}")
-    test(args, test_all_meta)
+
+    get_result(args.action_space,
+        args.model,
+        args.observation_type,
+        args.result_dir,
+        test_all_meta
+    )
+
+    # test(args, test_all_meta)

settings.json

@@ -1,3 +1,3 @@
 {
-    "time_limit": "60"
+    "time_limit": "1200"
 }