update timer

run.py

@@ -7,7 +7,8 @@ import json
 import logging
 import os
 import sys
-# import signal
+
+from tqdm # import tqdm
 import time
 import timeout_decorator
 
@@ -48,6 +49,7 @@ logger.addHandler(sdebug_handler)
 
 logger = logging.getLogger("desktopenv.experiment")
 
+
 # make sure each example won't exceed the time limit
 # def handler(signo, frame):
 #     raise RuntimeError("Time limit exceeded!")
@@ -73,7 +75,7 @@ def config() -> argparse.Namespace:
             "screenshot_a11y_tree",
             "som"
         ],
-        default="a11y_tree",
+        default="som",
         help="Observation type",
     )
     parser.add_argument("--screen_width", type=int, default=1920)
@@ -126,8 +128,8 @@ def test(
         headless=args.headless,
     )
 
-    for domain in test_all_meta:
-        for example_id in test_all_meta[domain]:
+    for domain in tqdm(test_all_meta, desc="Domain"):
+        for example_id in tqdm(test_all_meta[domain], desc="Example", leave=False):
             # example setting
             config_file = os.path.join(args.test_config_base_dir, f"examples/{domain}/{example_id}.json")
             with open(config_file, "r", encoding="utf-8") as f:
@@ -169,7 +171,7 @@ def test(
                         action_timestamp = datetime.datetime.now().strftime("%Y%m%d@%H%M%S")
                         logger.info("Step %d: %s", step_idx + 1, action)
 
-                        observation, reward, done, info = env.step(action, args.sleep_after_execution)
+                        obs, reward, done, info = env.step(action, args.sleep_after_execution)
 
                         logger.info("Reward: %.2f", reward)
                         logger.info("Done: %s", done)
@@ -177,8 +179,8 @@ def test(
 
                         # Save screenshot and trajectory information
                         with open(os.path.join(example_result_dir, f"step_{step_idx + 1}_{action_timestamp}.png"),
-                                "wb") as _f:
-                            with open(observation['screenshot'], "rb") as __f:
+                                  "wb") as _f:
+                            with open(obs['screenshot'], "rb") as __f:
                                 screenshot = __f.read()
                             _f.write(screenshot)
 
@@ -198,10 +200,12 @@ def test(
                             logger.info("The episode is done.")
                             break
                     step_idx += 1
-                
+
                 result = env.evaluate()
                 logger.info("Result: %.2f", result)
                 scores.append(result)
+                with open(os.path.join(example_result_dir, "result.txt"), "w", encoding="utf-8") as f:
+                    f.write(f"{result}\n")
                 env.controller.end_recording(os.path.join(example_result_dir, "recording.mp4"))
 
             # example start running
@@ -218,18 +222,23 @@ def test(
                     }))
                     f.write("\n")    
                 continue
+            except Exception as e:
+                logger.error(f"Error in example {domain}/{example_id}: {e}")
+                continue
+
     env.close()
     logger.info(f"Average score: {sum(scores) / len(scores)}")
 
 
 def get_unfinished(action_space, use_model, observation_type, result_dir, total_file_json):
     target_dir = os.path.join(result_dir, action_space, observation_type, use_model)
-    
+
     if not os.path.exists(target_dir):
         return total_file_json
-    
+
     finished = {}
     for domain in os.listdir(target_dir):
+        finished[domain] = []
         finished[domain] = []
         domain_path = os.path.join(target_dir, domain)
         if os.path.isdir(domain_path):
@@ -245,13 +254,14 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
 
     if not finished:
         return total_file_json
-    
+
     for domain, examples in finished.items():
-        if domain in total_file_json: 
+        if domain in total_file_json:
             total_file_json[domain] = [x for x in total_file_json[domain] if x not in examples]
-    
+
     return total_file_json
 
+
 if __name__ == '__main__':
     ####### The complete version of the list of examples #######
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
@@ -260,11 +270,16 @@ if __name__ == '__main__':
     with open("evaluation_examples/test_all.json", "r", encoding="utf-8") as f:
         test_all_meta = json.load(f)
 
-    test_file_list = get_unfinished(args.action_space, args.model, args.observation_type, args.result_dir, test_all_meta)
+    test_file_list = get_unfinished(
+        args.action_space,
+        args.model,
+        args.observation_type,
+        args.result_dir,
+        test_all_meta
+    )
     left_info = ""
     for domain in test_file_list:
         left_info += f"{domain}: {len(test_file_list[domain])}\n"
     logger.info(f"Left tasks:\n{left_info}")
 
-    os.environ['OPENAI_API_KEY'] = "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"
-    test(args, test_all_meta)
+    test(args, test_all_meta)