add wandb settings

run.py

@@ -6,6 +6,7 @@ import datetime
 import json
 import logging
 import os
+import random
 import sys
 import wandb
 
@@ -75,7 +76,7 @@ def config() -> argparse.Namespace:
             "screenshot_a11y_tree",
             "som"
         ],
-        default="som",
+        default="a11y_tree",
         help="Observation type",
     )
     parser.add_argument("--screen_width", type=int, default=1920)
@@ -88,7 +89,7 @@ def config() -> argparse.Namespace:
     parser.add_argument("--test_config_base_dir", type=str, default="evaluation_examples")
 
     # lm config
-    parser.add_argument("--model", type=str, default="gpt-4-vision-preview")
+    parser.add_argument("--model", type=str, default="gpt-4-0125-preview")
     parser.add_argument("--temperature", type=float, default=1.0)
     parser.add_argument("--top_p", type=float, default=0.9)
     parser.add_argument("--max_tokens", type=int, default=1500)
@@ -231,15 +232,13 @@ def get_unfinished(action_space, use_model, observation_type, result_dir, total_
 
 def get_result(action_space, use_model, observation_type, result_dir, total_file_json):
     target_dir = os.path.join(result_dir, action_space, observation_type, use_model)
+    if not os.path.exists(target_dir):
+        print("New experiment, no result yet.")
+        return None
 
     all_result = []
 
-    if not os.path.exists(target_dir):
-        return total_file_json
-
-    finished = {}
     for domain in os.listdir(target_dir):
-        finished[domain] = []
         domain_path = os.path.join(target_dir, domain)
         if os.path.isdir(domain_path):
             for example_id in os.listdir(domain_path):
@@ -247,10 +246,17 @@ def get_result(action_space, use_model, observation_type, result_dir, total_file
                 if os.path.isdir(example_path):
                     if "result.txt" in os.listdir(example_path):
                         # empty all files under example_id
-                        all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read()))
+                        try:
+                            all_result.append(float(open(os.path.join(example_path, "result.txt"), "r").read()))
+                        except:
+                            all_result.append(0.0)
 
-    print("Success Rate:", sum(all_result) / len(all_result) * 100, "%")
-    return all_result
+    if not all_result:
+        print("New experiment, no result yet.")
+        return None
+    else:
+        print("Current Success Rate:", sum(all_result) / len(all_result) * 100, "%")
+        return all_result
 
 
 if __name__ == '__main__':