Initialize all baselines: screenshot, a11y tree, both, SoM, SeeAct

2024-01-20 00:13:46 +08:00
parent 46bd3386dd
commit 09f3e776ae
14 changed files with 2588 additions and 1208 deletions
--- a/experiment_pure_text.py
+++ b/experiment_pure_text.py
@@ -62,6 +62,8 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr
        env.controller.start_recording()

    while not done and step_num < max_steps:
+        with open("accessibility_tree.xml", "w", encoding="utf-8") as f:
+            f.write(observation["accessibility_tree"])
        actions = agent.predict(observation)
        step_num += 1
        for action in actions: