lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FIx corner cases (val connection in chrome when using playwright, and action parsing for agent, and accessibility tree xml handling)

Browse Source
This commit is contained in:
Timothyxxx
2024-01-16 22:00:01 +08:00
parent 186bf2e97c
commit 20b1d950a0
5 changed files with 51 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
experiment_pure_text.py
View File

@@ -111,21 +111,25 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr
if __name__ == "__main__":
action_space = "pyautogui"
example_class = "chrome"
example_id = "06fe7178-4491-4589-810f-2e2bc9502122"
example_id = "7a5a7856-f1b6-42a4-ade9-1ca81ca0f263"
gpt4_model = "gpt-4-1106-preview"
gemini_model = "gemini-pro-vision"
with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r") as f:
example = json.load(f)
example["snapshot"] = "exp_setup4"
api_key = os.environ.get("OPENAI_API_KEY")
agent = GPT4_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space)
agent = GPT4_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'], action_space=action_space)
# api_key = os.environ.get("GENAI_API_KEY")
# agent = GeminiPro_Agent(api_key=api_key, instruction=example['instruction'], action_space=action_space)
# agent = GeminiPro_Agent(api_key=api_key, model=gemini_model, instruction=example['instruction'], action_space=action_space)
root_trajectory_dir = "exp_trajectory"
example_trajectory_dir = os.path.join(root_trajectory_dir, example_class, example_id)
example_trajectory_dir = os.path.join(root_trajectory_dir, "text", example_class, gpt4_model, example_id)
# example_trajectory_dir = os.path.join(root_trajectory_dir, "text", example_class, gemini_model, example_id)
os.makedirs(example_trajectory_dir, exist_ok=True)
run_one_example(example, agent, 10, example_trajectory_dir)
run_one_example(example, agent, 15, example_trajectory_dir)