lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ver Jan27th

Browse Source 
debugged at+screenshot implementation, no issues found
fixed a little bugs
This commit is contained in:
David Chang
2024-01-27 23:07:25 +08:00
parent 50c346af11
commit 5a486b6b37
9 changed files with 68 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
experiment_screenshot_a11y_tree.py
View File

@@ -41,7 +41,8 @@ logger.addHandler(sdebug_handler)
logger = logging.getLogger("desktopenv.experiment")
PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
#PATH_TO_VM = r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx"
PATH_TO_VM = "../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx"
def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_trajectory", recording=True):
@@ -109,8 +110,9 @@ def run_one_example(example, agent, max_steps=10, example_trajectory_dir="exp_tr
if __name__ == "__main__":
action_space = "pyautogui"
example_class = "chrome"
example_id = "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3"
example_class = "libreoffice_calc"
#example_id = "7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3"
example_id = "01b269ae-2111-4a07-81fd-3fcd711993b0"
gpt4_model = "gpt-4-vision-preview"
gemini_model = "gemini-pro-vision"
@@ -120,7 +122,8 @@ if __name__ == "__main__":
with open(f"evaluation_examples/examples/{example_class}/{example_id}.json", "r") as f:
example = json.load(f)
example["snapshot"] = "exp_setup4"
#example["snapshot"] = "exp_setup4"
example["snapshot"] = "Snapshot 30"
api_key = os.environ.get("OPENAI_API_KEY")
agent = GPT4v_Agent(api_key=api_key, model=gpt4_model, instruction=example['instruction'],