From c346b4379de295732bd8cf07a7556ece0aafceba Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 26 Jan 2024 20:33:24 +0800 Subject: [PATCH 1/7] Make up one OS example --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/getters/general.py | 4 ++ .../13584542-872b-42d8-b299-866967b5c3ef.json | 55 ++++++++++++++----- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index faa9f2e..4f4b43f 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -2,7 +2,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g get_shortcuts_on_desktop, get_history, get_enabled_experiments, get_chrome_language, get_chrome_font_size, \ get_profile_name, get_number_of_search_results, get_googledrive_file, get_active_tab_info from .file import get_cloud_file, get_vm_file, get_cache_file -from .general import get_vm_command_line +from .general import get_vm_command_line, get_vm_terminal_output from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree diff --git a/desktop_env/evaluators/getters/general.py b/desktop_env/evaluators/getters/general.py index c94654e..b694167 100644 --- a/desktop_env/evaluators/getters/general.py +++ b/desktop_env/evaluators/getters/general.py @@ -18,3 +18,7 @@ def get_vm_command_line(env, config: Dict[str, str]): else: logger.error("Failed to get vm command line. Status code: %d", response.status_code) return None + + +def get_vm_terminal_output(env, config: Dict[str, str]): + return env.controller.get_terminal_output() diff --git a/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json index 3abbd63..4e08188 100644 --- a/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json +++ b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json @@ -1,17 +1,46 @@ { - "id": "13584542-872b-42d8-b299-866967b5c3ef", - "snapshot": "os", - "instruction": "Set the default terminal size and screen x,y coordinates in order to make the terminal window always open at the same location and with the same size.", - "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { + "id": "13584542-872b-42d8-b299-866967b5c3ef", + "snapshot": "os", + "instruction": "I click in terminal: terminal->132x43 to change terminal size but after each reboot terminal size is set to default setting and I have to change it again. Help me set it permanently", + "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position", + "trajectory": "trajectories/", + "config": [ + ], + "related_apps": [ + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "sleep", + "parameters": { + "seconds": 1 + } }, - "expected": { + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(0.5); pyautogui.hotkey('ctrl', 'alt', 't'); time.sleep(0.5); pyautogui.write('stty size'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], + "func": "check_include_exclude", + "result": { + "type": "vm_terminal_output" + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + "43 132" + ], + "exclude": [ + ] } } - } \ No newline at end of file + } +} \ No newline at end of file From 6952b45de4f3df7972b806a885d30a412ed7f641 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Fri, 26 Jan 2024 23:30:04 +0800 Subject: [PATCH 2/7] Improve on agent and tasks configs --- desktop_env/controllers/python.py | 2 +- .../13584542-872b-42d8-b299-866967b5c3ef.json | 10 ++++ .../23393935-50c7-4a86-aeea-2b78fd089c5c.json | 10 ++++ .../28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json | 13 ++++- .../3ce045a0-877b-42aa-8d2c-b4a863336ab8.json | 13 ++++- .../43c2d64c-bab5-4dcb-a30c-b888321c319a.json | 15 +++++- .../5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json | 10 ++++ .../765d2b74-88a7-4d50-bf51-34e4106fd24a.json | 13 ++++- .../7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json | 10 ++++ .../94d95f96-9699-4208-98ba-3c3119edf9c2.json | 13 ++++- .../a462a795-fdc7-4b23-b689-e8b6df786b78.json | 10 ++++ .../a4d98375-215b-4a4d-aee9-3d4370fccc41.json | 12 +++++ .../ae039631-2b12-4637-84f6-c67d51511be3.json | 10 ++++ .../b6781586-6346-41cd-935a-a6b1487918fc.json | 10 ++++ .../bedcedc4-4d72-425e-ad62-21960b11fe0d.json | 13 ++++- .../ddc75b62-7311-4af8-bfb3-859558542b36.json | 12 +++++ .../e0df059f-28a6-4169-924f-b9623e7184cc.json | 10 ++++ .../ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json | 10 ++++ .../f9be0997-4b7c-45c5-b05c-4612b44a6118.json | 10 ++++ .../fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json | 54 +++++++++++-------- .../215dfd39-f493-4bc3-a027-8a97d72c61bf.json | 10 ++++ .../386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json | 10 ++++ .../59f21cfb-0120-4326-b255-a5b827b38967.json | 10 ++++ .../8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json | 10 ++++ .../8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json | 27 +++++++++- .../8f080098-ddb1-424c-b438-4e96e5e4786e.json | 10 ++++ .../9195653c-f4aa-453d-aa95-787f6ccfaae9.json | 10 ++++ .../a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json | 10 ++++ .../aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json | 10 ++++ .../bba3381f-b5eb-4439-bd9e-80c22218d5a7.json | 10 ++++ .../d06f0d4d-2cd5-4ede-8de9-598629438c6e.json | 10 ++++ .../efcf0d81-0835-4880-b2fd-d866e8bc2294.json | 10 ++++ .../f3977615-2b45-4ac5-8bba-80c17dbe2a37.json | 10 ++++ .../fba2c100-79e8-42df-ae74-b592418d54f4.json | 22 ++++++++ mm_agents/gpt_4v_agent.py | 12 ++--- mm_agents/prompts.py | 30 +++++++---- 36 files changed, 425 insertions(+), 46 deletions(-) diff --git a/desktop_env/controllers/python.py b/desktop_env/controllers/python.py index a4b2abb..1f0d32e 100644 --- a/desktop_env/controllers/python.py +++ b/desktop_env/controllers/python.py @@ -11,7 +11,7 @@ logger = logging.getLogger("desktopenv.pycontroller") class PythonController: - def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; {command}"): + def __init__(self, vm_ip: str, pkgs_prefix: str = "import pyautogui; import time; {command}"): self.vm_ip = vm_ip self.http_server = f"http://{vm_ip}:5000" self.pkgs_prefix = pkgs_prefix # fixme: this is a hacky way to execute python commands. fix it and combine it with installation of packages diff --git a/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json index 4e08188..da14f2c 100644 --- a/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json +++ b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json @@ -5,6 +5,16 @@ "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position", "trajectory": "trajectories/", "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } ], "related_apps": [ "os" diff --git a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json index 654a749..239fd23 100644 --- a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json +++ b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json @@ -80,6 +80,16 @@ } ] } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "related_apps": [ diff --git a/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json b/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json index a88aaf3..a5bc881 100644 --- a/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json +++ b/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json @@ -4,7 +4,18 @@ "instruction": "The volume of my system is too small. Can you help me turn up to the max volume?", "source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json index 99df0f7..6cb0bea 100644 --- a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json +++ b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json @@ -4,7 +4,18 @@ "instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?", "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json b/evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json index e537a60..bf6a665 100644 --- a/evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json +++ b/evaluation_examples/examples/os/43c2d64c-bab5-4dcb-a30c-b888321c319a.json @@ -3,7 +3,18 @@ "snapshot": "os", "instruction": "Could you please help me create a dir named 'test' in the root directory of current computer?", "source": "https://ubuntu.com/tutorials/command-line-for-beginners#4-creating-folders-and-files", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "os" @@ -12,7 +23,7 @@ "func": "exact_match", "result": { "type": "vm_command_line", - "command": "[ -d '/home/user/Desktop/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'", + "command": "[ -d '/test' ] && echo 'Directory exists.' || echo 'Directory does not exist.'", "shell": true }, "expected": { diff --git a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json index 7ce6f21..d51c886 100644 --- a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json +++ b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json @@ -22,6 +22,16 @@ "command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/", "shell": true } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "related_apps": [ diff --git a/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json b/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json index 690cdc8..98f51bd 100644 --- a/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json +++ b/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json @@ -3,7 +3,18 @@ "snapshot": "os", "instruction": "Can you help me delete the \"test\" file on my desktop?", "source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "trajectory": "trajectories/", "related_apps": [ "os" diff --git a/evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json b/evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json index a3b6035..be8d79e 100644 --- a/evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json +++ b/evaluation_examples/examples/os/7688b85f-87a4-4e4a-b2f8-f3d6c3f29b82.json @@ -17,7 +17,17 @@ "command": "echo 'password' | sudo -S chmod 777 ~/Desktop/todo.txt && sudo chmod 777 ~/Desktop/done", "shell": true } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] } + } ], "trajectory": "trajectories/", "related_apps": [ diff --git a/evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json b/evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json index 90249d0..027b7e2 100644 --- a/evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json +++ b/evaluation_examples/examples/os/94d95f96-9699-4208-98ba-3c3119edf9c2.json @@ -4,7 +4,18 @@ "instruction": "I want to install Spotify on my current system. Could you please help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-install.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json b/evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json index 72d0454..f043475 100644 --- a/evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json +++ b/evaluation_examples/examples/os/a462a795-fdc7-4b23-b689-e8b6df786b78.json @@ -10,6 +10,16 @@ "command": "echo password | sudo -S su - charles", "shell": true } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json index 3675d05..750fea6 100644 --- a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json +++ b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json @@ -4,6 +4,18 @@ "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json b/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json index c82d0b0..9c97d21 100644 --- a/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json +++ b/evaluation_examples/examples/os/ae039631-2b12-4637-84f6-c67d51511be3.json @@ -9,6 +9,16 @@ "parameters": { "command": "xdg-settings set default-web-browser firefox.desktop" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json index 46964d3..329673c 100644 --- a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json +++ b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json @@ -5,6 +5,16 @@ "source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en", "trajectory": "trajectories/", "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } ], "related_apps": [ "os" diff --git a/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json b/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json index 3061ba6..6c67c0c 100644 --- a/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json +++ b/evaluation_examples/examples/os/bedcedc4-4d72-425e-ad62-21960b11fe0d.json @@ -4,7 +4,18 @@ "instruction": "Could you set the 'Dim screen when inactive' to on in setting?", "source": "https://www.youtube.com/watch?v=D4WyNjt_hbQ&t=2s", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json index c54f368..662de63 100644 --- a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json +++ b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json @@ -4,6 +4,18 @@ "instruction": "I want to uninstall the Mahjongg on my system. Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } + } + ], "related_apps": [ "os" ], diff --git a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json index c9e41d8..3c54c5a 100644 --- a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json +++ b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json @@ -11,6 +11,16 @@ "command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1", "shell": true } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "related_apps": [ diff --git a/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json b/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json index 1b4a330..30daecb 100644 --- a/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json +++ b/evaluation_examples/examples/os/ec4e3f68-9ea4-4c18-a5c9-69f89d1178b3.json @@ -21,6 +21,16 @@ "parameters": { "command": "gsettings set org.gnome.shell favorite-apps \"['thunderbird.desktop', 'vim.desktop', 'google-chrome.desktop']\"" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json b/evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json index 96632aa..7a07888 100644 --- a/evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json +++ b/evaluation_examples/examples/os/f9be0997-4b7c-45c5-b05c-4612b44a6118.json @@ -10,7 +10,17 @@ "command": "gsettings set org.gnome.desktop.notifications show-banners true", "shell": true } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] } + } ], "trajectory": "trajectories/", "related_apps": [ diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json index c613775..79d8618 100644 --- a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -1,24 +1,36 @@ { - "id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3", - "snapshot": "os", - "instruction": "I want to see the battery percentage. Can you help me display it on my screen?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "exact_match", - "result": { - "type": "vm_command_line", - "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", - "shell": true - }, - "expected": { - "type": "rule", - "rules":{ - "expected": "true\n" - } + "id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3", + "snapshot": "os", + "instruction": "I want to see the battery percentage. Can you help me display it on my screen?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "expected": "true\n" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json b/evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json index 21f4222..afaa063 100644 --- a/evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json +++ b/evaluation_examples/examples/vlc/215dfd39-f493-4bc3-a027-8a97d72c61bf.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json b/evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json index 75dff19..d2d5dec 100644 --- a/evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json +++ b/evaluation_examples/examples/vlc/386dbd0e-0241-4a0a-b6a2-6704fba26b1c.json @@ -64,6 +64,16 @@ "import pyautogui; pyautogui.press('f11');" ] } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json index 14ce3cd..fa35f68 100644 --- a/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json +++ b/evaluation_examples/examples/vlc/59f21cfb-0120-4326-b255-a5b827b38967.json @@ -20,6 +20,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json b/evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json index d59d628..66b23fd 100644 --- a/evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json +++ b/evaluation_examples/examples/vlc/8ba5ae7a-5ae5-4eab-9fcc-5dd4fe3abf89.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json b/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json index 8c2fdb2..636eb25 100644 --- a/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json +++ b/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json @@ -4,10 +4,35 @@ "instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.", "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=13OhDRuINzQ-w0qmzdGJ8ZSWnKyKnTRje&export=download&authuser=0&confirm=t&uuid=39cf0297-23eb-47df-a642-fd6c2f981eff&at=APZUnTUR83eI1tenMxVdyFxNeBo7:1706275408621", + "path": "Desktop/Gen 2.mp4" + } + ] + } + }, { "type": "launch", "parameters": { - "command": "vlc" + "command": [ + "vlc", + "--start-time=15", + "Desktop/Gen 2.mp4" + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] } } ], diff --git a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json index a39bd31..7b521e8 100644 --- a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json +++ b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json @@ -24,6 +24,16 @@ "Desktop/Baby Justin Bieber.mp4" ] } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json b/evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json index 73d0fd2..3345bf0 100644 --- a/evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json +++ b/evaluation_examples/examples/vlc/9195653c-f4aa-453d-aa95-787f6ccfaae9.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json b/evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json index ff39f72..5b67718 100644 --- a/evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json +++ b/evaluation_examples/examples/vlc/a5bbbcd5-b398-4c91-83d4-55e1e31bbb81.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json b/evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json index f525f96..e472f97 100644 --- a/evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json +++ b/evaluation_examples/examples/vlc/aa4b5023-aef6-4ed9-bdc9-705f59ab9ad6.json @@ -20,6 +20,16 @@ "parameters": { "command": ["vlc", "Desktop/flipped_1984_Apple_Macintosh_Commercial.mp4"] } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json index a5cd48d..b6c397c 100644 --- a/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json +++ b/evaluation_examples/examples/vlc/bba3381f-b5eb-4439-bd9e-80c22218d5a7.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json b/evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json index c29a8bc..a6bcc8f 100644 --- a/evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json +++ b/evaluation_examples/examples/vlc/d06f0d4d-2cd5-4ede-8de9-598629438c6e.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json index 3b07f3c..b07c6de 100644 --- a/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json +++ b/evaluation_examples/examples/vlc/efcf0d81-0835-4880-b2fd-d866e8bc2294.json @@ -26,6 +26,16 @@ "Desktop/Interstellar Movie - Official Trailer.mp4" ] } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json b/evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json index e1ee77e..3be1e5d 100644 --- a/evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json +++ b/evaluation_examples/examples/vlc/f3977615-2b45-4ac5-8bba-80c17dbe2a37.json @@ -9,6 +9,16 @@ "parameters": { "command": "vlc" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json index 0cd51d9..f71069e 100644 --- a/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json +++ b/evaluation_examples/examples/vlc/fba2c100-79e8-42df-ae74-b592418d54f4.json @@ -26,6 +26,28 @@ "Desktop/Interstellar Movie - Official Trailer.mp4" ] } + }, + { + "type": "sleep", + "parameters": { + "seconds": 1 + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Interstellar Movie - Interstellar Movie - Official Trailer - VLC media player" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.click(500, 500); time.sleep(0.5);" + ] + } } ], "trajectory": "trajectories/", diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py index cc9e9d5..423e554 100644 --- a/mm_agents/gpt_4v_agent.py +++ b/mm_agents/gpt_4v_agent.py @@ -225,8 +225,8 @@ class GPT4v_Agent: }) # Append trajectory - assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts)\ - , "The number of observations and actions should be the same." + assert len(self.observations) == len(self.actions) and len(self.actions) == len(self.thoughts) \ + , "The number of observations and actions should be the same." if len(self.observations) > self.max_trajectory_length: _observations = self.observations[-self.max_trajectory_length:] @@ -255,7 +255,7 @@ class GPT4v_Agent: { "type": "image_url", "image_url": { - "url": f"data:image/jpeg;base64,{_screenshot}", + "url": f"data:image/png;base64,{_screenshot}", "detail": "high" } } @@ -315,14 +315,14 @@ class GPT4v_Agent: ] }) else: - raise ValueError("Invalid experiment type: " + self.exp) # 1}}} + raise ValueError("Invalid experiment type: " + self.exp) # 1}}} messages.append({ "role": "assistant", "content": [ { "type": "text", - "text": previous_thought.stip() if len(previous_thought)>0 else "No valid action" + "text": previous_thought.strip() if len(previous_thought) > 0 else "No valid action" }, ] }) @@ -436,7 +436,7 @@ class GPT4v_Agent: ] }) else: - raise ValueError("Invalid experiment type: " + self.exp) # 1}}} + raise ValueError("Invalid experiment type: " + self.exp) # 1}}} with open("messages.json", "w") as f: f.write(json.dumps(messages, indent=4)) diff --git a/mm_agents/prompts.py b/mm_agents/prompts.py index 90ce22f..85295de 100644 --- a/mm_agents/prompts.py +++ b/mm_agents/prompts.py @@ -3,8 +3,9 @@ You are an agent which follow my instruction and perform desktop computer tasks You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard. For each step, you will get an observation of an image, which is the screenshot of the computer screen and you will predict the action of the computer based on the image. -You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. -Return one line or multiple lines of python code to perform the action each time, be time efficient. +You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot. +Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take +You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct. You ONLY need to return the code inside a code block, like this: ```python # your code here @@ -14,6 +15,7 @@ When you think you have to wait for some time, return ```WAIT```; When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task; When you think the task is done, return ```DONE```. +My computer's password is 'password', feel free to use it when you need sudo rights. First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE. """.strip() @@ -267,8 +269,9 @@ You are an agent which follow my instruction and perform desktop computer tasks You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard. For each step, you will get an observation of the desktop by accessibility tree, which is based on AT-SPI library. And you will predict the action of the computer based on the accessibility tree. -You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. -Return one line or multiple lines of python code to perform the action each time, be time efficient. +You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot. +Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take +You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct. You ONLY need to return the code inside a code block, like this: ```python # your code here @@ -278,6 +281,7 @@ When you think you have to wait for some time, return ```WAIT```; When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task; When you think the task is done, return ```DONE```. +My computer's password is 'password', feel free to use it when you need sudo rights. First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE. """.strip() @@ -532,8 +536,9 @@ You have good knowledge of computer and good internet connection and assume your For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library. And you will predict the action of the computer based on the screenshot and accessibility tree. -You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. -Return one line or multiple lines of python code to perform the action each time, be time efficient. +You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot. +Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take +You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct. You ONLY need to return the code inside a code block, like this: ```python # your code here @@ -543,6 +548,7 @@ When you think you have to wait for some time, return ```WAIT```; When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task; When you think the task is done, return ```DONE```. +My computer's password is 'password', feel free to use it when you need sudo rights. First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE. """.strip() @@ -797,7 +803,7 @@ You are an agent which follow my instruction and perform desktop computer tasks You have good knowledge of computer and good internet connection and assume your code will run on a computer for controlling the mouse and keyboard. For each step, you will get an observation of the desktop by 1) a screenshot; and 2) accessibility tree, which is based on AT-SPI library. -You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. +You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot. You can replace x, y in the code with the tag of the element you want to operate with. such as: ```python pyautogui.moveTo(tag#3) @@ -806,7 +812,8 @@ pyautogui.dragTo(tag#1, button='left') ``` When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly. But you should be careful to ensure that the coordinates are correct. -Return one line or multiple lines of python code to perform the action each time, be time efficient. +Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take +You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct. You ONLY need to return the code inside a code block, like this: ```python # your code here @@ -816,6 +823,7 @@ When you think you have to wait for some time, return ```WAIT```; When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task; When you think the task is done, return ```DONE```. +My computer's password is 'password', feel free to use it when you need sudo rights. First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE. """.strip() @@ -845,7 +853,7 @@ Then, based on your analysis, in conjunction with human desktop using habits and """ ACTION_GROUNDING_PROMPT_SEEACT = """ -You are required to use `pyautogui` to perform the action, but don't use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. +You are required to use `pyautogui` to perform the action grounded to the observation, but DONOT use the `pyautogui.locateCenterOnScreen` function to locate the element you want to operate with since we have no image of the element you want to operate with. DONOT USE `pyautogui.screenshot()` to make screenshot. You can replace x, y in the code with the tag of the element you want to operate with. such as: ```python pyautogui.moveTo(tag#3) @@ -854,7 +862,8 @@ pyautogui.dragTo(tag#1, button='left') ``` When you think you can directly output precise x and y coordinates or there is no tag on which you want to interact, you can also use them directly. But you should be careful to ensure that the coordinates are correct. -Return one line or multiple lines of python code to perform the action each time, be time efficient. +Return one line or multiple lines of python code to perform the action each time, be time efficient. When predicting multiple lines of code, make some small sleep like `time.sleep(0.5);` interval so that the machine could take +You need to to specify the coordinates of by yourself based on your observation of current observation, but you should be careful to ensure that the coordinates are correct. You ONLY need to return the code inside a code block, like this: ```python # your code here @@ -864,5 +873,6 @@ When you think you have to wait for some time, return ```WAIT```; When you think the task can not be done, return ```FAIL```, don't easily say ```FAIL```, try your best to do the task; When you think the task is done, return ```DONE```. +My computer's password is 'password', feel free to use it when you need sudo rights. First give the current screenshot and previous things we did a short reflection, then RETURN ME THE CODE OR SPECIAL CODE I ASKED FOR. NEVER EVER RETURN ME ANYTHING ELSE. """ From b7e3e1f7d074f15cc1e6ae289ce71e72d85f61b7 Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 26 Jan 2024 23:40:20 +0800 Subject: [PATCH 3/7] ver Jan26thv2 fixed bugs by incomplete merging --- desktop_env/server/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 83d1c67..032b290 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -5,7 +5,7 @@ import shlex import subprocess from pathlib import Path from typing import Any, Optional -from typing import List, Dict +from typing import List, Dict, Tuple import Xlib import lxml.etree @@ -16,7 +16,7 @@ from PIL import Image from Xlib import display, X from flask import Flask, request, jsonify, send_file, abort # , send_from_directory from lxml.etree import _Element -from pyatspi import Accessible, StateType +from pyatspi import Accessible, StateType, STATE_SHOWING from pyatspi import Action as ATAction from pyatspi import Component, Document from pyatspi import Text as ATText From eef5158663d9f0d2bfe4a573cbb8db68dc65aadc Mon Sep 17 00:00:00 2001 From: David Chang Date: Fri, 26 Jan 2024 23:47:46 +0800 Subject: [PATCH 4/7] ver Jan26thv3 fixed bug caused by an empty node.text remove nodes whose name and text are all empty --- mm_agents/accessibility_tree_wrap/heuristic_retrieve.py | 2 ++ mm_agents/gpt_4v_agent.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py b/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py index 47bbca0..ccb4886 100644 --- a/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py +++ b/mm_agents/accessibility_tree_wrap/heuristic_retrieve.py @@ -40,6 +40,8 @@ def filter_nodes(nodes): filtered_nodes.append(node) elif node.tag == 'text': continue + elif node.get("name") == "" and node.text is None: + continue else: coords = tuple( map(int, node.attrib.get('{uri:deskat:component.at-spi.gnome.org}screencoord').strip('()').split(', '))) diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py index 323e08f..cc08b79 100644 --- a/mm_agents/gpt_4v_agent.py +++ b/mm_agents/gpt_4v_agent.py @@ -39,7 +39,10 @@ def linearize_accessibility_tree(accessibility_tree): for node in filtered_nodes: linearized_accessibility_tree += node.tag + "\t" linearized_accessibility_tree += node.attrib.get('name') + "\t" - linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t" + if node.text: + linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t" + else: + linearized_accessibility_tree += '""\t' linearized_accessibility_tree += node.attrib.get( '{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t" linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n" @@ -87,7 +90,7 @@ def parse_actions_from_string(input_string): try: action_dict = json.loads(input_string) return [action_dict] - except json.JSONDecodeError as e: + except json.JSONDecodeError: raise ValueError("Invalid response format: " + input_string) From 63852755d299feaa6ecd0f6136b90341717a77c9 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 27 Jan 2024 11:40:05 +0800 Subject: [PATCH 5/7] Make up postconfig for libreoffice writer examples --- .../eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json | 2 +- .../0810415c-bde4-4443-9047-d5f70165a697.json | 25 +++++++++++++ .../0b17a146-2934-46c7-8727-73ff6b6483e8.json | 25 +++++++++++++ .../0e47de2a-32e0-456c-a366-8c607ef7a9d2.json | 25 +++++++++++++ .../0e763496-b6bb-4508-a427-fad0b6c3e195.json | 25 +++++++++++++ .../3ef2b351-8a84-4ff2-8724-d86eae9b842e.json | 25 +++++++++++++ .../41c621f7-3544-49e1-af8d-dafd0f834f75.json | 27 +++++++++++++- .../45d61a06-6545-4422-97b7-bc76cfa964c1.json | 25 +++++++++++++ .../663876c7-3471-43db-ba51-f410b13d9d7d.json | 25 +++++++++++++ .../66399b0d-8fda-4618-95c4-bfc6191617e9.json | 25 +++++++++++++ .../6ada715d-3aae-4a32-a6a7-429b2e43fb93.json | 35 +++++++++++++++++++ .../936321ce-5236-426a-9a20-e0e3c5dc536f.json | 25 +++++++++++++ .../adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json | 25 +++++++++++++ .../e528b65e-1107-4b8c-8988-490e4fece599.json | 25 +++++++++++++ .../ecc2413d-8a48-416e-a3a2-d30106ca36cb.json | 25 +++++++++++++ 15 files changed, 362 insertions(+), 2 deletions(-) diff --git a/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json b/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json index e6f192c..080e333 100644 --- a/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json +++ b/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey([\"ctrl\", \"s\"]);" ] } } diff --git a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json index 09dd8ac..47a3a8d 100644 --- a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json +++ b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Novels_Intro_Packet.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_line_spacing", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json index 5b4ad99..6b938d5 100644 --- a/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json +++ b/evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "H2O_Factsheet_WA.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_files", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json index f5cc925..cc78afa 100644 --- a/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json +++ b/evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "LibreOffice_Open_Source_Word_Processing.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "has_page_numbers_in_footers", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json index 6d886c8..e5381c8 100644 --- a/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json +++ b/evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Dublin_Zoo_Intro.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_font_names", "expected": { "type": "rule", diff --git a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json index 17196f6..e5b74b8 100644 --- a/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json +++ b/evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Constitution_Template_With_Guidelines.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "is_first_line_centered", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json b/evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json index 531cff0..fc76286 100644 --- a/evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json +++ b/evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json @@ -18,7 +18,7 @@ { "type": "open", "parameters": { - "path": "How_to_read_a_scientific_article.docx" + "path": "Desktop/How_to_read_a_scientific_article.docx" } } ], @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "How_to_read_a_scientific_article.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": [ "compare_highlighted_text", "compare_docx_files" diff --git a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json index 99a960c..6f4202e 100644 --- a/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json +++ b/evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "NOVEL_Submission_Guidelines.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_line_spacing", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json index d04915a..8cbc2c6 100644 --- a/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json +++ b/evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Factoring_Perfect_Square_Trinomials.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_insert_equation", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json index a45e7fd..b4e4c4d 100644 --- a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json +++ b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Table_Of_Work_Effort_Instructions.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_tables", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json index ff6fa76..5440b89 100644 --- a/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json +++ b/evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json @@ -31,6 +31,16 @@ "parameters": { "path": "Desktop/Viewing_Your_Class_Schedule_and_Textbooks.docx" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=8); time.sleep(1); pyautogui.scroll(-2)" + ] + } } ], "trajectory": "trajectories/", @@ -38,6 +48,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Viewing_Your_Class_Schedule_and_Textbooks.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_contains_image", "result": { "type": "vm_file", diff --git a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json index 92591f0..de31187 100644 --- a/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json +++ b/evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Graphemes_Sound_Letter_Patterns_Gold.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_tables", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json index a932884..8a5a353 100644 --- a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json +++ b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Essay_Writing_English_for_uni.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json index 451a68d..0077101 100644 --- a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json +++ b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Geography_And_Magical_Realism.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "compare_docx_files", "expected": { "type": "cloud_file", diff --git a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json index 25a9d38..d029bb6 100644 --- a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json +++ b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json @@ -27,6 +27,31 @@ "libreoffice_writer" ], "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Sample_Statutory_Declaration.docx - LibreOffice Writer", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')" + ] + } + } + ], "func": "contains_page_break", "result": { "type": "vm_file", From f8ff612b854506d2c561ff61cc9e018c8cf275ad Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Sat, 27 Jan 2024 14:09:39 +0800 Subject: [PATCH 6/7] Fix errors found in libreoffice writer examples --- .../4bcb1253-a636-4df4-8cb0-a35c04dfef31.json | 14 +++++++++++--- .../66399b0d-8fda-4618-95c4-bfc6191617e9.json | 12 +++++++++++- .../adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json | 2 +- .../e528b65e-1107-4b8c-8988-490e4fece599.json | 2 +- .../ecc2413d-8a48-416e-a3a2-d30106ca36cb.json | 2 +- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json index ba1cb2c..1f31717 100644 --- a/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json +++ b/evaluation_examples/examples/libreoffice_writer/4bcb1253-a636-4df4-8cb0-a35c04dfef31.json @@ -27,8 +27,16 @@ "libreoffice_writer" ], "evaluator": { - "func": "check_file_exists", - "file_name": "View_Person_Organizational_Summary.pdf", - "directory": "/home/user/Downloads/" + "func": "compare_pdfs", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1Spn-Gw7D-bRvV4udNQoGNEOViUqf6bL0&export=download&authuser=0&confirm=t&uuid=dcc0eb01-89ed-4852-a7cb-d0400d977ac8&at=APZUnTX57XnHwmb-y3m4JdNkvu6z:1706328786805", + "dest": "Constitution_Template_With_Guidelines_Gold.pdf" + }, + "result": { + "type": "vm_file", + "path": "Desktop/View_Person_Organizational_Summary.pdf", + "dest": "Constitution_Template_With_Guidelines.pdf" + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json index b4e4c4d..47dde68 100644 --- a/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json +++ b/evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1kXBP0jMxTeVahzFLYbYHJtjjgmuzrA8R&export=download&authuser=0&confirm=t&uuid=f8b9bad3-415d-4d39-a4fb-05a4cf881cf0&at=APZUnTXaohwzl8_2RDF_tgUsP9cH:1704181463579", + "url": "https://drive.usercontent.google.com/download?id=1yeD0YIFCSEAdi6MPiiqLe6F4a19I5wZP&export=download&authuser=0&confirm=t&uuid=b6693633-00d7-4a8b-b35e-77ec452dd6a3&at=APZUnTVHRdoysZFBTmGmX6rCJLHK:1706330570587", "path": "Desktop/Table_Of_Work_Effort_Instructions.docx" } ] @@ -20,6 +20,16 @@ "parameters": { "path": "Desktop/Table_Of_Work_Effort_Instructions.docx" } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(1); pyautogui.press(\"down\", presses=38, interval=0.1); time.sleep(1); pyautogui.scroll(-2)" + ] + } } ], "trajectory": "trajectories/", diff --git a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json index 8a5a353..ba3c432 100644 --- a/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json +++ b/evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json @@ -1,7 +1,7 @@ { "id": "adf5e2c3-64c7-4644-b7b6-d2f0167927e7", "snapshot": "libreoffice_writer", - "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference where my cursor is located (in the fourth paragraph).", + "instruction": "Help me adding \"Steinberg, F. M., Bearden, M. M., & Keen, C. L. (2003). Cocoa and chocolate flavonoids: Implications for cardiovascular health. Journal of the American Dietetic Association, 103(2), 215-223. doi: 10.1053/jada.2003.50028\" to my reference list, and add a cross reference in the fourth paragraph where I marked \"\".", "source": "https://seekstar.github.io/2022/04/11/libreoffice%E5%BC%95%E7%94%A8%E6%96%87%E7%8C%AE/", "config": [ { diff --git a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json index 0077101..c63d0c9 100644 --- a/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json +++ b/evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json @@ -10,7 +10,7 @@ "files": [ { "url": "https://drive.usercontent.google.com/download?id=181hyG_NZUJaUp7kUdeFoUpNGpOcjzYFV&export=download&authuser=0&confirm=t&uuid=fdb8aee4-ea1f-43c1-bc66-9d44099512e1&at=APZUnTVUrypBeVnTfkJsriv7S2GO:1704969486759", - "path": "Geography_And_Magical_Realism.docx" + "path": "Desktop/Geography_And_Magical_Realism.docx" } ] } diff --git a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json index d029bb6..9bea82e 100644 --- a/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json +++ b/evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json @@ -1,7 +1,7 @@ { "id": "ecc2413d-8a48-416e-a3a2-d30106ca36cb", "snapshot": "libreoffice_writer", - "instruction": "Help me insert a blank page where my cursor is located.", + "instruction": "Hey, can you throw in a blank page right after this one?", "source": "https://www.quora.com/How-can-I-insert-a-blank-page-on-libreoffice", "config": [ { From 8025bf19f0ac0ed47d850582d9deaa92eab687d2 Mon Sep 17 00:00:00 2001 From: David Chang Date: Sat, 27 Jan 2024 19:46:06 +0800 Subject: [PATCH 7/7] ver Jan27th corrected usage of pyautogui in calc postconfig --- .../libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json | 2 +- .../libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json | 2 +- .../libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json | 2 +- .../libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json | 2 +- .../libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json | 2 +- .../libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json | 2 +- .../libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json | 2 +- .../libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json | 2 +- .../libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json | 2 +- .../libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json | 2 +- .../libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json | 2 +- .../libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json | 2 +- .../libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json | 2 +- .../libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json | 2 +- .../libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json | 2 +- .../libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json | 2 +- .../libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json | 2 +- .../libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json | 2 +- .../libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json | 2 +- .../libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json | 2 +- .../libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json | 2 +- .../a01fbce3-2793-461f-ab86-43680ccbae25.json.json | 2 +- .../libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json | 2 +- .../libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json | 2 +- .../libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json | 2 +- .../libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json | 2 +- .../libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json | 2 +- .../libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json index 07bd39e..3ae596d 100644 --- a/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json +++ b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json index cd82473..4340397 100644 --- a/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json +++ b/evaluation_examples/examples/libreoffice_calc/0bf05a7d-b28b-44d2-955a-50b41e24012a.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json index 7f6ca85..e12377c 100644 --- a/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json +++ b/evaluation_examples/examples/libreoffice_calc/0cecd4f3-74de-457b-ba94-29ad6b5dafb6.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json index 6885485..afc36a8 100644 --- a/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json +++ b/evaluation_examples/examples/libreoffice_calc/1334ca3e-f9e3-4db8-9ca7-b4c653be7d17.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json index d02c5f7..328a996 100644 --- a/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json +++ b/evaluation_examples/examples/libreoffice_calc/21df9241-f8d7-4509-b7f1-37e501a823f7.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index fbf8b33..6b92cc1 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json index e21370f..9115022 100644 --- a/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json +++ b/evaluation_examples/examples/libreoffice_calc/347ef137-7eeb-4c80-a3bb-0951f26a8aff.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json b/evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json index 5ec4316..544a464 100644 --- a/evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json +++ b/evaluation_examples/examples/libreoffice_calc/357ef137-7eeb-4c80-a3bb-0951f26a8aff.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json index 2967f16..1e37657 100644 --- a/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json +++ b/evaluation_examples/examples/libreoffice_calc/37608790-6147-45d0-9f20-1137bb35703d.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json index ed98840..d642f24 100644 --- a/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json +++ b/evaluation_examples/examples/libreoffice_calc/39aa4e37-dc91-482e-99af-132a612d40f3.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json index 79f5087..7f1137d 100644 --- a/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json +++ b/evaluation_examples/examples/libreoffice_calc/4188d3a4-077d-46b7-9c86-23e1a036f6c1.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json b/evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json index 772c3b1..65df118 100644 --- a/evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json +++ b/evaluation_examples/examples/libreoffice_calc/4e6fcf72-daf3-439f-a232-c434ce416af6.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json index 7403b7f..dae73c2 100644 --- a/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json +++ b/evaluation_examples/examples/libreoffice_calc/4f07fbe9-70de-4927-a4d5-bb28bc12c52c.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json b/evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json index ec1694d..b89606d 100644 --- a/evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json +++ b/evaluation_examples/examples/libreoffice_calc/51b11269-2ca8-4b2a-9163-f21758420e78.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json b/evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json index b5e9241..833a9cb 100644 --- a/evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json +++ b/evaluation_examples/examples/libreoffice_calc/6054afcb-5bab-4702-90a0-b259b5d3217c.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json b/evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json index 8409ecc..980aa57 100644 --- a/evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json +++ b/evaluation_examples/examples/libreoffice_calc/6e99a1ad-07d2-4b66-a1ce-ece6d99c20a5.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json b/evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json index 1f614cb..7bf9cef 100644 --- a/evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json +++ b/evaluation_examples/examples/libreoffice_calc/7a4e4bc8-922c-4c84-865c-25ba34136be1.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json b/evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json index 1f167e5..c06cde7 100644 --- a/evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json +++ b/evaluation_examples/examples/libreoffice_calc/7e429b8d-a3f0-4ed0-9b58-08957d00b127.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json b/evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json index b3c98a2..43fcb6b 100644 --- a/evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json +++ b/evaluation_examples/examples/libreoffice_calc/7efeb4b1-3d19-4762-b163-63328d66303b.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json b/evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json index 084bab4..1016e37 100644 --- a/evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json +++ b/evaluation_examples/examples/libreoffice_calc/8b1ce5f2-59d2-4dcc-b0b0-666a714b9a14.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json index 6943b91..b19b14a 100644 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json index 9844796..e7bc03b 100644 --- a/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json +++ b/evaluation_examples/examples/libreoffice_calc/a01fbce3-2793-461f-ab86-43680ccbae25.json.json @@ -29,7 +29,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }, diff --git a/evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json b/evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json index ba797f0..8b50b36 100644 --- a/evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json +++ b/evaluation_examples/examples/libreoffice_calc/a9f325aa-8c05-4e4f-8341-9e4358565f4f.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json b/evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json index 4683d1d..e3d8cd1 100644 --- a/evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json +++ b/evaluation_examples/examples/libreoffice_calc/abed40dc-063f-4598-8ba5-9fe749c0615d.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json b/evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json index 21cb7df..111f16c 100644 --- a/evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json +++ b/evaluation_examples/examples/libreoffice_calc/d681960f-7bc3-4286-9913-a8812ba3261a.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json b/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json index 080e333..d2a30ab 100644 --- a/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json +++ b/evaluation_examples/examples/libreoffice_calc/eb03d19a-b88d-4de4-8a64-ca0ac66f426b.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.hotkey([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json b/evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json index 2bc7183..3ba14cf 100644 --- a/evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json +++ b/evaluation_examples/examples/libreoffice_calc/ecb0df7a-4e8d-4a03-b162-053391d3afaf.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } } diff --git a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json index f538979..b522e4d 100644 --- a/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json +++ b/evaluation_examples/examples/libreoffice_calc/f9584479-3d0d-4c79-affa-9ad7afdd8850.json @@ -47,7 +47,7 @@ "command": [ "python", "-c", - "import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" ] } }