From 0fcdbf63d154e5e7403193255c4936d7f610ba88 Mon Sep 17 00:00:00 2001 From: Liu Yitao Date: Fri, 26 Jan 2024 04:53:41 -0500 Subject: [PATCH 01/16] Update tasks --- .../5812b315-e7bd-4265-b51f-863c02174c28.json | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json b/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json index f9930e8..0f5446e 100644 --- a/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json +++ b/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json @@ -1,18 +1,33 @@ { "id": "5812b315-e7bd-4265-b51f-863c02174c28", "snapshot": "os", - "instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".", + "instruction": "Please create an SSH user named \"charles\" with password \"Ex@mpleP@55w0rd!\" on Ubuntu who is only allowed to access the folder \"test1\".", "source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": "mkdir /test1", + "shell": true + } + } + ], "trajectory": "trajectories/", "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "[[ $(getent passwd charles) && $(getent passwd charles | cut -d: -f6) = \"/test1\" ]] && echo \"User charles exists and has /test1 as their home directory\" || echo \"User charles does not exist or doesn't have /test1 as their home directory\"", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "User charles exists and has /test1 as their home directory" + } } } } \ No newline at end of file From 9e91b8a5a8a57585fa6258dc3e475f0a1e22145c Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 30 Jan 2024 00:25:00 +0800 Subject: [PATCH 02/16] ver Jan29thv2 check som implementation --- experiment_screenshot_som.py | 4 ++-- mm_agents/gpt_4v_agent.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/experiment_screenshot_som.py b/experiment_screenshot_som.py index a8682ba..8247ad8 100644 --- a/experiment_screenshot_som.py +++ b/experiment_screenshot_som.py @@ -153,7 +153,7 @@ def main(example_class, example_id): if __name__ == '__main__': - xx_list = [ "01b269ae-2111-4a07-81fd-3fcd711993b0" + xx_list = [ "35253b65-1c19-4304-8aa4-6884b8218fc0" ] for example_id in xx_list: - main("libreoffice_calc", example_id) + main("thunderbird", example_id) diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py index ff8d0d0..4fb52ea 100644 --- a/mm_agents/gpt_4v_agent.py +++ b/mm_agents/gpt_4v_agent.py @@ -278,6 +278,7 @@ class GPT4v_Agent: elif self.exp in ["som", "seeact"]: _screenshot = previous_obs["screenshot"] _linearized_accessibility_tree = previous_obs["accessibility_tree"] + logger.debug("LINEAR AT: %s", _linearized_accessibility_tree) messages.append({ "role": "user", From 8edd3b9aad2cf4d47cd983c88e30d7a997599cc6 Mon Sep 17 00:00:00 2001 From: Liu Yitao Date: Mon, 29 Jan 2024 18:22:13 -0500 Subject: [PATCH 03/16] Fix examples --- .../c56de254-a3ec-414e-81a6-83d2ce8c41fa.json | 23 ++++++++++++++++--- .../cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json | 22 +++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json index 27abbf7..7583dab 100644 --- a/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json +++ b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json @@ -1,17 +1,34 @@ { "id": "c56de254-a3ec-414e-81a6-83d2ce8c41fa", "snapshot": "os", - "instruction": "Could you please help me to extract text from a non-indexed PDF document on the current system?", + "instruction": "I want to install the lxml package on my ubuntu system. Can you help me?", "source": "https://superuser.com/questions/28426/how-to-extract-text-with-ocr-from-a-pdf-on-linux", + "config": [ + { + "type": "execute", + "parameters": { + "command": "sudo apt-get update && sudo apt-get install -y python3", + "shell": true + } + } + ], "trajectory": "trajectories/", "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "python3 -c \"import lxml\" >/dev/null 2>&1 && echo \"lxml is installed\" || echo \"lxml is not installed\"", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "lxml is installed" + } } } - } \ No newline at end of file + } + \ No newline at end of file diff --git a/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json b/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json index b8d4d30..f1ff95a 100644 --- a/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json +++ b/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json @@ -1,17 +1,33 @@ { "id": "cc9d4f34-1ca0-4a1b-8ff2-09302696acb9", "snapshot": "os", - "instruction": "I need you to execute a specific process or script from the terminal and ensure that it continues to run independently, even after the terminal session is terminated. This involves redirecting the process's output to a file and disassociating the process from the terminal session's control.", - "source": "https://superuser.com/questions/178587/how-do-i-detach-a-process-from-terminal-entirely", + "instruction": "Currently, the program my_process.sh is running in the system and I want to end the process. Can you help me?", + "source": "https://stackoverflow.com/questions/9346211/how-to-kill-a-process-on-a-port-on-ubuntu", + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo -e '#!/bin/bash\n\nwhile true; do\necho \"Running my_process...\"\nsleep 1\ndone' > my_process.sh && chmod +x my_process.sh && nohup ./my_process.sh > /dev/null 2>&1 &", + "shell": true + } + } + ], "trajectory": "trajectories/", "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "[[ -z $(ps -ef | grep -v grep | grep my_process) ]] && echo \"Process killed\" || echo \"Process still running\"", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "Process killed" + } } } } \ No newline at end of file From 2df7de25aba69a99c2cd2653e46831ac874451a4 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Tue, 30 Jan 2024 11:32:36 +0800 Subject: [PATCH 04/16] add multi-app examples --- desktop_env/evaluators/getters/__init__.py | 2 +- desktop_env/evaluators/getters/vlc.py | 28 ++++- desktop_env/evaluators/getters/vscode.py | 3 +- .../2b9493d7-49b8-493a-a71b-56cd1f4d6908.json | 59 ++++----- .../2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json | 51 +++++--- .../510f64c8-9bcc-4be1-8d30-638705850618.json | 116 ++++++++++++++++++ .../937087b6-f668-4ba6-9110-60682ee33441.json | 26 ++++ .../53ad5833-3455-407b-bbc6-45b4c79ab8fb.json | 21 ++++ 8 files changed, 246 insertions(+), 60 deletions(-) create mode 100644 evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json create mode 100644 evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index bb30d12..3535ddc 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -21,5 +21,5 @@ from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay -from .vlc import get_vlc_playing_info, get_vlc_config +from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player from .vscode import get_vscode_config diff --git a/desktop_env/evaluators/getters/vlc.py b/desktop_env/evaluators/getters/vlc.py index 37b00b2..ef0adfb 100644 --- a/desktop_env/evaluators/getters/vlc.py +++ b/desktop_env/evaluators/getters/vlc.py @@ -1,7 +1,8 @@ import logging import os from typing import Dict - +from collections import Counter +from .general import get_vm_command_line import requests logger = logging.getLogger("desktopenv.getters.vlc") @@ -58,3 +59,28 @@ def get_vlc_config(env, config: Dict[str, str]): f.write(content) return _path + + +def get_default_video_player(env, config: dict): + """ Gets the default application for a category or file extension. + """ + + os_type = env.vm_platform + + if os_type == "Linux": + extensions = ['3gp', '3gp', '3gpp', '3gpp', '3gpp2', '3gpp2', 'avi', 'avi', 'divx', 'divx', 'dv', 'dv', 'fli', 'fli', 'flv', 'flv', 'mp2t', 'mp2t', 'mp4', 'mp4', 'mp4v-es', 'mp4v-es', 'mpeg', 'mpeg', 'mpeg-system', 'mpeg-system', 'msvideo', 'msvideo', 'ogg', 'ogg', 'quicktime', 'quicktime', 'vnd.divx', 'vnd.divx', 'vnd.mpegurl', 'vnd.mpegurl', 'vnd.rn-realvideo', 'vnd.rn-realvideo', 'webm', 'webm', 'x-anim', 'x-anim', 'x-avi', 'x-avi', 'x-flc', 'x-flc', 'x-fli', 'x-fli', 'x-flv', 'x-flv', 'x-m4v', 'x-m4v', 'x-matroska', 'x-matroska', 'x-mpeg', 'x-mpeg', 'x-mpeg-system', 'x-mpeg-system', 'x-mpeg2', 'x-mpeg2', 'x-ms-asf', 'x-ms-asf', 'x-ms-asf-plugin', 'x-ms-asf-plugin', 'x-ms-asx', 'x-ms-asx', 'x-ms-wm', 'x-ms-wm', 'x-ms-wmv', 'x-ms-wmv', 'x-ms-wmx', 'x-ms-wmx', 'x-ms-wvx', 'x-ms-wvx', 'x-msvideo', 'x-msvideo', 'x-nsv', 'x-nsv', 'x-ogm', 'x-ogm', 'x-ogm+ogg', 'x-theora', 'x-theora', 'x-theora+ogg', 'x-theora+ogg'] + apps = [] + for ext in extensions: + app = get_vm_command_line(env, {"command": ["xdg-mime", "query", "default", f"video/{ext}"]}) + if app: + apps.append(app) + if len(apps) == 0: + return 'unknown' + else: + return Counter(apps).most_common(1)[0][0] + elif os_type == "Darwin": + raise Exception("Unsupported operating system", os_type) + elif os_type == "Windows": + raise Exception("Unsupported operating system", os_type) + else: + raise Exception("Unsupported operating system", os_type) \ No newline at end of file diff --git a/desktop_env/evaluators/getters/vscode.py b/desktop_env/evaluators/getters/vscode.py index 8a725ef..bf8f351 100644 --- a/desktop_env/evaluators/getters/vscode.py +++ b/desktop_env/evaluators/getters/vscode.py @@ -1,6 +1,6 @@ import logging from typing import Any, Dict - +import time from .file import get_vm_file from .replay import get_replay @@ -27,6 +27,7 @@ def get_vscode_config(env, config: Dict[str, Any]) -> str: ] get_replay(env, trajectory) + time.sleep(1.0) return get_vm_file(env, { "path": config["path"], diff --git a/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json index d83ed1b..99e148b 100644 --- a/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json +++ b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json @@ -36,7 +36,8 @@ "parameters": { "command": [ "gnome-terminal", - "--maximize" + "--maximize", + "--working-directory=/home/user/" ] } } @@ -59,43 +60,27 @@ } } ], - "func": ["check_include_exclude", "check_include_exclude"], - "conj": "and", - "result": [ - { - "type": "vm_command_line", - "command": [ - "/bin/bash", - "-c", - "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi" - ] - }, - { - "type": "vm_command_line", - "command": [ - "/bin/bash", - "-c", - "output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi" + "func": "check_include_exclude", + "result": { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"no libreoffice is running\"; else echo \"libreoffice is still running\"; fi; output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"not killed from terminal\"; else echo \"killed from terminal\"; fi" + ] + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + "no libreoffice is running", + "killed from terminal" + ], + "exclude": [ + "libreoffice is still running", + "not killed from terminal" ] } - ], - "expected": [ - { - "type": "rule", - "rules": { - "include": [ - "true\n" - ] - } - }, - { - "type": "rule", - "rules": { - "include": [ - "true\n" - ] - } - } - ] + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json index 3926120..e3eac08 100644 --- a/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json +++ b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json @@ -6,15 +6,12 @@ "config": [ { "type": "download", - "path": "", - }, - { - "type": "execute", "parameters": { - "command": [ - "/bin/bash", - "-c", - "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git" + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1aANBCEHT6K8GmHDMEYtL1LQXdApqVoGv&export=download", + "path": "/home/user/Downloads/binder.zip" + } ] } }, @@ -24,7 +21,17 @@ "command": [ "/bin/bash", "-c", - "mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md" + "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main && git config receive.denyCurrentBranch ignore " + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "unzip -q /home/user/Downloads/binder.zip -d /home/user/projects/ && cd /home/user/projects/binder && git init --initial-branch main && git remote add origin /home/user/projects/remote_project" ] } }, @@ -34,7 +41,7 @@ "command": [ "gnome-terminal", "--maximize", - "--working-directory=/home/user/projects/hello_world" + "--working-directory=/home/user/projects/binder" ] } } @@ -45,19 +52,23 @@ "terminal" ], "evaluator": { - "func": "compare_docx_files", + "func": "check_include_exclude", "result": { - "type": "vm_file", - "path": "/home/user/Desktop/notes.docx", - "dest": "notes.docx" + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "cd /home/user/projects/remote_project; git log --oneline | head -n 1 | awk '{for (i=2; i<=NF; i++) {printf \"%s%s\", $i, (i==NF ? \"\" : \" \")}; print \"\"}'; remote_id=$(git log --oneline | head -n 1 | awk '{print $1}'); cd /home/user/projects/binder; local_id=$(git log --oneline | head -n 1 | awk '{print $1}'); if [ \"${local_id}\" = \"${remote_id}\" ]; then echo \"repo is synchronous\"; else echo \"repo is not synchronous\"; fi" + ] }, "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download", - "dest": "notes_gold.docx" - }, - "options": { - "ignore_blanks": true + "type": "rule", + "rules": { + "include": [ + "daily update", + "repo is synchronous" + ] + } } } } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json b/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json new file mode 100644 index 0000000..de6e9f4 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json @@ -0,0 +1,116 @@ +{ + "id": "510f64c8-9bcc-4be1-8d30-638705850618", + "snapshot": "vscode", + "instruction": "Could you start VS Code in folder ~/Desktop/project from the terminal?", + "source": "https://www.geeksforgeeks.org/how-to-start-vs-code-from-the-terminal-command-line/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download", + "path": "/home/user/Downloads/vscodeEvalExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip && mkdir -p ~/Desktop/project/.vscode && history -c && echo > ~/.bash_history" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052", + "path": "/home/user/Desktop/project/main.py" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290", + "path": "/home/user/Desktop/project/README.md" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443", + "path": "/home/user/Desktop/project/.vscode/settings.json" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize", + "--working-directory=/home/user/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode", + "terminal" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "killall gnome-terminal-server" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "func": ["check_include_exclude", "compare_config"], + "result": [ + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "use_terminal=$(cat ~/.bash_history | grep \"[c]ode \"); if [ -z \"$use_terminal\" ]; then echo \"false\"; else echo \"true\"; fi" + ] + }, + { + "type": "vscode_config", + "vscode_extension_command": "OpenProject", + "path": "/home/user/OpenProject.txt", + "dest": "OpenProject.txt" + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "include": [ + "true" + ] + } + }, + { + "type": "rule", + "rules": { + "expect": "project" + } + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json b/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json new file mode 100644 index 0000000..58c4d6d --- /dev/null +++ b/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json @@ -0,0 +1,26 @@ +{ + "id": "937087b6-f668-4ba6-9110-60682ee33441", + "snapshot": "vlc", + "instruction": "I am currently using a ubuntu system. Could you help me set the default video player as VLC?", + "source": "https://superuser.com/questions/187440/set-default-ubuntu-video-player-as-vlc", + "config": [], + "trajectory": "trajectories/", + "related_apps": [ + "vlc", + "os" + ], + "evaluator": { + "func": "check_include_exclude", + "result": { + "type": "default_video_player" + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + "vlc.desktop" + ] + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index 55dfb09..b597a03 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -4,6 +4,27 @@ "instruction": "Please help me use VS Code to open the \"project\" in the \"user\" folder under \"home\".", "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download", + "path": "/home/user/Downloads/vscodeEvalExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip" + ] + } + }, { "type": "launch", "parameters": { From 8db2eca8d0219551c7febdd4b94ff959babb7504 Mon Sep 17 00:00:00 2001 From: BlankCheng <913501223@qq.com> Date: Tue, 30 Jan 2024 14:35:49 +0800 Subject: [PATCH 05/16] Fix triangle position --- desktop_env/evaluators/metrics/gimp.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 8269892..a279d23 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -270,14 +270,13 @@ def check_triangle_position(tgt_path): img = Image.open(tgt_path) img_array = np.array(img) - # We will determine if the triangle is in the middle of the picture by checking the centroid # We assume the triangle is a different color from the background # Find the unique colors - unique_colors = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0) + unique_colors, counts = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0, return_counts=True) + unique_colors_sorted = unique_colors[np.argsort(counts)] - # Assuming the background is the most common color and the triangle is a different color, - # we identify the triangle's color as the least common one - triangle_color = unique_colors[-1] + # Assuming the background is the most common color and the triangle is a different color + triangle_color = unique_colors_sorted[1] # Create a mask where the triangle pixels are True triangle_mask = np.all(img_array == triangle_color, axis=2) @@ -481,3 +480,7 @@ if __name__ == "__main__": src_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/green_background_with_object.png" tgt_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/white_background_with_object.png" print(check_green_background(src_path, tgt_path)) + + tgt_path = "../../../cache/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce/Triangle_In_The_Middle.png" + print(check_triangle_position(tgt_path)) + From 8885ea0aefac5bd4565dd73d64a6e98147767a06 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Tue, 30 Jan 2024 15:50:12 +0800 Subject: [PATCH 06/16] Change the bool into float for printing; add path check in gimp --- desktop_env/evaluators/metrics/gimp.py | 50 +++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 8d976f6..5ad41ba 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -197,6 +197,9 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path): Check the brightness of src is lower than tgt and the structures are similar gimp:7a4deb26-d57d-4ea9-9a73-630f66a7b568 """ + if src_path is None or tgt_path is None: + return 0. + img_src = Image.open(src_path) img_tgt = Image.open(tgt_path) @@ -222,6 +225,9 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path): Check the saturation of src is higher than tgt and the structures are similar gimp:554785e9-4523-4e7a-b8e1-8016f565f56a """ + if src_path is None or tgt_path is None: + return 0. + img_src = Image.open(src_path) hsv_img_src = img_src.convert('HSV') img_tgt = Image.open(tgt_path) @@ -254,17 +260,23 @@ def check_file_exists_and_structure_sim(src_path, tgt_path): Check if the image has been exported to the desktop gimp:77b8ab4d-994f-43ac-8930-8ca087d7c4b4 """ + if src_path is None or tgt_path is None: + return 0. + # Check if the file exists export_file_exists = os.path.isfile(src_path) if not export_file_exists: - return False + return 0. # Check whether the target image is the same as the source image img_src = Image.open(src_path) img_tgt = Image.open(tgt_path) structure_same = structure_check_by_ssim(img_src, img_tgt) - return structure_same + if structure_same: + return 1. + else: + return 0. def check_triangle_position(tgt_path): @@ -272,6 +284,9 @@ def check_triangle_position(tgt_path): Check if the triangle is in the middle of the image. gimp:f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce """ + if tgt_path is None: + return 0. + # Load the image img = Image.open(tgt_path) img_array = np.array(img) @@ -322,6 +337,9 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path): Check if the src image has higher contrast than the tgt image and the structures are similar gimp:f723c744-e62c-4ae6-98d1-750d3cd7d79d """ + if src_path is None or tgt_path is None: + return 0. + # Load images source_image = Image.open(src_path) target_image = Image.open(tgt_path) @@ -344,6 +362,9 @@ def check_config_status(actual_config_path, rule): """ Check if the GIMP status is as expected """ + if actual_config_path is None: + return 0. + with open(actual_config_path, 'r') as f: content = f.readlines() @@ -367,6 +388,10 @@ def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=Non Check if the size of the src image is correct and the structure of the two images are similar. gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15 """ + + if src_path is None or tgt_path is None: + return 0. + # Load images source_image = Image.open(src_path) target_image = Image.open(tgt_path) @@ -396,6 +421,9 @@ def check_palette_and_structure_sim(src_path, tgt_path): Check if the src image is palette-based and the structure of the two images are similar gimp:06ca5602-62ca-47f6-ad4f-da151cde54cc """ + if src_path is None or tgt_path is None: + return 0. + # Check if the source image is palette-based source_image = Image.open(src_path) palette_based = source_image.mode == 'P' @@ -415,6 +443,9 @@ def check_textbox_on_leftside(src_path): Check if the textbox is on the left side of the image. gimp:e2dd0213-26db-4349-abe5-d5667bfd725c """ + if src_path is None: + return 0. + source_image = Image.open(src_path) gray_image = source_image.convert("L") width, height = source_image.size @@ -440,6 +471,9 @@ def check_image_mirror(src_path, tgt_path): Check if the image is mirrored gimp:72f83cdc-bf76-4531-9a1b-eb893a13f8aa """ + if src_path is None or tgt_path is None: + return 0. + # Load images source_image = Image.open(src_path) target_image = Image.open(tgt_path) @@ -448,7 +482,10 @@ def check_image_mirror(src_path, tgt_path): transposed_image = source_image.transpose(Image.FLIP_LEFT_RIGHT) # Use 0.99 because the image may not be exactly mirrored by gimp mirrored = structure_check_by_ssim(transposed_image, target_image, 0.99) - return mirrored + if mirrored: + return 1. + else: + return 0. def check_green_background(src_path, tgt_path): @@ -456,6 +493,9 @@ def check_green_background(src_path, tgt_path): Check if the background of the source image is green. gimp:734d6579-c07d-47a8-9ae2-13339795476b """ + if src_path is None or tgt_path is None: + return 0. + # Load images source_image = Image.open(src_path) target_image = Image.open(tgt_path) @@ -471,9 +511,9 @@ def check_green_background(src_path, tgt_path): # Here, "green" means more green than red or blue r, g, b = source_pixels[x, y][:3] if not (g > r and g > b): - return False + return 0. - return True + return 1. if __name__ == "__main__": From d5d9fc56debea1b1440f9054a46c3815881d46d4 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Tue, 30 Jan 2024 18:48:00 +0800 Subject: [PATCH 07/16] Fix minor bugs of get_terminal output caused by a11y tree depth --- desktop_env/evaluators/metrics/general.py | 8 +++++++- desktop_env/server/main.py | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 3f66f02..4e1f50a 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -17,10 +17,16 @@ from .utils import _match_record, _match_value_to_rule def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float: + if result is None: + return 0. + print(result, rules) include = rules.get("include", []) exclude = rules.get("exclude", []) - return all(r in result for r in include) and all(r not in result for r in exclude) + if all(r in result for r in include) and all(r not in result for r in exclude): + return 1. + else: + return 0. def exact_match(result, rules) -> float: diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 9ed1295..d470a9d 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -176,8 +176,9 @@ def get_terminal_output(): # raise NotImplementedError return "Currently not implemented for platform {:}.".format(platform.platform()), 500 return jsonify({"output": output, "status": "success"}) - except: - return jsonify({"output": None, "status": "error"}) + except Exception as e: + logger.error("Failed to get terminal output. Error: %s", e) + return jsonify({"status": "error", "message": str(e)}), 500 _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" @@ -191,7 +192,7 @@ _accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" } -def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element: +def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = None) -> _Element: if node.getRoleName() == "document spreadsheet": flag = "calc" if node.getRoleName() == "application" and node.name=="Thunderbird": From 4726ea7edb4c69fca311d8bb8cde133f9ea163d5 Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Tue, 30 Jan 2024 19:24:29 +0800 Subject: [PATCH 08/16] update multi-app examples --- desktop_env/evaluators/metrics/__init__.py | 2 +- desktop_env/evaluators/metrics/gimp.py | 2 +- .../3680a5ee-6870-426a-a997-eba929a0d25c.json | 71 ++++++++++++++ .../46407397-a7d5-4c6b-92c6-dbe038b1457b.json | 6 +- .../b52b40a5-ad70-4c53-b5b0-5650a8387052.json | 6 +- .../ee9a3c83-f437-4879-8918-be5efbb9fac7.json | 95 +++++++++++++++++++ .../settings/googledrive/credentials.json | 2 +- 7 files changed, 175 insertions(+), 9 deletions(-) create mode 100644 evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json create mode 100644 evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index 82bce50..6e327f1 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -66,7 +66,7 @@ from .gimp import ( check_triangle_position, check_structure_sim, check_config_status, - compare_images, + compare_image_list, increase_saturation, decrease_brightness, check_file_exists, diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 5ad41ba..fadf265 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -4,7 +4,7 @@ from skimage.metrics import structural_similarity as ssim from PIL import Image, ImageChops, ImageStat -def compare_images(pred_img_path_list: Union[str, List[str]], +def compare_image_list(pred_img_path_list: Union[str, List[str]], gold_img_path_list: Union[str, List[str]]) -> float: """ Compare two image lists, only if all images are the same, return 1.0, otherwise return 0.0 """ diff --git a/evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json b/evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json new file mode 100644 index 0000000..2c3df48 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/3680a5ee-6870-426a-a997-eba929a0d25c.json @@ -0,0 +1,71 @@ +{ + "id": "3680a5ee-6870-426a-a997-eba929a0d25c", + "snapshot": "libreoffice_calc", + "instruction": "I have file1.xlsx and file2.ods on the Desktop and each has one column. Help me use only the command line to merge these two columns into one LibreOffice Calc file called output.csv and open it from terminal.", + "source": "https://unix.stackexchange.com/questions/510850/how-to-open-calc-from-terminal-and-insert-files", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1ofMPSBPXtt0h6t0arjYo0GMtKAAhfEPz&export=download", + "path": "/home/user/Desktop/file1.xlsx" + }, + { + "url": "https://drive.usercontent.google.com/download?id=15QOxr0_MiHR80ppMX02aHnnsWs25aXgZ&export=download", + "path": "/home/user/Desktop/file2.ods" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize", + "--working-directory=/home/user/Desktop" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "terminal" + ], + "evaluator": { + "func": ["check_include_exclude", "compare_csv"], + "result": [ + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "output=$(ps aux | grep \"[s]office\" | awk '{print $7}' | grep -E \"pts/|tty\"); if [ -z \"$output\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi;" + ] + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/output.csv", + "dest": "output.csv" + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "include": [ + "use terminal" + ] + } + }, + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1OBQXIJ06HQeodaCwU7z0Kq_-d3YuGNqE&export=download", + "dest": "output_gold.csv" + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json index 0bf4bcb..79b1ecc 100644 --- a/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json +++ b/evaluation_examples/examples/multi_apps/46407397-a7d5-4c6b-92c6-dbe038b1457b.json @@ -1,7 +1,7 @@ { "id": "46407397-a7d5-4c6b-92c6-dbe038b1457b", "snapshot": "chrome", - "instruction": "Help me export charts, graph or other images from docx files received in email xxx in Thunderbird and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).", + "instruction": "Help me export charts, graph or other images from docx files received in email \"Lecture Document\" in Notes folder and upload these png files to the figures/ folder in Google Drive for later use (use numbers to name them).", "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", "config": [ { @@ -58,7 +58,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "url": "https://drive.usercontent.google.com/download?id=18jdi0OanMtAQenm4ODTivsxTSzdj4HUV&export=download&authuser=0&confirm=t&uuid=e858d3cc-4535-4419-a651-8856ac517d19&at=APZUnTW7g4ygfrkKTPBWCO13twRj:1706611460571", "path": "/home/user/thunderbird-profile.tar.gz" } ] @@ -93,7 +93,7 @@ "chrome" ], "evaluator": { - "func": "compare_images", + "func": "compare_image_list", "result": { "type": "googledrive_file", "settings_file": "evaluation_examples/settings/googledrive/settings.yml", diff --git a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json index 9440ddb..340927a 100644 --- a/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json +++ b/evaluation_examples/examples/multi_apps/b52b40a5-ad70-4c53-b5b0-5650a8387052.json @@ -1,7 +1,7 @@ { "id": "b52b40a5-ad70-4c53-b5b0-5650a8387052", "snapshot": "chrome", - "instruction": "Could you help me merge all PDF files in the email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?", + "instruction": "Could you help me merge all PDF files in the \"Paper Recommendation\" email attachment in Thunderbird into one file and upload it to attachment_full.pdf in Google Drive?", "source": "https://marketplace.uipath.com/listings/merge-pdfs-from-gmail-email-attachments-and-upload-to-gogle-drive", "config": [ { @@ -58,7 +58,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=11b93787-7076-47ba-b04b-b63a4e9aab02&at=APZUnTV_3ASC-R55FmBXgTLcC46e:1706187828620", + "url": "https://drive.usercontent.google.com/download?id=1_SujjgvE6SkrIINB7bd-4quti7ICGaB_&export=download&authuser=0&confirm=t&uuid=aa95822d-55ca-46e0-8b52-14d495b41995&at=APZUnTX7f8cbJ00STW_kV8kMY1KW:1706609819112", "path": "/home/user/thunderbird-profile.tar.gz" } ] @@ -104,7 +104,7 @@ }, "expected": { "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=b0ad08b7-5b4e-4372-99fa-0952cd096144&at=APZUnTVrZK9_alT_gchTKE6ZYeod:1706194464805", + "path": "https://drive.usercontent.google.com/download?id=1-FaONI6f5g9XyJAlx8vPFgTP_SEZ-1vV&export=download&authuser=0&confirm=t&uuid=de7b274a-25b8-4361-a614-4910a88ba4bd&at=APZUnTVRw21zrIY4ydp9sagr9ZhZ:1706609799766", "dest": "attachment_full_gold.pdf" } } diff --git a/evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json b/evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json new file mode 100644 index 0000000..55c7a6c --- /dev/null +++ b/evaluation_examples/examples/multi_apps/ee9a3c83-f437-4879-8918-be5efbb9fac7.json @@ -0,0 +1,95 @@ +{ + "id": "ee9a3c83-f437-4879-8918-be5efbb9fac7", + "snapshot": "libreoffice_calc", + "instruction": "Could you help me convert the opened ods file in the desktop to csv file with the same file name using command line when Libreoffice instance is running?", + "source": "https://stackoverflow.com/questions/64589140/convert-ods-to-csv-using-command-line-when-libreoffice-instance-is-running", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1V6e4KB6Sabp6nAHkq5Seun0qbjwEfdaF&export=download", + "path": "/home/user/Desktop/file_example_ODS_5000.ods" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "/home/user/Desktop/file_example_ODS_5000.ods" + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "history -c && echo > ~/.bash_history && sleep 3" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize", + "--working-directory=/home/user/Desktop" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "libreoffice_calc", + "terminal" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "killall gnome-terminal-server" + ] + } + } + ], + "func": ["check_include_exclude", "compare_csv"], + "result": [ + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "use_terminal=$(cat ~/.bash_history | grep \"\\(soffice\\|libreoffice\\).\\+--convert-to\\s\\+csv\"); if [ -z \"$use_terminal\" ]; then echo \"use no terminal\"; else echo \"use terminal\"; fi" + ] + }, + { + "type": "vm_file", + "path": "/home/user/Desktop/file_example_ODS_5000.csv", + "dest": "file_example_ODS_5000.csv" + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "include": [ + "use terminal" + ] + } + }, + { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1L8cJjHYDtZymQp4-QdNZ9i6jzZzsXJm1&export=download", + "dest": "file_example_ODS_5000_gold.csv" + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/settings/googledrive/credentials.json b/evaluation_examples/settings/googledrive/credentials.json index ef8905d..5235e65 100644 --- a/evaluation_examples/settings/googledrive/credentials.json +++ b/evaluation_examples/settings/googledrive/credentials.json @@ -1 +1 @@ -{"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-29T05:13:41Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byB77Ran1kP3F1FKu9xL-zMeffAX-m3Z8JFvP2UD6iMM8_s4FQoNlOK2gstGSrW0G9seRlOmDG129Qq6XwI5BiwfxB1ZDGUKuikOYl6ZFgS69tzNXXzuKbLUivkQqoBZl28njdWUsVBFKjy_IvFzlDQAC6-YOrPkPAaCgYKAVwSARISFQHGX2Mi8GhWBz1GC2iqsEtbet6ETA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file +{"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "client_id": "786888752612-6cv6lermep9n6704s4kv20h08lotias9.apps.googleusercontent.com", "client_secret": "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", "refresh_token": "1//0e0qXy4xW1Ud5CgYIARAAGA4SNwF-L9IrWfaomed_CK0R7zZffcpT-GIXf3y2ZjqqAD0UP6UkbaMV9F_OEC6pBVaaX4TYnBKx3os", "token_expiry": "2024-01-30T11:20:53Z", "token_uri": "https://oauth2.googleapis.com/token", "user_agent": null, "revoke_uri": "https://oauth2.googleapis.com/revoke", "id_token": null, "id_token_jwt": null, "token_response": {"access_token": "ya29.a0AfB_byBdrcgbmWKGyEUOxxuJBhxOs1uu0AqWeLgJKKKLG_dVg4iQKJAdiWD2oetHrKn17p4ZtfX-vt1VQ0BiF2MPD2exX1oESsQkXaO8q4TM1olIpadvlUBqUkqKJcjCqn1dp1oaTVYU-Srf2wQCGnDt3ozjljdkHXN_MQaCgYKAa4SARISFQHGX2MiWnixlrP3Se3vEV73_4fenA0173", "expires_in": 3599, "scope": "https://www.googleapis.com/auth/drive", "token_type": "Bearer"}, "scopes": ["https://www.googleapis.com/auth/drive"], "token_info_uri": "https://oauth2.googleapis.com/tokeninfo", "invalid": false, "_class": "OAuth2Credentials", "_module": "oauth2client.client"} \ No newline at end of file From da306376da825834b83acc60c5dfd3af229dc577 Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 30 Jan 2024 20:06:58 +0800 Subject: [PATCH 09/16] ver Jan30th updated function to get AT on Windows --- desktop_env/server/main.py | 203 +++++++++++++++++++++++++++++++++++-- mm_agents/gpt_4v_agent.py | 4 + 2 files changed, 196 insertions(+), 11 deletions(-) diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index eef0afc..9f57659 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -21,6 +21,8 @@ from pyatspi import Action as ATAction from pyatspi import Component, Document from pyatspi import Text as ATText from pyatspi import Value as ATValue +from pywinauto import Desktop +from pywinauto.base_wrapper import BaseWrapper from pyxcursor import Xcursor @@ -170,18 +172,20 @@ def get_terminal_output(): return jsonify({"output": None, "status": "error"}) -_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } +_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org" + , "attr": "uri:deskat:attributes.at-spi.gnome.org" + , "cp": "uri:deskat:component.at-spi.gnome.org" + , "doc": "uri:deskat:document.at-spi.gnome.org" + , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" + , "txt": "uri:deskat:text.at-spi.gnome.org" + , "val": "uri:deskat:value.at-spi.gnome.org" + , "act": "uri:deskat:action.at-spi.gnome.org" + , "win": "uri:deskat:uia.windows.microsoft.org" + } -def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) -> _Element: +def _create_atspi_node(node: Accessible, depth: int = 0, flag: Optional[str] = None) -> _Element: + # function _create_atspi_node {{{ # if node.getRoleName() == "document spreadsheet": flag = "calc" if node.getRoleName() == "application" and node.name=="Thunderbird": @@ -370,6 +374,175 @@ def _create_atspi_node(node: Accessible, depth: int, flag: Optional[str] = None) break xml_node.append(_create_atspi_node(ch, depth+1, flag)) return xml_node + # }}} function _create_atspi_node # + +def _create_pywinauto_node(node: BaseWrapper, depth: int = 0, flag: Optional[str] = None) -> _Element: + # function _create_pywinauto_node {{{ # + #element_info: ElementInfo = node.element_info + attribute_dict: Dict[str, Any] = {"name": node.element_info.name} + + # States {{{ # + attribute_dict["{{{:}}}enabled".format(_accessibility_ns_map["st"])] = str(node.is_enabled()).lower() + attribute_dict["{{{:}}}visible".format(_accessibility_ns_map["st"])] = str(node.is_visible()).lower() + attribute_dict["{{{:}}}active".format(_accessibility_ns_map["st"])] = str(node.is_active()).lower() + + if hasattr(node, "is_minimized"): + try: + attribute_dict["{{{:}}}minimized".format(_accessibility_ns_map["st"])] = str(node.is_minimized()).lower() + except: + pass + if hasattr(node, "is_maximized"): + try: + attribute_dict["{{{:}}}maximized".format(_accessibility_ns_map["st"])] = str(node.is_maximized()).lower() + except: + pass + if hasattr(node, "is_normal"): + try: + attribute_dict["{{{:}}}normal".format(_accessibility_ns_map["st"])] = str(node.is_normal()).lower() + except: + pass + + if hasattr(node, "is_unicode"): + try: + attribute_dict["{{{:}}}unicode".format(_accessibility_ns_map["st"])] = str(node.is_unicode()).lower() + except: + pass + + if hasattr(node, "is_collapsed"): + try: + attribute_dict["{{{:}}}collapsed".format(_accessibility_ns_map["st"])] = str(node.is_collapsed()).lower() + except: + pass + if hasattr(node, "is_checkable"): + try: + attribute_dict["{{{:}}}checkable".format(_accessibility_ns_map["st"])] = str(node.is_checkable()).lower() + except: + pass + if hasattr(node, "is_checked"): + try: + attribute_dict["{{{:}}}checked".format(_accessibility_ns_map["st"])] = str(node.is_checked()).lower() + except: + pass + if hasattr(node, "is_focused"): + try: + attribute_dict["{{{:}}}focused".format(_accessibility_ns_map["st"])] = str(node.is_focused()).lower() + except: + pass + if hasattr(node, "is_keyboard_focused"): + try: + attribute_dict["{{{:}}}keyboard_focused".format(_accessibility_ns_map["st"])] = str(node.is_keyboard_focused()).lower() + except: + pass + if hasattr(node, "is_selected"): + try: + attribute_dict["{{{:}}}selected".format(_accessibility_ns_map["st"])] = str(node.is_selected()).lower() + except: + pass + if hasattr(node, "is_selection_required"): + try: + attribute_dict["{{{:}}}selection_required".format(_accessibility_ns_map["st"])] = str(node.is_selection_required()).lower() + except: + pass + if hasattr(node, "is_pressable"): + try: + attribute_dict["{{{:}}}pressable".format(_accessibility_ns_map["st"])] = str(node.is_pressable()).lower() + except: + pass + if hasattr(node, "is_pressed"): + try: + attribute_dict["{{{:}}}pressed".format(_accessibility_ns_map["st"])] = str(node.is_pressed()).lower() + except: + pass + + if hasattr(node, "is_expanded"): + try: + attribute_dict["{{{:}}}expanded".format(_accessibility_ns_map["st"])] = str(node.is_expanded()).lower() + except: + pass + if hasattr(node, "is_editable"): + try: + attribute_dict["{{{:}}}editable".format(_accessibility_ns_map["st"])] = str(node.is_editable()).lower() + except: + pass + # }}} States # + + # Component {{{ # + rectangle = node.rectangle() + attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = "({:d}, {:d})".format(rectangle.left, rectangle.top) + attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = "({:d}, {:d})".format(rectangle.width(), rectangle.height()) + # }}} Component # + + # Text {{{ # + text: str = node.window_text() + if text==attribute_dict["name"]: + text = "" + #if hasattr(node, "texts"): + #texts: List[str] = node.texts()[1:] + #texts: Iterable[str] = map(lambda itm: itm if isinstance(itm, str) else "".join(itm), texts) + #text += "\n".join(texts) + #text = text.strip() + # }}} Text # + + # Selection {{{ # + if hasattr(node, "select"): + attribute_dict["selection"] = "true" + # }}} Selection # + + # Value {{{ # + if hasattr(node, "get_step"): + attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(node.get_step()) + if hasattr(node, "value"): + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.value()) + if hasattr(node, "get_value"): + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.get_value()) + elif hasattr(node, "get_position"): + attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(node.get_position()) + if hasattr(node, "min_value"): + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(node.min_value()) + elif hasattr(node, "get_range_min"): + attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(node.get_range_min()) + if hasattr(node, "max_value"): + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(node.max_value()) + elif hasattr(node, "get_range_max"): + attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(node.get_range_max()) + # }}} Value # + + attribute_dict["{{{:}}}class".format(_accessibility_ns_map["win"])] = str(type(node)) + + node_role_name: str = node.class_name().lower().replace(" ", "-") + node_role_name = "".join( map( lambda ch: ch if ch.isidentifier()\ + or ch in {"-"}\ + or ch.isalnum() + else "-" + , node_role_name + ) + ) + if node_role_name.strip() == "": + node_role_name = "unknown" + + xml_node = lxml.etree.Element( + node_role_name, + attrib=attribute_dict, + nsmap=_accessibility_ns_map + ) + if text is not None and len(text)>0 and text!=attribute_dict["name"]: + xml_node.text = text + + # HYPERPARAMETER + if depth==50: + logger.warning("Max depth reached") + #print("Max depth reached") + return xml_node + + for i, ch in enumerate(node.children()): + # HYPERPARAMETER + if i>=2048: + logger.warning("Max width reached") + #print("Max width reached") + break + xml_node.append(_create_pywinauto_node(ch, depth+1, flag)) + return xml_node + # }}} function _create_pywinauto_node # @app.route("/accessibility", methods=["GET"]) def get_accessibility_tree(): @@ -381,7 +554,15 @@ def get_accessibility_tree(): desktop_xml: _Element = _create_atspi_node(desktop, 0) return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")}) - # TODO: Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned + elif os_name == "Windows": + # Windows AT may be read through `pywinauto` module, however, two different backends `win32` and `uia` are supported and different results may be returned + desktop: Desktop = Desktop(backend="uia") + xml_node = lxml.etree.Element("desktop", nsmap=_accessibility_ns_map) + for wnd in desktop.windows(): + logger.debug("Win UIA AT parsing: %s(%d)", wnd.element_info.name, len(wnd.children())) + node: _Element = _create_pywinauto_node(wnd, 1) + xml_node.append(node) + return jsonify({"AT": lxml.etree.tostring(xml_node, encoding="unicode")}) else: return "Currently not implemented for platform {:}.".format(platform.platform()), 500 diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py index 4fb52ea..507f2c6 100644 --- a/mm_agents/gpt_4v_agent.py +++ b/mm_agents/gpt_4v_agent.py @@ -45,6 +45,10 @@ def linearize_accessibility_tree(accessibility_tree): linearized_accessibility_tree += node.attrib.get('name') + "\t" if node.text: linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t" + elif node.get("{uri:deskat:uia.windows.microsoft.org}class").endswith("EditWrapper")\ + and node.get("{uri:deskat:value.at-spi.gnome.org}value"): + text: str = node.get("{uri:deskat:value.at-spi.gnome.org}value") + linearized_accessibility_tree += (text if '"' not in text else '"{:}"'.format(text.replace('"', '""'))) + "\t" else: linearized_accessibility_tree += '""\t' linearized_accessibility_tree += node.attrib.get( From 14dbc708a45edc3fa3854cc0e7874ec8862a6e5c Mon Sep 17 00:00:00 2001 From: David Chang Date: Tue, 30 Jan 2024 21:09:53 +0800 Subject: [PATCH 10/16] ver Jan30thv2 debugged on windows platform with new _create_pywinauto_node function migrated example task from calc to excel --- branch_flag | 2 +- desktop_env/server/main.py | 25 ++++-- ...269ae-2111-4a07-81fd-3fcd711993b0-win.json | 76 +++++++++++++++++++ main.py | 6 +- 4 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json diff --git a/branch_flag b/branch_flag index 7e5437f..9daeafb 100644 --- a/branch_flag +++ b/branch_flag @@ -1 +1 @@ -baseline_som +test diff --git a/desktop_env/server/main.py b/desktop_env/server/main.py index 9f57659..947fc92 100644 --- a/desktop_env/server/main.py +++ b/desktop_env/server/main.py @@ -9,20 +9,29 @@ from typing import List, Dict, Tuple import Xlib import lxml.etree -import pyatspi import pyautogui import requests from PIL import Image from Xlib import display, X from flask import Flask, request, jsonify, send_file, abort # , send_from_directory from lxml.etree import _Element -from pyatspi import Accessible, StateType, STATE_SHOWING -from pyatspi import Action as ATAction -from pyatspi import Component, Document -from pyatspi import Text as ATText -from pyatspi import Value as ATValue -from pywinauto import Desktop -from pywinauto.base_wrapper import BaseWrapper + +platform_name: str = platform.system() + +if platform_name=="Linux": + import pyatspi + from pyatspi import Accessible, StateType, STATE_SHOWING + from pyatspi import Action as ATAction + from pyatspi import Component, Document + from pyatspi import Text as ATText + from pyatspi import Value as ATValue + + BaseWrapper = Any +elif platform_name=="Windows": + from pywinauto import Desktop + from pywinauto.base_wrapper import BaseWrapper + + Accessible = Any from pyxcursor import Xcursor diff --git a/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json new file mode 100644 index 0000000..dfa3dcb --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json @@ -0,0 +1,76 @@ +{ + "id": "01b269ae-2111-4a07-81fd-3fcd711993b0", + "snapshot": "libreoffice_calc", + "instruction": "Fill all the blank cells with the value in the cell above it", + "source": "https://www.youtube.com/shorts/VrUzPTIwQ04", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1FuOZ-5YoKgLLwl_oZd4R3D8pZACf_ukS&export=download&authuser=0&confirm=t&uuid=2051e7a6-5930-4cef-8d77-20ebf66ec6e6&at=APZUnTX1fXqlxy6rluq-Kw-LUhS5:1705919461032", + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx" + } + ] + } + }, + { + "type": "open", + "parameters": { + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx" + } + } + ], + "trajectory": "trajectories/01b269ae-2111-4a07-81fd-3fcd711993b0", + "related_apps": [ + "msoffice_excel" + ], + "evaluator": { + "postconfig": [ + { + "type": "activate_window", + "parameters": { + "window_name": "Student_Level_Fill_Blank.xlsx - Excel", + "strict": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 0.5 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" + ] + } + } + ], + "func": "compare_table", + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1HTle3vgdZSjJIK_wjXyjtWwbiYJeguwv&export=download&authuser=0&confirm=t&uuid=c5d0868b-bed2-48fb-949b-8a9f3f61e8cf&at=APZUnTVqS9CTZFJ1rPqCGQPDCv3p:1705919542916", + "dest": "Student_Level_Fill_Blank_gold.xlsx" + }, + "result": { + "type": "vm_file", + "path": "C:\\Users\\user\\Student_Level_Fill_Blank.xlsx", + "dest": "Student_Level_Fill_Blank.xlsx" + }, + "options": { + "rules": [ + { + "type": "sheet_data", + "sheet_idx0": 0, + "sheet_idx1": "EI0" + } + ] + } + } +} diff --git a/main.py b/main.py index a3a01df..d971e8c 100644 --- a/main.py +++ b/main.py @@ -46,11 +46,11 @@ def human_agent(): Runs the Gym environment with human input. """ - with open("evaluation_examples/examples/thunderbird/7b6c7e24-c58a-49fc-a5bb-d57b80e5b4c3.json", "r") as f: + with open("evaluation_examples/examples/libreoffice_calc/01b269ae-2111-4a07-81fd-3fcd711993b0-win.json", "r") as f: example = json.load(f) - example["snapshot"] = "Snapshot 30" + example["snapshot"] = "Snapshot 1" - env = DesktopEnv( path_to_vm="../../../../大文件/镜像/Ubuntu-1218/Ubuntu/Ubuntu.vmx" + env = DesktopEnv( path_to_vm="~/vmware/Windows 10 x64/Windows 10 x64.vmx" , action_space="computer_13" , task_config=example ) From 84cb8ba56d3c203f31b62da183ec43f112a48670 Mon Sep 17 00:00:00 2001 From: tsuky_chen <91684733+chenjix@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:46:20 +0800 Subject: [PATCH 11/16] Update 0810415c-bde4-4443-9047-d5f70165a697.json --- .../0810415c-bde4-4443-9047-d5f70165a697.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json index bd1cc22..619707d 100644 --- a/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json +++ b/evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json @@ -9,7 +9,7 @@ "parameters": { "files": [ { - "url": "https://drive.usercontent.google.com/download?id=1-svVsH-l2ofufEKuN-cYrIrvXNobtATE&export=download&authuser=0&confirm=t&uuid=be7f891a-f858-48f5-a72d-4e42bbfb8b65&at=APZUnTXzBnaeSJjmxeh4zG03pzA0:1704179807785", + "url": "https://drive.usercontent.google.com/download?id=1YhHNlRsL7lJBsqRYctz4CmEoD1g8oAm0&export=download&authuser=0&confirm=t&uuid=16776039-9eae-4ee7-ae0b-8b2d71cb25e1&at=APZUnTWVT6sfD3MQEADssAEc4Pwn:1706622286569", "path": "Desktop/Novels_Intro_Packet.docx" } ] @@ -64,4 +64,4 @@ "dest": "Novels_Intro_Packet.docx" } } -} \ No newline at end of file +} From d5f4cfdbcd65f4f315a30a793dd0c2c09194cf7f Mon Sep 17 00:00:00 2001 From: rhythmcao Date: Wed, 31 Jan 2024 00:17:33 +0800 Subject: [PATCH 12/16] add 2 multi-app examples --- .../2fe4b718-3bd7-46ec-bdce-b184f5653624.json | 53 +++++++++++ .../58565672-7bfe-48ab-b828-db349231de6b.json | 93 +++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json create mode 100644 evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json diff --git a/evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json b/evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json new file mode 100644 index 0000000..81dc68f --- /dev/null +++ b/evaluation_examples/examples/multi_apps/2fe4b718-3bd7-46ec-bdce-b184f5653624.json @@ -0,0 +1,53 @@ +{ + "id": "2fe4b718-3bd7-46ec-bdce-b184f5653624", + "snapshot": "vlc", + "instruction": "Could you help me create an Animated GIF src_clip.gif from a video file using VLC and GIMP from the source of video \"src.mp4\" in the desktop, 5-second clip beginning at 00:03?", + "source": "https://www.maketecheasier.com/create-gif-from-video-gimp/", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "gimp" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1krQ_vN3QiboMttWrjDY-YROCWZq6cBtb&export=download", + "path": "/home/user/Desktop/src.mp4" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "vlc" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vlc", + "gimp" + ], + "evaluator": { + "func": "compare_images", + "result": { + "type": "vm_file", + "path": "/home/user/Desktop/src_clip.gif", + "dest": "src_clip.gif" + }, + "expected": { + "type": "cloud_file", + "path": "https://drive.usercontent.google.com/download?id=1kDq8pakgdZvKh8CFpR5368stf14gwFCp&export=download", + "dest": "src_clip_gold.gif" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json b/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json new file mode 100644 index 0000000..6732dba --- /dev/null +++ b/evaluation_examples/examples/multi_apps/58565672-7bfe-48ab-b828-db349231de6b.json @@ -0,0 +1,93 @@ +{ + "id": "58565672-7bfe-48ab-b828-db349231de6b", + "snapshot": "chrome", + "instruction": "Can you assist me by opening the first link in the latest email in Bills folder from Thunderbird and displaying it in a new Chrome tab?", + "source": "https://superuser.com/questions/1792660/open-link-from-other-application-does-not-open-the-url-in-firefox", + "config": [ + { + "type": "launch", + "parameters": { + "command": [ + "google-chrome", + "--remote-debugging-port=1337" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "socat", + "tcp-listen:9222,fork", + "tcp:localhost:1337" + ] + } + }, + { + "type": "chrome_open_tabs", + "parameters": { + "urls_to_open": [ + "https://www.apple.com/", + "https://en.sjtu.edu.cn/", + "https://scholar.google.com/" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EHLRWzBCOsyERkSMUnTF2pnsR0n6ZvtR&export=download&authuser=0&confirm=t&uuid=88b71c06-b5b9-4108-a516-24c46fe9273d&at=APZUnTXK2q16yG7u43034ou5uCoA:1706600462917", + "path": "/home/user/thunderbird-profile.tar.gz" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "tar", + "-xz", + "--recursive-unlink", + "-f", + "/home/user/thunderbird-profile.tar.gz", + "-C", + "/home/user/" + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "/usr/bin/thunderbird" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "thunderbird", + "chrome" + ], + "evaluator": { + "func": "is_expected_tabs", + "result": { + "type": "open_tabs_info" + }, + "expected": { + "type": "rule", + "rules": { + "type": "url", + "urls": [ + "https://www.apple.com/", + "https://en.sjtu.edu.cn/", + "https://scholar.google.com/", + "https://www.amazon.com/" + ] + } + } + } +} \ No newline at end of file From 4d9ac08ea29296c9ee94d516597ada16079fca3c Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 31 Jan 2024 00:29:16 +0800 Subject: [PATCH 13/16] Update README.md for LibreOffice setup --- desktop_env/evaluators/README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/desktop_env/evaluators/README.md b/desktop_env/evaluators/README.md index 04a3b84..266e6e7 100644 --- a/desktop_env/evaluators/README.md +++ b/desktop_env/evaluators/README.md @@ -1,10 +1,20 @@ # Setup Instructions +## LibreOffice +For LibreOffice, please enter into the app first, and then enable the no pop-up when 'ctrl + s'. + +## LibreOffice Press +### Setting Up the python-pptx Library +```shell +pip install python-pptx +``` + ## LibreOffice Writer -### Setting Up the python-docx Library +### Setting Up the python-docx and odfpy Library ```shell pip install python-docx +pip install odfpy ``` ## LibreOffice Calc From 606fab4cfa346657c6e06f4e99ba2dc622a421bd Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 31 Jan 2024 00:29:51 +0800 Subject: [PATCH 14/16] Fix minor bug when get a11y tree and linearize for agent --- mm_agents/gpt_4v_agent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mm_agents/gpt_4v_agent.py b/mm_agents/gpt_4v_agent.py index 507f2c6..0dd15cf 100644 --- a/mm_agents/gpt_4v_agent.py +++ b/mm_agents/gpt_4v_agent.py @@ -45,15 +45,15 @@ def linearize_accessibility_tree(accessibility_tree): linearized_accessibility_tree += node.attrib.get('name') + "\t" if node.text: linearized_accessibility_tree += (node.text if '"' not in node.text else '"{:}"'.format(node.text.replace('"', '""'))) + "\t" - elif node.get("{uri:deskat:uia.windows.microsoft.org}class").endswith("EditWrapper")\ + elif node.get("{uri:deskat:uia.windows.microsoft.org}class", "").endswith("EditWrapper")\ and node.get("{uri:deskat:value.at-spi.gnome.org}value"): text: str = node.get("{uri:deskat:value.at-spi.gnome.org}value") linearized_accessibility_tree += (text if '"' not in text else '"{:}"'.format(text.replace('"', '""'))) + "\t" else: linearized_accessibility_tree += '""\t' linearized_accessibility_tree += node.attrib.get( - '{uri:deskat:component.at-spi.gnome.org}screencoord') + "\t" - linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size') + "\n" + '{uri:deskat:component.at-spi.gnome.org}screencoord', "") + "\t" + linearized_accessibility_tree += node.attrib.get('{uri:deskat:component.at-spi.gnome.org}size', "") + "\n" return linearized_accessibility_tree From e70c65de1689043ead935926f6865e3c709b3d3f Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 31 Jan 2024 00:43:48 +0800 Subject: [PATCH 15/16] Fix typos in instruction --- .../455d3c66-7dc6-4537-a39a-36d3e9119df7.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json b/evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json index 06423da..0194b91 100644 --- a/evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json +++ b/evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json @@ -1,7 +1,7 @@ { "id": "455d3c66-7dc6-4537-a39a-36d3e9119df7", "snapshot": "libreoffice_impress", - "instruction": "Could you help me export an Impress file to a .jpg image file and save it as res.jpg on the Desktop? ", + "instruction": "Could you help me export an Impress file to a .png image file and save it as res.png on the Desktop? ", "source": "https://stackoverflow.com/questions/75626383/how-export-libreoffice-impress-to-image", "config": [ { From 0e5eefbb8605974c9bdd0dfed79f2bb04639e886 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 31 Jan 2024 01:06:42 +0800 Subject: [PATCH 16/16] FIx minor bugs from impress --- desktop_env/evaluators/metrics/slides.py | 22 +++++++++---------- .../ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/desktop_env/evaluators/metrics/slides.py b/desktop_env/evaluators/metrics/slides.py index d5888b0..f398045 100644 --- a/desktop_env/evaluators/metrics/slides.py +++ b/desktop_env/evaluators/metrics/slides.py @@ -267,19 +267,17 @@ def check_left_panel(accessibility_tree): root = ET.fromstring(accessibility_tree) for root_pane in root.iter('root-pane'): - for split_pane in root_pane.iter('split-pane'): - for panel in split_pane.iter('panel'): - for scroll_panel in panel.iter('scroll-pane'): - for document_frame in scroll_panel.iter('document-frame'): - # Get the left panel - panel_name = document_frame.get("name") - # visible = scroll_bar.attrib.get(f"{{{namespaces['st']}}}visible") - if panel_name == "Slides View": - # Left panel is open - return 1.0 + for panel in root_pane.iter('panel'): + for split_pane in panel.iter('split-pane'): + # Get the left panel + if split_pane.attrib.get("{{{}}}parentcoord".format(namespaces['cp'])) == "(0, 0)": + # Get the visible attribute + visible = split_pane.attrib.get("{{{}}}visible".format(namespaces['st'])) + if visible: + # decide if it is left panel + return 1. - # Left panel is not open - return 0.0 + return 0. def check_transition(pptx_file, rules): diff --git a/evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json b/evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json index 72f613c..3c6d02b 100644 --- a/evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json +++ b/evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json @@ -19,7 +19,7 @@ "command": [ "python", "-c", - "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')" + "import pyautogui; import time; pyautogui.click(960, 540); time.sleep(5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')" ] } }