diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index bb30d12..3535ddc 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -21,5 +21,5 @@ from .impress import get_audio_in_slide from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay -from .vlc import get_vlc_playing_info, get_vlc_config +from .vlc import get_vlc_playing_info, get_vlc_config, get_default_video_player from .vscode import get_vscode_config diff --git a/desktop_env/evaluators/getters/vlc.py b/desktop_env/evaluators/getters/vlc.py index 37b00b2..ef0adfb 100644 --- a/desktop_env/evaluators/getters/vlc.py +++ b/desktop_env/evaluators/getters/vlc.py @@ -1,7 +1,8 @@ import logging import os from typing import Dict - +from collections import Counter +from .general import get_vm_command_line import requests logger = logging.getLogger("desktopenv.getters.vlc") @@ -58,3 +59,28 @@ def get_vlc_config(env, config: Dict[str, str]): f.write(content) return _path + + +def get_default_video_player(env, config: dict): + """ Gets the default application for a category or file extension. + """ + + os_type = env.vm_platform + + if os_type == "Linux": + extensions = ['3gp', '3gp', '3gpp', '3gpp', '3gpp2', '3gpp2', 'avi', 'avi', 'divx', 'divx', 'dv', 'dv', 'fli', 'fli', 'flv', 'flv', 'mp2t', 'mp2t', 'mp4', 'mp4', 'mp4v-es', 'mp4v-es', 'mpeg', 'mpeg', 'mpeg-system', 'mpeg-system', 'msvideo', 'msvideo', 'ogg', 'ogg', 'quicktime', 'quicktime', 'vnd.divx', 'vnd.divx', 'vnd.mpegurl', 'vnd.mpegurl', 'vnd.rn-realvideo', 'vnd.rn-realvideo', 'webm', 'webm', 'x-anim', 'x-anim', 'x-avi', 'x-avi', 'x-flc', 'x-flc', 'x-fli', 'x-fli', 'x-flv', 'x-flv', 'x-m4v', 'x-m4v', 'x-matroska', 'x-matroska', 'x-mpeg', 'x-mpeg', 'x-mpeg-system', 'x-mpeg-system', 'x-mpeg2', 'x-mpeg2', 'x-ms-asf', 'x-ms-asf', 'x-ms-asf-plugin', 'x-ms-asf-plugin', 'x-ms-asx', 'x-ms-asx', 'x-ms-wm', 'x-ms-wm', 'x-ms-wmv', 'x-ms-wmv', 'x-ms-wmx', 'x-ms-wmx', 'x-ms-wvx', 'x-ms-wvx', 'x-msvideo', 'x-msvideo', 'x-nsv', 'x-nsv', 'x-ogm', 'x-ogm', 'x-ogm+ogg', 'x-theora', 'x-theora', 'x-theora+ogg', 'x-theora+ogg'] + apps = [] + for ext in extensions: + app = get_vm_command_line(env, {"command": ["xdg-mime", "query", "default", f"video/{ext}"]}) + if app: + apps.append(app) + if len(apps) == 0: + return 'unknown' + else: + return Counter(apps).most_common(1)[0][0] + elif os_type == "Darwin": + raise Exception("Unsupported operating system", os_type) + elif os_type == "Windows": + raise Exception("Unsupported operating system", os_type) + else: + raise Exception("Unsupported operating system", os_type) \ No newline at end of file diff --git a/desktop_env/evaluators/getters/vscode.py b/desktop_env/evaluators/getters/vscode.py index 8a725ef..bf8f351 100644 --- a/desktop_env/evaluators/getters/vscode.py +++ b/desktop_env/evaluators/getters/vscode.py @@ -1,6 +1,6 @@ import logging from typing import Any, Dict - +import time from .file import get_vm_file from .replay import get_replay @@ -27,6 +27,7 @@ def get_vscode_config(env, config: Dict[str, Any]) -> str: ] get_replay(env, trajectory) + time.sleep(1.0) return get_vm_file(env, { "path": config["path"], diff --git a/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json index d83ed1b..99e148b 100644 --- a/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json +++ b/evaluation_examples/examples/multi_apps/2b9493d7-49b8-493a-a71b-56cd1f4d6908.json @@ -36,7 +36,8 @@ "parameters": { "command": [ "gnome-terminal", - "--maximize" + "--maximize", + "--working-directory=/home/user/" ] } } @@ -59,43 +60,27 @@ } } ], - "func": ["check_include_exclude", "check_include_exclude"], - "conj": "and", - "result": [ - { - "type": "vm_command_line", - "command": [ - "/bin/bash", - "-c", - "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"true\"; else echo \"false\"; fi" - ] - }, - { - "type": "vm_command_line", - "command": [ - "/bin/bash", - "-c", - "output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"false\"; else echo \"true\"; fi" + "func": "check_include_exclude", + "result": { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "output=$(ps aux | grep \"[s]office\"]); if [ -z \"$output\" ]; then echo \"no libreoffice is running\"; else echo \"libreoffice is still running\"; fi; output=$(cat ~/.bash_history | grep \"[k]ill\"); if [ -z \"$output\" ]; then echo \"not killed from terminal\"; else echo \"killed from terminal\"; fi" + ] + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + "no libreoffice is running", + "killed from terminal" + ], + "exclude": [ + "libreoffice is still running", + "not killed from terminal" ] } - ], - "expected": [ - { - "type": "rule", - "rules": { - "include": [ - "true\n" - ] - } - }, - { - "type": "rule", - "rules": { - "include": [ - "true\n" - ] - } - } - ] + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json index 3926120..e3eac08 100644 --- a/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json +++ b/evaluation_examples/examples/multi_apps/2c9fc0de-3ee7-45e1-a5df-c86206ad78b5.json @@ -6,15 +6,12 @@ "config": [ { "type": "download", - "path": "", - }, - { - "type": "execute", "parameters": { - "command": [ - "/bin/bash", - "-c", - "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main --bare .git" + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1aANBCEHT6K8GmHDMEYtL1LQXdApqVoGv&export=download", + "path": "/home/user/Downloads/binder.zip" + } ] } }, @@ -24,7 +21,17 @@ "command": [ "/bin/bash", "-c", - "mkdir -p /home/user/projects/hello_world && cd /home/user/projects/hello_world && git init --initial-branch main && git remote add origin /home/user/projects/remote_project && echo \"Hello World!\" > README.md" + "git config --global user.name \"xlang\" && git config --global user.email \"xlang2024anonym@gmail.com\" && mkdir -p /home/user/projects/remote_project && cd /home/user/projects/remote_project && git init --initial-branch=main && git config receive.denyCurrentBranch ignore " + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "unzip -q /home/user/Downloads/binder.zip -d /home/user/projects/ && cd /home/user/projects/binder && git init --initial-branch main && git remote add origin /home/user/projects/remote_project" ] } }, @@ -34,7 +41,7 @@ "command": [ "gnome-terminal", "--maximize", - "--working-directory=/home/user/projects/hello_world" + "--working-directory=/home/user/projects/binder" ] } } @@ -45,19 +52,23 @@ "terminal" ], "evaluator": { - "func": "compare_docx_files", + "func": "check_include_exclude", "result": { - "type": "vm_file", - "path": "/home/user/Desktop/notes.docx", - "dest": "notes.docx" + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "cd /home/user/projects/remote_project; git log --oneline | head -n 1 | awk '{for (i=2; i<=NF; i++) {printf \"%s%s\", $i, (i==NF ? \"\" : \" \")}; print \"\"}'; remote_id=$(git log --oneline | head -n 1 | awk '{print $1}'); cd /home/user/projects/binder; local_id=$(git log --oneline | head -n 1 | awk '{print $1}'); if [ \"${local_id}\" = \"${remote_id}\" ]; then echo \"repo is synchronous\"; else echo \"repo is not synchronous\"; fi" + ] }, "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1Xl6tgQ0K5qA1BDA2fKTK2xFLzXwbtkZ6&export=download", - "dest": "notes_gold.docx" - }, - "options": { - "ignore_blanks": true + "type": "rule", + "rules": { + "include": [ + "daily update", + "repo is synchronous" + ] + } } } } \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json b/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json new file mode 100644 index 0000000..de6e9f4 --- /dev/null +++ b/evaluation_examples/examples/multi_apps/510f64c8-9bcc-4be1-8d30-638705850618.json @@ -0,0 +1,116 @@ +{ + "id": "510f64c8-9bcc-4be1-8d30-638705850618", + "snapshot": "vscode", + "instruction": "Could you start VS Code in folder ~/Desktop/project from the terminal?", + "source": "https://www.geeksforgeeks.org/how-to-start-vs-code-from-the-terminal-command-line/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download", + "path": "/home/user/Downloads/vscodeEvalExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip && mkdir -p ~/Desktop/project/.vscode && history -c && echo > ~/.bash_history" + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1akdsiRVdq6CUtT-FX8Dpf8ruPTq6DcFn&export=download&authuser=0&confirm=t&uuid=ce2fa96a-454e-43d9-bbe3-98553b7eed0d&at=APZUnTVw_YQ1URTvP34vrmKcw0b4:1705222451052", + "path": "/home/user/Desktop/project/main.py" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1BkwtqtAzv_K2CrTbJZ0HbMHBffzdD9vc&export=download&authuser=0&confirm=t&uuid=28f77090-deef-49a1-b156-91317881e75e&at=APZUnTXuaR6i_3t3Prslk535GaO5:1705222457290", + "path": "/home/user/Desktop/project/README.md" + }, + { + "url": "https://drive.usercontent.google.com/download?id=1ea_zF2tbcXOB8w9neBV-U5xI2nnPzIw_&export=download&authuser=0&confirm=t&uuid=9cf8c5bb-a880-475c-b80b-967a0c4fbea4&at=APZUnTUdjIj80F3Mbgi72eZDTZLO:1705222462443", + "path": "/home/user/Desktop/project/.vscode/settings.json" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gnome-terminal", + "--maximize", + "--working-directory=/home/user/" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "vscode", + "terminal" + ], + "evaluator": { + "postconfig": [ + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "killall gnome-terminal-server" + ] + } + }, + { + "type": "activate_window", + "parameters": { + "window_name": "Visual Studio Code" + } + } + ], + "func": ["check_include_exclude", "compare_config"], + "result": [ + { + "type": "vm_command_line", + "command": [ + "/bin/bash", + "-c", + "use_terminal=$(cat ~/.bash_history | grep \"[c]ode \"); if [ -z \"$use_terminal\" ]; then echo \"false\"; else echo \"true\"; fi" + ] + }, + { + "type": "vscode_config", + "vscode_extension_command": "OpenProject", + "path": "/home/user/OpenProject.txt", + "dest": "OpenProject.txt" + } + ], + "expected": [ + { + "type": "rule", + "rules": { + "include": [ + "true" + ] + } + }, + { + "type": "rule", + "rules": { + "expect": "project" + } + } + ] + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json b/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json new file mode 100644 index 0000000..58c4d6d --- /dev/null +++ b/evaluation_examples/examples/multi_apps/937087b6-f668-4ba6-9110-60682ee33441.json @@ -0,0 +1,26 @@ +{ + "id": "937087b6-f668-4ba6-9110-60682ee33441", + "snapshot": "vlc", + "instruction": "I am currently using a ubuntu system. Could you help me set the default video player as VLC?", + "source": "https://superuser.com/questions/187440/set-default-ubuntu-video-player-as-vlc", + "config": [], + "trajectory": "trajectories/", + "related_apps": [ + "vlc", + "os" + ], + "evaluator": { + "func": "check_include_exclude", + "result": { + "type": "default_video_player" + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + "vlc.desktop" + ] + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json index 55dfb09..b597a03 100644 --- a/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json +++ b/evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json @@ -4,6 +4,27 @@ "instruction": "Please help me use VS Code to open the \"project\" in the \"user\" folder under \"home\".", "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ", "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1PnC-zxTtntYvuGlnIV2_05jiHfkX_1E-&export=download", + "path": "/home/user/Downloads/vscodeEvalExtension.zip" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "/bin/bash", + "-c", + "cd /home/user/Downloads && unzip -q vscodeEvalExtension.zip && code --install-extension vscodeEvalExtension/eval-0.0.1.vsix && rm -rf vscodeEvalExtension vscodeEvalExtension.zip" + ] + } + }, { "type": "launch", "parameters": {