diff --git a/desktop_env/evaluators/getters/__init__.py b/desktop_env/evaluators/getters/__init__.py index 9e2e636..0ce7ea7 100644 --- a/desktop_env/evaluators/getters/__init__.py +++ b/desktop_env/evaluators/getters/__init__.py @@ -4,7 +4,7 @@ from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, g from .file import get_cloud_file, get_vm_file, get_cache_file from .general import get_vm_command_line from .impress import get_audio_in_slide -from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper +from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper, get_list_directory from .misc import get_rule, get_accessibility_tree from .replay import get_replay from .vlc import get_vlc_playing_info, get_vlc_config diff --git a/desktop_env/evaluators/getters/info.py b/desktop_env/evaluators/getters/info.py index 3d41ee5..0c88fd2 100644 --- a/desktop_env/evaluators/getters/info.py +++ b/desktop_env/evaluators/getters/info.py @@ -18,3 +18,7 @@ def get_vm_wallpaper(env, config: dict) -> Union[str, bytes]: f.write(content) return _path + + +def get_list_directory(env, config: dict) -> dict: + return env.controller.get_vm_directory_tree(config["path"]) diff --git a/desktop_env/evaluators/metrics/__init__.py b/desktop_env/evaluators/metrics/__init__.py index b2214fd..d863dc7 100644 --- a/desktop_env/evaluators/metrics/__init__.py +++ b/desktop_env/evaluators/metrics/__init__.py @@ -25,4 +25,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co check_qt_slider_colours, check_global_key_play_pause from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \ check_json_keybindings -from .os import check_gnome_favorite_apps +from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs diff --git a/desktop_env/evaluators/metrics/os.py b/desktop_env/evaluators/metrics/os.py index 3386466..55688fb 100644 --- a/desktop_env/evaluators/metrics/os.py +++ b/desktop_env/evaluators/metrics/os.py @@ -17,54 +17,43 @@ def check_gnome_favorite_apps(apps_str: str, rule): return 0 -# TODO: log in to the system before running this -def is_logout_successful(): - try: - subprocess.run(["whoami"]) - return 0 # Task not successful - except subprocess.CalledProcessError: - return 1 # Task successful +def is_utc_0(timedatectl_output): + """ + Format as: + Local time: Thu 2024-01-25 12:56:06 WET + Universal time: Thu 2024-01-25 12:56:06 UTC + RTC time: Thu 2024-01-25 12:56:05 + Time zone: Atlantic/Faroe (WET, +0000) +System clock synchronized: yes + NTP service: inactive + RTC in local TZ: no + """ + + utc_line = timedatectl_output.split("\n")[3] + + if utc_line.endswith("+0000)"): + return 1 + else: + return 0 -def is_battery_percentage_displayed(): - # GNOME schema and key for the setting - schema = "org.gnome.desktop.interface" - key = "show-battery-percentage" - - try: - # use gsettings to get the current setting - result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True) - # examine the output is 'true' - if result.stdout.strip() == 'true': - return 1. - else: - return 0. - except Exception as e: - print(f"An error occurred: {e}") - return 0. +def check_text_enlarged(scaling_factor_str): + scaling_factor = float(scaling_factor_str) + if scaling_factor > 1.0: + return 1 + else: + return 0 -def check_auto_lock_settings(): - # The schema and keys for the GNOME desktop lock settings - schema = "org.gnome.desktop.screensaver" - lock_enabled_key = "lock-enabled" - lock_delay_key = "lock-delay" +def check_moved_jpgs(directory_list, rule): - try: - # Check if the screen lock is enabled - lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True) - is_lock_enabled = lock_enabled.stdout.strip() == 'true' + expected_jpgs = rule["expected"] + moved_jpgs = [node['name'] for node in directory_list['children']] - # Check the delay before the screen is locked - lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True) - # Extract the numerical value from the output - delay_seconds = lock_delay + if len(moved_jpgs) != len(expected_jpgs): + return 0 - # Print the results - if is_lock_enabled: - return 1. - else: - return 0. - - except Exception as e: - return 0. + if set(moved_jpgs) == set(expected_jpgs): + return 1 + else: + return 0 diff --git a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json index ee41922..654a749 100644 --- a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json +++ b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json @@ -1,17 +1,101 @@ { - "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", - "snapshot": "os", - "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", - "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "snapshot": "os", + "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", + "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos && mkdir -p ~/Desktop/cpjpg", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation && mkdir -p ~/Desktop/photos/family && mkdir -p ~/Desktop/photos/events", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "mkdir -p ~/Desktop/photos/vacation/thailand && mkdir -p ~/Desktop/photos/vacation/hk", + "shell": true + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1H7R6flpv6vgyBOHurzR0CyvPRIobuIpI&export=download&authuser=0&confirm=t&uuid=1b41e87d-7a84-4aa6-af40-cf940e80d956&at=APZUnTWus2n1pP6l2wmIyXkCUHGs:1706196492307", + "path": "/home/user/Desktop/photos/vacation/thailand/monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1Mh3kIViHaayxNC88TpgpjbmIEl17LA3d&export=download&authuser=0&confirm=t&uuid=a8355b84-aa2e-4502-8d0f-3723ffe7d442&at=APZUnTW4nzDFgDgNeERMRKEQ4bTU:1706196493503", + "path": "/home/user/Desktop/photos/vacation/hk/hong-kong-china.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1EhmbZ3QP_mdvvppC5zKA3o--Ce2F-ocO&export=download&authuser=0&confirm=t&uuid=70e05133-dcac-4168-b555-82da15a10d58&at=APZUnTXJ371jG7qiPLHBLt2zy7ij:1706196494612", + "path": "/home/user/Desktop/photos/vacation/hk/hk_group_photo.jpg" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1p_r_BMCLFG872uH9Qa-sLnqXppHvDx95&export=download&authuser=0&confirm=t&uuid=af59d107-8e46-4d2c-b72a-e4b42655959a&at=APZUnTU2hBidSnefJuUQTyJCEVFx:1706196491030", + "path": "/home/user/Desktop/photos/family/us_3.png" + } + ] + } + }, + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1YxryNHfLPcFODnUh31CYGHGj3ZFRhvbA&export=download&authuser=0&confirm=t&uuid=0925ae33-ab33-42b9-901c-d90f11e0afcd&at=APZUnTVey7VeC9zYOmTwl0x02Iwf:1706196495770", + "path": "/home/user/Desktop/photos/events/emnlp2023.jpg" + } + ] } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "check_moved_jpgs", + "result": { + "type": "list_directory", + "path": "/home/user/Desktop/cpjpg" + }, + "expected": { + "type": "rule", + "rules": { + "expected": ["emnlp2023.jpg", "hk_group_photo.jpg", "hong-kong-china.jpg", "monk252520thailand252520wat252520arun252520scaled25255B225255D.jpg"] + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json index 9c67040..99df0f7 100644 --- a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json +++ b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json @@ -1,7 +1,7 @@ { "id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8", "snapshot": "os", - "instruction": "I want to make the text on the screen larger", + "instruction": "My glasses are broken, and I'm having trouble seeing small things clearly. Could you help me enlarge the text on my screen so it's easier to read?", "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en", "trajectory": "trajectories/", "config": [], @@ -9,10 +9,11 @@ "os" ], "evaluator": { - "func": "", + "func": "check_text_enlarged", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface text-scaling-factor", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json index 5482ee2..ebbb114 100644 --- a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json +++ b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json @@ -1,7 +1,7 @@ { "id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63", "snapshot": "os", - "instruction": "I want to change my system language to Chinese(simplified). Can you help me?", + "instruction": "I want to change my system language to Chinese (simplified). Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN", "trajectory": "trajectories/", "config": [], diff --git a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json index b6c00af..7ce6f21 100644 --- a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json +++ b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json @@ -1,18 +1,44 @@ { "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", "snapshot": "os", - "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?", + "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a poster of party night. Could you help me recover it from the Trash?", "source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.usercontent.google.com/download?id=1XaTnC_lLbR_tGTz8tcN2Tp6cNrMlNW3R&export=download&authuser=0&confirm=t&uuid=89e69a23-43cf-4316-833a-fb9d3e281460&at=APZUnTWn5zZTH4GlClO6lV1i4WwP:1706184669922", + "path": "poster_party_night.webp" + } + ] + } + }, + { + "type": "execute", + "parameters": { + "command": "mv ~/poster_party_night.webp ~/.local/share/Trash/files/", + "shell": true + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -f /home/user/Desktop/poster_party_night.webp ] && echo 'File exists.' || echo 'File does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "File exists.\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json index 1d253f4..673163b 100644 --- a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json +++ b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json @@ -4,15 +4,60 @@ "instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?", "source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu", "trajectory": "trajectories/", - "config": [], + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S apt-get install zsh -y", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S chsh -s $(which zsh)", + "shell": true + } + }, + { + "type": "execute", + "parameters": { + "command": "echo password | sudo -S loginctl terminate-user user", + "shell": true + } + }, + { + "type": "sleep", + "parameters": { + "seconds": 10 + } + }, + { + "type": "execute", + "parameters": { + "command": [ + "python", + "-c", + "import pyautogui; import time; time.sleep(2); pyautogui.press(\"enter\"); pyautogui.type(\"password\"); time.sleep(2); pyautogui.press(\"enter\");" + ] + } + } + ], "related_apps": [ "os" ], "evaluator": { - "func": "", - "result": { - }, - "expected": { - } + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "echo $SHELL", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "/bin/bash\n" + } + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json index 9b6cd0f..3675d05 100644 --- a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json +++ b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json @@ -1,17 +1,24 @@ { - "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", - "snapshot": "os", - "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", + "snapshot": "os", + "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.screensaver lock-enabled", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "expected": "true\n" } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json index 7b43485..46964d3 100644 --- a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json +++ b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json @@ -4,15 +4,17 @@ "instruction": "I want to set my current time zone to UTC+0. Can you help me?", "source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en", "trajectory": "trajectories/", - "config": [], + "config": [ + ], "related_apps": [ "os" ], "evaluator": { - "func": "", + "func": "is_utc_0", "result": { - }, - "expected": { + "type": "vm_command_line", + "command": "timedatectl status", + "shell": true } } } \ No newline at end of file diff --git a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json index b50c3a9..c54f368 100644 --- a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json +++ b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json @@ -1,17 +1,36 @@ { - "id": "ddc75b62-7311-4af8-bfb3-859558542b36", - "snapshot": "os", - "instruction": "I want to uninstall the Firefox on my system. Can you help me", - "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" + "id": "ddc75b62-7311-4af8-bfb3-859558542b36", + "snapshot": "os", + "instruction": "I want to uninstall the Mahjongg on my system. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "postconfig": [ + { + "type": "sleep", + "parameters": { + "seconds": 1 + } + } ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "func": "check_include_exclude", + "result": { + "type": "vm_command_line", + "command": "dpkg -l | grep mahjongg", + "shell": true + }, + "expected": { + "type": "rule", + "rules": { + "include": [ + ], + "exclude": [ + "mahjongg" + ] } } - } \ No newline at end of file + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json index 0563b9a..c9e41d8 100644 --- a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json +++ b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json @@ -1,17 +1,33 @@ { - "id": "e0df059f-28a6-4169-924f-b9623e7184cc", - "snapshot": "os", - "instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?", - "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", - "trajectory": "trajectories/", - "related_apps": [ - "os" - ], - "evaluator": { - "func": "", - "result": { - }, - "expected": { + "id": "e0df059f-28a6-4169-924f-b9623e7184cc", + "snapshot": "os", + "instruction": "I have a directory named \"todo_list_Jan_1\". Can you help me change its name into \"todo_list_Jan_2\"?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", + "trajectory": "trajectories/", + "config": [ + { + "type": "execute", + "parameters": { + "command": "echo 'password' | sudo -S mkdir ~/Desktop/todo_list_Jan_1", + "shell": true } } - } \ No newline at end of file + ], + "related_apps": [ + "os" + ], + "evaluator": { + "func": "exact_match", + "result": { + "type": "vm_command_line", + "command": "[ -d ~/Desktop/todo_list_Jan_2 ] && echo 'Directory exists.' || echo 'Directory does not exist.'", + "shell": true + }, + "expected": { + "type": "rule", + "rules":{ + "expected": "Directory exists.\n" + } + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json index b5ee6cd..c613775 100644 --- a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -8,10 +8,17 @@ "os" ], "evaluator": { - "func": "", + "func": "exact_match", "result": { + "type": "vm_command_line", + "command": "gsettings get org.gnome.desktop.interface show-battery-percentage", + "shell": true }, "expected": { + "type": "rule", + "rules":{ + "expected": "true\n" + } } } } \ No newline at end of file