diff --git a/desktop_env/evaluators/metrics/os.py b/desktop_env/evaluators/metrics/os.py index b97648c..49e149b 100644 --- a/desktop_env/evaluators/metrics/os.py +++ b/desktop_env/evaluators/metrics/os.py @@ -1,5 +1,6 @@ -import subprocess import os +import subprocess + def is_spotify_installed() -> bool: # Use the 'which' command to check if the 'spotify' executable exists @@ -8,35 +9,39 @@ def is_spotify_installed() -> bool: return True except subprocess.CalledProcessError: return False - + + def is_dim_screen_inactive_off(): # Get the current value of "Dim screen when inactive" setting - result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True, text=True) + result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True, + text=True) # Check if the setting is set to "uint32 0" if result.stdout.strip() == "uint32 0": return 1 # Task successful else: return 0 # Task not successful - + + def evaluate_create_test_directory(): try: # Specify the path to the directory path = '/test' - + # Check if the directory already exists if os.path.exists(path): return 0 # Directory already exists, task not successful - + # Check if the user has sufficient permissions to create the directory if not os.access("/", os.W_OK | os.X_OK): return 0 # Insufficient permissions to create the directory, task not successful - + return 1 # Task can be considered successful except Exception as e: print(f"Error: {str(e)}") return 0 # Any other errors, task not successful + # TODO: create a file named test.txt and a directory dir1 at home dir before running this def is_file_in_directory(): # Specify the paths of the file and directory @@ -48,7 +53,8 @@ def is_file_in_directory(): return 1 # Task successful else: return 0 # Task not successful - + + # TODO: log in to the system before running this def is_logout_successful(): try: @@ -57,6 +63,7 @@ def is_logout_successful(): except subprocess.CalledProcessError: return 1 # Task successful + def is_do_not_disturb_mode_enabled(): try: subprocess.run(["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"], check=True) @@ -64,6 +71,7 @@ def is_do_not_disturb_mode_enabled(): except subprocess.CalledProcessError: return 0 # Task not successful + def get_default_browser(answer: str): try: result = subprocess.run(["xdg-settings", "get", "default-web-browser"], capture_output=True, text=True) @@ -72,11 +80,15 @@ def get_default_browser(answer: str): except subprocess.CalledProcessError: return 0 + # TODO: should confirm initial state, i.e. initial list of favorite apps & the first app -first_app = "thunderbird.desktop" # to be changed +first_app = "thunderbird.desktop" # to be changed + + def is_first_favorite_app_removed(): try: - result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True, text=True) + result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True, + text=True) output = result.stdout.strip() # Remove brackets and spaces from the output @@ -90,4 +102,51 @@ def is_first_favorite_app_removed(): else: return 0 # First favorite app not removed except subprocess.CalledProcessError: - return 0 # Task not successful \ No newline at end of file + return 0 # Task not successful + + +def is_battery_percentage_displayed(): + # GNOME schema and key for the setting + schema = "org.gnome.desktop.interface" + key = "show-battery-percentage" + + try: + # use gsettings to get the current setting + result = subprocess.run(['gsettings', 'get', schema, key], capture_output=True, text=True) + # examine the output is 'true' + if result.stdout.strip() == 'true': + return 1. + else: + return 0. + except Exception as e: + print(f"An error occurred: {e}") + return 0. + + +def check_auto_lock_settings(): + # The schema and keys for the GNOME desktop lock settings + schema = "org.gnome.desktop.screensaver" + lock_enabled_key = "lock-enabled" + lock_delay_key = "lock-delay" + + try: + # Check if the screen lock is enabled + lock_enabled = subprocess.run(['gsettings', 'get', schema, lock_enabled_key], capture_output=True, text=True) + is_lock_enabled = lock_enabled.stdout.strip() == 'true' + + # Check the delay before the screen is locked + lock_delay = subprocess.run(['gsettings', 'get', schema, lock_delay_key], capture_output=True, text=True) + # Extract the numerical value from the output + delay_seconds = lock_delay + + # Print the results + if is_lock_enabled: + return 1. + else: + return 0. + + except Exception as e: + return 0. + + + diff --git a/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json new file mode 100644 index 0000000..3abbd63 --- /dev/null +++ b/evaluation_examples/examples/os/13584542-872b-42d8-b299-866967b5c3ef.json @@ -0,0 +1,17 @@ +{ + "id": "13584542-872b-42d8-b299-866967b5c3ef", + "snapshot": "os", + "instruction": "Set the default terminal size and screen x,y coordinates in order to make the terminal window always open at the same location and with the same size.", + "source": "https://superuser.com/questions/72176/linux-set-default-terminal-size-and-screen-position", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json new file mode 100644 index 0000000..ee41922 --- /dev/null +++ b/evaluation_examples/examples/os/23393935-50c7-4a86-aeea-2b78fd089c5c.json @@ -0,0 +1,17 @@ +{ + "id": "23393935-50c7-4a86-aeea-2b78fd089c5c", + "snapshot": "os", + "instruction": "Recursively go through the folders of the 'photos' directory and copy any .jpg files found into another directory named 'cpjpg'.", + "source": "https://superuser.com/questions/91307/copying-only-jpg-from-a-directory-structure-to-another-location-linux", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json b/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json new file mode 100644 index 0000000..0044e02 --- /dev/null +++ b/evaluation_examples/examples/os/28cc3b7e-b194-4bc9-8353-d04c0f4d56d2.json @@ -0,0 +1,17 @@ +{ + "id": "28cc3b7e-b194-4bc9-8353-d04c0f4d56d2", + "snapshot": "os", + "instruction": "The volume of my system is too small. Can you help me turn up to the max volumn?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/sound-volume.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json new file mode 100644 index 0000000..d8894a1 --- /dev/null +++ b/evaluation_examples/examples/os/3ce045a0-877b-42aa-8d2c-b4a863336ab8.json @@ -0,0 +1,17 @@ +{ + "id": "3ce045a0-877b-42aa-8d2c-b4a863336ab8", + "snapshot": "os", + "instruction": "I want to make the text on the screen larger", + "source": "https://help.ubuntu.com/lts/ubuntu-help/a11y-font-size.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json b/evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json new file mode 100644 index 0000000..f09c9b0 --- /dev/null +++ b/evaluation_examples/examples/os/4d2b519e-e872-4100-8ea3-fe71ab0f9133.json @@ -0,0 +1,17 @@ +{ + "id": "4d2b519e-e872-4100-8ea3-fe71ab0f9133", + "snapshot": "os", + "instruction": "Could you please help me to add a new entry \"/home/david/pear/bin\" to the PATH variable in the ZSH terminal?", + "source": "https://stackoverflow.com/questions/11530090/adding-a-new-entry-to-the-path-variable-in-zsh", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json b/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json new file mode 100644 index 0000000..ffbe51e --- /dev/null +++ b/evaluation_examples/examples/os/5812b315-e7bd-4265-b51f-863c02174c28.json @@ -0,0 +1,17 @@ +{ + "id": "5812b315-e7bd-4265-b51f-863c02174c28", + "snapshot": "os", + "instruction": "Please create an SSH user on Ubuntu who is only allowed to access the folder \"test1\".", + "source": "https://superuser.com/questions/149404/create-an-ssh-user-who-only-has-permission-to-access-specific-folders", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json new file mode 100644 index 0000000..c482bc7 --- /dev/null +++ b/evaluation_examples/examples/os/5c433d22-ed9a-4e31-91f5-54cf3e8acd63.json @@ -0,0 +1,17 @@ +{ + "id": "5c433d22-ed9a-4e31-91f5-54cf3e8acd63", + "snapshot": "os", + "instruction": "I want to change my system language to Chinese(simplified). Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/session-language.html.zh-CN", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json new file mode 100644 index 0000000..9111f9a --- /dev/null +++ b/evaluation_examples/examples/os/5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57.json @@ -0,0 +1,17 @@ +{ + "id": "5ea617a3-0e86-4ba6-aab2-dac9aa2e8d57", + "snapshot": "os", + "instruction": "I am currently using an Ubuntu system, and I have wrongly deleted a file named \"test\". Could you help me recover it from the Trash?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-recover.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json new file mode 100644 index 0000000..ac3de88 --- /dev/null +++ b/evaluation_examples/examples/os/6ebbfb01-ea72-4226-a2a6-dc428e111ed2.json @@ -0,0 +1,17 @@ +{ + "id": "6ebbfb01-ea72-4226-a2a6-dc428e111ed2", + "snapshot": "os", + "instruction": "Could you please help me to set Bash as my default shell on the current Ubuntu system?", + "source": "https://superuser.com/questions/46748/how-do-i-make-bash-my-default-shell-on-ubuntu", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json b/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json new file mode 100644 index 0000000..ebac87b --- /dev/null +++ b/evaluation_examples/examples/os/765d2b74-88a7-4d50-bf51-34e4106fd24a.json @@ -0,0 +1,17 @@ +{ + "id": "765d2b74-88a7-4d50-bf51-34e4106fd24a", + "snapshot": "os", + "instruction": "Can you help me delete the \"test\" file on my desktop?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-delete.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json new file mode 100644 index 0000000..9b6cd0f --- /dev/null +++ b/evaluation_examples/examples/os/a4d98375-215b-4a4d-aee9-3d4370fccc41.json @@ -0,0 +1,17 @@ +{ + "id": "a4d98375-215b-4a4d-aee9-3d4370fccc41", + "snapshot": "os", + "instruction": "I want to have my computer automatically locked after I leaved. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/privacy-screen-lock.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json b/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json new file mode 100644 index 0000000..1c0d9b2 --- /dev/null +++ b/evaluation_examples/examples/os/b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa.json @@ -0,0 +1,17 @@ +{ + "id": "b3d4a89c-53f2-4d6b-8b6a-541fb5d205fa", + "snapshot": "os", + "instruction": "I want to switch off the Bluetooth. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/bluetooth-turn-on-off.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json new file mode 100644 index 0000000..7c892d7 --- /dev/null +++ b/evaluation_examples/examples/os/b6781586-6346-41cd-935a-a6b1487918fc.json @@ -0,0 +1,17 @@ +{ + "id": "b6781586-6346-41cd-935a-a6b1487918fc", + "snapshot": "os", + "instruction": "I want to set my current time zone to UTC+0. Can you help me?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/clock-timezone.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/c288e301-e626-4b98-a1ab-159dcb162af5.json b/evaluation_examples/examples/os/c288e301-e626-4b98-a1ab-159dcb162af5.json new file mode 100644 index 0000000..9a3de40 --- /dev/null +++ b/evaluation_examples/examples/os/c288e301-e626-4b98-a1ab-159dcb162af5.json @@ -0,0 +1,17 @@ +{ + "id": "c288e301-e626-4b98-a1ab-159dcb162af5", + "snapshot": "os", + "instruction": "Please set the default Python version to Python3 on my Ubuntu system.", + "source": "https://stackoverflow.com/questions/41986507/unable-to-set-default-python-version-to-python3-in-ubuntu", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json new file mode 100644 index 0000000..27abbf7 --- /dev/null +++ b/evaluation_examples/examples/os/c56de254-a3ec-414e-81a6-83d2ce8c41fa.json @@ -0,0 +1,17 @@ +{ + "id": "c56de254-a3ec-414e-81a6-83d2ce8c41fa", + "snapshot": "os", + "instruction": "Could you please help me to extract text from a non-indexed PDF document on the current system?", + "source": "https://superuser.com/questions/28426/how-to-extract-text-with-ocr-from-a-pdf-on-linux", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json b/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json new file mode 100644 index 0000000..b8d4d30 --- /dev/null +++ b/evaluation_examples/examples/os/cc9d4f34-1ca0-4a1b-8ff2-09302696acb9.json @@ -0,0 +1,17 @@ +{ + "id": "cc9d4f34-1ca0-4a1b-8ff2-09302696acb9", + "snapshot": "os", + "instruction": "I need you to execute a specific process or script from the terminal and ensure that it continues to run independently, even after the terminal session is terminated. This involves redirecting the process's output to a file and disassociating the process from the terminal session's control.", + "source": "https://superuser.com/questions/178587/how-do-i-detach-a-process-from-terminal-entirely", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json new file mode 100644 index 0000000..b50c3a9 --- /dev/null +++ b/evaluation_examples/examples/os/ddc75b62-7311-4af8-bfb3-859558542b36.json @@ -0,0 +1,17 @@ +{ + "id": "ddc75b62-7311-4af8-bfb3-859558542b36", + "snapshot": "os", + "instruction": "I want to uninstall the Firefox on my system. Can you help me", + "source": "https://help.ubuntu.com/lts/ubuntu-help/addremove-remove.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json new file mode 100644 index 0000000..0563b9a --- /dev/null +++ b/evaluation_examples/examples/os/e0df059f-28a6-4169-924f-b9623e7184cc.json @@ -0,0 +1,17 @@ +{ + "id": "e0df059f-28a6-4169-924f-b9623e7184cc", + "snapshot": "os", + "instruction": "I have a directory named \"test1\". Can you help me change its name into \"test2\"?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/files-rename.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json b/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json new file mode 100644 index 0000000..fdf0ede --- /dev/null +++ b/evaluation_examples/examples/os/e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15.json @@ -0,0 +1,17 @@ +{ + "id": "e2eb4bf1-aa93-4192-b55d-03e2fb6dfd15", + "snapshot": "os", + "instruction": "Can you help me add Charles to my contact?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/contacts-add-remove.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/f10b16e1-c160-4cb3-989f-7b2ec89bc073.json b/evaluation_examples/examples/os/f10b16e1-c160-4cb3-989f-7b2ec89bc073.json new file mode 100644 index 0000000..f7e297f --- /dev/null +++ b/evaluation_examples/examples/os/f10b16e1-c160-4cb3-989f-7b2ec89bc073.json @@ -0,0 +1,17 @@ +{ + "id": "f10b16e1-c160-4cb3-989f-7b2ec89bc073", + "snapshot": "os", + "instruction": "Please install the latest version of the GNOME Desktop on the current Ubuntu system.", + "source": "https://www.wikihow.com/Install-Gnome-on-Ubuntu", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file diff --git a/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json new file mode 100644 index 0000000..b5ee6cd --- /dev/null +++ b/evaluation_examples/examples/os/fe41f596-a71b-4c2f-9b2f-9dcd40b568c3.json @@ -0,0 +1,17 @@ +{ + "id": "fe41f596-a71b-4c2f-9b2f-9dcd40b568c3", + "snapshot": "os", + "instruction": "I want to see the battery percentage. Can you help me display it on my screen?", + "source": "https://help.ubuntu.com/lts/ubuntu-help/power-percentage.html.en", + "trajectory": "trajectories/", + "related_apps": [ + "os" + ], + "evaluator": { + "func": "", + "result": { + }, + "expected": { + } + } + } \ No newline at end of file