Merge pull request #1 from xlang-ai/lyt

OS evals update

desktop_env/evaluators/metrics/general.py

@@ -16,6 +16,11 @@ from .utils import _match_record, _match_value_to_rule
 
 import sqlite3
 
+def check_include_exclude(result: str, rules: Dict[str, List[str]]) -> float:
+    include = rules.get("include", [])
+    exclude = rules.get("exclude", [])
+    return all(r in result for r in include) and all(r not in result for r in exclude)
+
 def exact_match(result, rules) -> float:
     expect = rules["expected"]
     print(result, expect)
@@ -195,3 +200,4 @@ def check_json(result: str, rules: Dict[str, List[Dict[str, Union[List[str], str
             value = value[k]
         metric = metric and not _match_value_to_rule(value, r)
     return metric
+

desktop_env/evaluators/metrics/os.py

@@ -0,0 +1,93 @@
+import subprocess
+import os
+
+def is_spotify_installed() -> bool:
+    # Use the 'which' command to check if the 'spotify' executable exists
+    try:
+        subprocess.check_output(['which', 'spotify'])
+        return True
+    except subprocess.CalledProcessError:
+        return False
+    
+def is_dim_screen_inactive_off():
+    # Get the current value of "Dim screen when inactive" setting
+    result = subprocess.run(["gsettings", "get", "org.gnome.desktop.session", "idle-delay"], capture_output=True, text=True)
+
+    # Check if the setting is set to "uint32 0"
+    if result.stdout.strip() == "uint32 0":
+        return 1  # Task successful
+    else:
+        return 0  # Task not successful
+    
+def evaluate_create_test_directory():
+    try:
+        # Specify the path to the directory
+        path = '/test'
+        
+        # Check if the directory already exists
+        if os.path.exists(path):
+            return 0  # Directory already exists, task not successful
+        
+        # Check if the user has sufficient permissions to create the directory
+        if not os.access("/", os.W_OK | os.X_OK):
+            return 0  # Insufficient permissions to create the directory, task not successful
+        
+        return 1  # Task can be considered successful
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        return 0  # Any other errors, task not successful
+
+# TODO: create a file named test.txt and a directory dir1 at home dir before running this
+def is_file_in_directory():
+    # Specify the paths of the file and directory
+    file_path = "/test.txt"
+    directory_path = "/dir1"
+
+    # Check if the file exists in the directory
+    if os.path.isfile(os.path.join(directory_path, os.path.basename(file_path))):
+        return 1  # Task successful
+    else:
+        return 0  # Task not successful
+    
+# TODO: log in to the system before running this
+def is_logout_successful():
+    try:
+        subprocess.run(["whoami"])
+        return 0  # Task not successful
+    except subprocess.CalledProcessError:
+        return 1  # Task successful
+
+def is_do_not_disturb_mode_enabled():
+    try:
+        subprocess.run(["gsettings", "set", "org.gnome.desktop.notifications", "show-banners", "false"], check=True)
+        return 1  # Task successful
+    except subprocess.CalledProcessError:
+        return 0  # Task not successful
+
+def get_default_browser(answer: str):
+    try:
+        result = subprocess.run(["xdg-settings", "get", "default-web-browser"], capture_output=True, text=True)
+        default_browser = result.stdout.strip()
+        return default_browser == answer
+    except subprocess.CalledProcessError:
+        return 0
+
+# TODO: should confirm initial state, i.e. initial list of favorite apps & the first app
+first_app = "thunderbird.desktop" # to be changed
+def is_first_favorite_app_removed():
+    try:
+        result = subprocess.run(["gsettings", "get", "org.gnome.shell", "favorite-apps"], capture_output=True, text=True)
+        output = result.stdout.strip()
+
+        # Remove brackets and spaces from the output
+        favorites = output[1:-1].replace(" ", "")
+
+        # Split output by comma to get individual favorite apps
+        favorite_apps = favorites.split(",")
+
+        if favorite_apps[0] != first_app:
+            return 1  # First favorite app removed
+        else:
+            return 0  # First favorite app not removed
+    except subprocess.CalledProcessError:
+        return 0  # Task not successful