Resolve conflicts

desktop_env/evaluators/getters/__init__.py

@@ -3,3 +3,5 @@ from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
 from .misc import get_rule, get_accessibility_tree
 from .vlc import get_vlc_playing_info, get_vlc_config
 from .chrome import get_default_search_engine, get_cookie_data, get_bookmarks, get_open_tabs_info, get_pdf_from_url, get_shortcuts_on_desktop
+from .replay import get_replay
+from .vscode import get_vscode_config

desktop_env/evaluators/getters/replay.py

@@ -0,0 +1,19 @@
+from typing import List, Dict, Any
+
+def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
+
+    def parse(action):
+        if action["type"] == "hotkey":
+            keys = "', '".join(action["param"])
+            return f"pyautogui.hotkey('{keys}')"
+
+        if action["type"] == "typewrite":
+            text = action["param"]
+            return f"pyautogui.typewrite('{text}')"
+
+        if action["type"] == "press":
+            key = action["param"]
+            return f"pyautogui.press('{key}')"
+
+    for action in trajectory:
+        env.controller.execute_python_command(parse(action))

desktop_env/evaluators/getters/vscode.py

@@ -0,0 +1,13 @@
+from typing import Dict
+from typing import Any
+from replay import get_replay
+from file import get_vm_file
+
+def get_vscode_config(env, config: Dict[str, Any]) -> str:
+
+    get_replay(env, config["trajectory"])
+
+    return get_vm_file(env, {
+            "path": config["path"],
+            "dest": config["dest"]
+        })

desktop_env/evaluators/metrics/__init__.py

@@ -10,8 +10,7 @@ from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom
 from .table import compare_table
 from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
     compare_videos
-
-from .gimp import increase_saturation, decrease_brightness
-from .general import check_csv, check_accessibility_tree, check_list
+from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
+from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
 from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
 

desktop_env/evaluators/metrics/general.py

@@ -13,6 +13,8 @@ from rapidfuzz import fuzz
 
 from .utils import _match_record
 
+import sqlite3
+
 def exact_match(result, rules) -> float:
     expect = rules["expected"]
     print(result, expect)
@@ -146,3 +148,8 @@ def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
 
 # def check_existence(result: str, *args) -> float:
 # return 1. - (result is None)
+
+def run_sqlite3(result: str, rules: Dict[str, Any]) -> float:
+    connection: sqlite3.Connection = sqlite3.connect(result)
+    cursor: sqlite3.Cursor = connection.execute(rules["sql"])
+    return float(cursor.fetchone()[0] or 0)

desktop_env/evaluators/metrics/gimp.py

@@ -20,6 +20,10 @@ def get_gimp_export_path():
         # Handle the case where the configuration file is not found
         print("GIMP configuration file not found")
         return False
+    
+def check_file_exists(directory, filename):
+    file_path = os.path.join(directory, filename)
+    return 1 if os.path.isfile(file_path) else 0
 
 def increase_saturation(image1_path: str, image2_path: str) -> float:
     def calculate_saturation(image):
@@ -61,6 +65,49 @@ def decrease_brightness(image1_path: str, image2_path: str) -> float:
     brightness2 = calculate_brightness(image2)
 
     return 1 if brightness1 > brightness2 else 0
+import cv2
+import numpy as np
+
+def find_yellow_triangle(image):
+    # Convert the image to RGBA
+    rgba = cv2.cvtColor(image, cv2.COLOR_BGR2RGBA)
+
+    # define range of yellow color in HSV
+    lower_yellow = np.array([0, 0, 0], dtype=np.uint8)
+    upper_yellow = np.array([255, 255, 255], dtype=np.uint8)
+
+    # expand the dimensions of lower and upper yellow to match the image dimensions
+    lower_yellow = np.reshape(lower_yellow, (1, 1, 3))
+    upper_yellow = np.reshape(upper_yellow, (1, 1, 3))
+    # build a mask for the yellow color
+    mask = cv2.inRange(rgba, lower_yellow, upper_yellow)
+
+    # search for contours in the mask
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    # choose the largest contour
+    max_contour = max(contours, key=cv2.contourArea)
+
+    # calculate the center of the contour
+    M = cv2.moments(max_contour)
+    cx = int(M['m10'] / M['m00'])
+    cy = int(M['m01'] / M['m00'])
+
+    return cx, cy
+
+def compare_triangle_positions(image1, image2):
+    image1 = cv2.imread(image1, cv2.IMREAD_COLOR)
+    image2 = cv2.imread(image2, cv2.IMREAD_COLOR)
+    # find the center of the yellow triangle in each image
+    cx1, cy1 = find_yellow_triangle(image1)
+    cx2, cy2 = find_yellow_triangle(image2)
+
+    # calculate the distance between the center of the triangle and the center of the image
+    center_distance1 = np.sqrt((cx1 - image1.shape[1] // 2)**2 + (cy1 - image1.shape[0] // 2)**2)
+    center_distance2 = np.sqrt((cx2 - image2.shape[1] // 2)**2 + (cy2 - image2.shape[0] // 2)**2)
+
+    return 1 if center_distance1 > center_distance2 else 0
+
 
 if __name__ == "__main__":
     image1_path = "../Downloads/1.png"

desktop_env/evaluators/metrics/thunderbird.py

@@ -85,7 +85,7 @@ def check_thunderbird_prefs(result: str, rule: Dict[str, Dict[str, Dict[str, Any
 
 _value_processor: Callable[[str], str] = lambda val: val.replace("\\\"", "\"").replace("\\\\", "\\")
 #_condition_pattern: Pattern[str] = re.compile(r'(?P<type>AND|OR) \((?P<key>[\w ]+),(?P<rel>[\w ' + '\'' + r']+),(?:"(?P<val2>(?:[^"]|\")+)"|(?P<val1>[^)]+))\)')
-_condition_pattern: Pattern[str] = re.compile(r'(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)')
+_condition_pattern: Pattern[str] = re.compile(r'\b(?:AND|OR) \((?:[\w ]+),(?:[\w ' + '\'' + r']+),(?:"(?:(?:[^"]|\")+)"|(?:[^)]+))\)|\bALL\b')
 def check_thunderbird_filter(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
     """
     Args:

desktop_env/evaluators/metrics/vscode.py

@@ -1,11 +1,8 @@
 def compare_text_file(actual: str, expected: str, **options) -> float:
     """
     Args:
-        actual (str): path to result xlsx
-        expected (str): path to gold xlsx
-        options (Dict[str, List[str]]): dict like
-          {
-          }
+        actual (str): path to result text file
+        expected (str): path to gold text file
 
     Return:
         float: the score
@@ -20,7 +17,32 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
         return 1.0
     return 0.0
 
+def compare_config(actual: str, expected: str, **options) -> float:
+    """
+    Args:
+        actual (str): path to result text file
+        expected (str): gold string
+
+    Return:
+        float: the score
+    """
+    
+    with open(actual) as f1:
+        actual_text = f1.read()
+
+    if actual_text == expected:
+        return 1.0
+    return 0.0
+
 def compare_answer(actual: str, expected: str, **options) -> float:
+    """
+    Args:
+        actual (str): result string
+        expected (str): gold string
+
+    Return:
+        float: the score
+    """
 
     if actual == expected:
         return 1.0