Update GIMP getters and metrics

desktop_env/envs/desktop_env.py

@@ -285,8 +285,7 @@ class DesktopEnv(gym.Env):
         logger.info("Emulator started.")
 
         logger.info("Get meta info of the VM...")
-        # self.vm_platform = self.controller.get_vm_platform()
-        self.vm_platform = "Darwin"
+        self.vm_platform = self.controller.get_vm_platform()
         self.vm_screen_size = self.controller.get_vm_screen_size()
         print(self.vm_screen_size)
 

desktop_env/evaluators/getters/__init__.py

@@ -10,4 +10,4 @@ from .replay import get_replay
 from .vlc import get_vlc_playing_info, get_vlc_config
 from .vscode import get_vscode_config
 from .impress import get_audio_in_slide
-from .gimp import get_gimp_config
+from .gimp import get_gimp_config_file

desktop_env/evaluators/getters/gimp.py

@@ -5,12 +5,13 @@ from typing import Dict
 logger = logging.getLogger("desktopenv.getters.gimp")
 
 
-def get_gimp_config(env, config: Dict[str, str]):
+def get_gimp_config_file(env, config: Dict[str, str]):
     """
     Gets the config setting of GIMP.
     """
 
     os_type = env.vm_platform
+    print(os_type)
 
     if os_type == "Linux":
         config_path = \

desktop_env/evaluators/metrics/__init__.py

@@ -24,9 +24,15 @@ from .gimp import (
     check_brightness_decrease_and_structure_sim,
     check_contrast_increase_and_structure_sim,
     check_saturation_increase_and_structure_sim,
+    check_image_size_and_structure_sim,
+    check_image_mirror,
+    check_palette_and_structure_sim,
+    check_textbox_on_leftside,
+    check_green_background,
     check_file_exists_and_structure_sim,
     check_triangle_position,
     check_structure_sim,
+    check_config_status,
 )
 from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, \
     compare_videos, check_qt_bgcone, check_one_instance_when_started_from_file, check_qt_minimal_view, \

desktop_env/evaluators/metrics/gimp.py

@@ -1,8 +1,6 @@
 import os
-from PIL import Image, ImageStat
+from PIL import Image, ImageStat, ImageChops
 from skimage.metrics import structural_similarity as ssim
-import numpy as np
-import sexpdata
 
 
 def get_gimp_export_path():
@@ -166,18 +164,20 @@ def structure_check_by_mse(img1, img2, threshold=0.03):
         (np.array(img1, dtype=np.float32) / 255
          - np.array(img2, dtype=np.float32) / 255) ** 2)
     structure_same = True if mse < threshold else False
+    print("MSE: ", mse)
     return structure_same
 
 
 def structure_check_by_ssim(img1, img2, threshold=0.9):
     """Check if two images are approximately the same by SSIM"""
     similarity = ssim(np.array(img1), np.array(img2), multichannel=True)
+    print("SSIM: ", similarity)
     return similarity >= threshold
 
 
 def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
     """
-    Compare the brightness and structure of two images
+    Check the brightness of src is lower than tgt and the structures are similar
     gimp:7a4deb26-d57d-4ea9-9a73-630f66a7b568
     """
     img_src = Image.open(src_path)
@@ -186,7 +186,7 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
     # Brightness comparison
     brightness_src = calculate_brightness(img_src)
     brightness_tgt = calculate_brightness(img_tgt)
-    brightness_reduced = brightness_tgt < brightness_src
+    brightness_reduced = brightness_tgt > brightness_src
 
     # Normalize and compare images
     target_brightness = 128
@@ -199,7 +199,7 @@ def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
 
 def check_saturation_increase_and_structure_sim(src_path, tgt_path):
     """
-    Compare the saturation of two images
+    Check the saturation of src is higher than tgt and the structures are similar
     gimp:554785e9-4523-4e7a-b8e1-8016f565f56a
     """
     img_src = Image.open(src_path)
@@ -211,7 +211,7 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
     src_saturation = measure_saturation(hsv_img_src)
     tgt_saturation = measure_saturation(hsv_img_tgt)
 
-    saturation_increased = tgt_saturation > src_saturation
+    saturation_increased = tgt_saturation < src_saturation
 
     # Structure comparison
     h1, s1, v1 = hsv_img_src.split()
@@ -226,23 +226,22 @@ def check_saturation_increase_and_structure_sim(src_path, tgt_path):
     return saturation_increased and structure_same
 
 
-def check_file_exists_and_structure_sim(src_path):
+def check_file_exists_and_structure_sim(src_path, tgt_path):
     """
     Check if the image has been exported to the desktop
     gimp:77b8ab4d-994f-43ac-8930-8ca087d7c4b4
     """
-    desktop_directory = os.path.expanduser("~/Desktop")
-    target_image = os.path.join(desktop_directory, "export.jpg")
-
-    # Check if the target image exists
-    file_exists = os.path.isfile(target_image)
+    # Check if the file exists
+    export_file_exists = os.path.isfile(src_path)
+    if not export_file_exists:
+        return False
 
     # Check whether the target image is the same as the source image
     img_src = Image.open(src_path)
-    img_tgt = Image.open(target_image)
+    img_tgt = Image.open(tgt_path)
     structure_same = structure_check_by_ssim(img_src, img_tgt)
 
-    return file_exists and structure_same
+    return structure_same
 
 
 def check_triangle_position(tgt_path):
@@ -287,8 +286,6 @@ def check_structure_sim(src_path, tgt_path):
     Check if the structure of the two images are similar
     gimp:2a729ded-3296-423d-aec4-7dd55ed5fbb3
     """
-    print("src_path:", src_path)
-    print("tgt_path:", tgt_path)
     img_src = Image.open(src_path)
     img_tgt = Image.open(tgt_path)
     structure_same = structure_check_by_ssim(img_src, img_tgt)
@@ -297,7 +294,7 @@ def check_structure_sim(src_path, tgt_path):
 
 def check_contrast_increase_and_structure_sim(src_path, tgt_path):
     """
-    Check if the contrast of the image has increased
+    Check if the src image has higher contrast than the tgt image and the structures are similar
     gimp:f723c744-e62c-4ae6-98d1-750d3cd7d79d
     """
     # Load images
@@ -307,10 +304,10 @@ def check_contrast_increase_and_structure_sim(src_path, tgt_path):
     # Calculate contrast
     source_contrast = calculate_contrast(source_image)
     target_contrast = calculate_contrast(target_image)
-    higher_contrast = target_contrast > source_contrast
+    higher_contrast = target_contrast < source_contrast
 
     # Check structure
-    structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.8)
+    structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.65)
 
     return higher_contrast and structure_same
 
@@ -337,6 +334,111 @@ def check_config_status(actual_config_path, rule):
     return False
 
 
+def check_image_size_and_structure_sim(src_path, tgt_path, height=512, width=None):
+    """
+    Check if the size of the src image is correct and the structure of the two images are similar.
+    gimp:d16c99dc-2a1e-46f2-b350-d97c86c85c15
+    """
+    # Load images
+    source_image = Image.open(src_path)
+    target_image = Image.open(tgt_path)
+
+    # Check size
+    if width is not None:
+        width_same = source_image.size[0] == width
+    else:
+        width_same = True
+    if height is not None:
+        height_same = source_image.size[1] == height
+    else:
+        height_same = True
+
+    # Check structure
+    resized_target_image = target_image.resize(source_image.size)
+    structure_same = structure_check_by_ssim(source_image, resized_target_image)
+
+    return width_same and height_same and structure_same
+
+
+def check_palette_and_structure_sim(src_path, tgt_path):
+    """
+    Check if the src image is palette-based and the structure of the two images are similar
+    gimp:06ca5602-62ca-47f6-ad4f-da151cde54cc
+    """
+    # Check if the source image is palette-based
+    source_image = Image.open(src_path)
+    palette_based = source_image.mode == 'P'
+
+    # Check structure
+    target_image = Image.open(tgt_path)
+    source_image = source_image.convert('RGB')
+    structure_same = structure_check_by_ssim(source_image, target_image)
+    return palette_based and structure_same
+
+
+def check_textbox_on_leftside(src_path):
+    """
+    Check if the textbox is on the left side of the image.
+    gimp:e2dd0213-26db-4349-abe5-d5667bfd725c
+    """
+    source_image = Image.open(src_path)
+    gray_image = source_image.convert("L")
+    width, height = source_image.size
+
+    # Find the bounds of the black text
+    left_most_dark_pixel = width  # Start with the farthest possible left position
+    for y in range(height):
+        for x in range(width):
+            # If the pixel is dark, consider it as part of the text
+            if gray_image.getpixel((x, y)) < 128:  # Arbitrary threshold for "dark"
+                left_most_dark_pixel = min(left_most_dark_pixel, x)
+                break  # Stop after finding the first dark pixel in this row
+
+    # Here we define "almost" on the left side as being within the left 5% of the image
+    return left_most_dark_pixel < width * 0.05
+
+
+def check_image_mirror(src_path, tgt_path):
+    """
+    Check if the image is mirrored
+    gimp:72f83cdc-bf76-4531-9a1b-eb893a13f8aa
+    """
+    # Load images
+    source_image = Image.open(src_path)
+    target_image = Image.open(tgt_path)
+
+    # Check if the image is mirrored
+    transposed_image = source_image.transpose(Image.FLIP_LEFT_RIGHT)
+    # Use 0.99 because the image may not be exactly mirrored by gimp
+    mirrored = structure_check_by_ssim(transposed_image, target_image, 0.99)
+    return mirrored
+
+
+def check_green_background(src_path, tgt_path):
+    """
+    Check if the background of the source image is green.
+    gimp:734d6579-c07d-47a8-9ae2-13339795476b
+    """
+    # Load images
+    source_image = Image.open(src_path)
+    target_image = Image.open(tgt_path)
+
+    source_pixels = np.array(source_image)
+    target_pixels = np.array(target_image)
+
+    for x in range(target_image.width):
+        for y in range(target_image.height):
+            # Identify background pixel in target image (not black)
+            if tuple(target_pixels[x, y][:3]) != (0, 0, 0):
+                # Check if corresponding pixel in source image is green
+                # Here, "green" means more green than red or blue
+                r, g, b = source_pixels[x, y][:3]
+                if not (g > r and g > b):
+                    return False
+
+    return True
+
+
 if __name__ == "__main__":
     actual_config_path = "../../../cache/sessionrc_test"
     rule = {
@@ -358,3 +460,7 @@ if __name__ == "__main__":
         "value": "100"
     }
     print(check_config_status(actual_config_path, rule))
+
+    src_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/green_background_with_object.png"
+    tgt_path = "../../../cache/734d6579-c07d-47a8-9ae2-13339795476b/white_background_with_object.png"
+    print(check_green_background(src_path, tgt_path))