diff --git a/desktop_env/evaluators/metrics/gimp.py b/desktop_env/evaluators/metrics/gimp.py index 1e919ef..99cd5f7 100644 --- a/desktop_env/evaluators/metrics/gimp.py +++ b/desktop_env/evaluators/metrics/gimp.py @@ -1,5 +1,8 @@ import os -from PIL import Image, ImageChops, ImageStat +from PIL import Image, ImageStat +from skimage.metrics import structural_similarity as ssim +import numpy as np + def get_gimp_export_path(): # Path to GIMP's configuration file. This example assumes GIMP version 2.10. @@ -20,11 +23,13 @@ def get_gimp_export_path(): # Handle the case where the configuration file is not found print("GIMP configuration file not found") return False - + + def check_file_exists(directory, filename): file_path = os.path.join(directory, filename) return 1 if os.path.isfile(file_path) else 0 + def increase_saturation(image1_path: str, image2_path: str) -> float: def calculate_saturation(image): # convert the image to HSV mode @@ -47,6 +52,7 @@ def increase_saturation(image1_path: str, image2_path: str) -> float: return 1 if saturation1 < saturation2 else 0 + def decrease_brightness(image1_path: str, image2_path: str) -> float: def calculate_brightness(image): # Convert the image to grayscale mode @@ -65,9 +71,12 @@ def decrease_brightness(image1_path: str, image2_path: str) -> float: brightness2 = calculate_brightness(image2) return 1 if brightness1 > brightness2 else 0 + + import cv2 import numpy as np + def find_yellow_triangle(image): # Convert the image to RGBA rgba = cv2.cvtColor(image, cv2.COLOR_BGR2RGBA) @@ -95,6 +104,7 @@ def find_yellow_triangle(image): return cx, cy + def compare_triangle_positions(image1, image2): image1 = cv2.imread(image1, cv2.IMREAD_COLOR) image2 = cv2.imread(image2, cv2.IMREAD_COLOR) @@ -103,12 +113,205 @@ def compare_triangle_positions(image1, image2): cx2, cy2 = find_yellow_triangle(image2) # calculate the distance between the center of the triangle and the center of the image - center_distance1 = np.sqrt((cx1 - image1.shape[1] // 2)**2 + (cy1 - image1.shape[0] // 2)**2) - center_distance2 = np.sqrt((cx2 - image2.shape[1] // 2)**2 + (cy2 - image2.shape[0] // 2)**2) + center_distance1 = np.sqrt( + (cx1 - image1.shape[1] // 2) ** 2 + (cy1 - image1.shape[0] // 2) ** 2) + center_distance2 = np.sqrt( + (cx2 - image2.shape[1] // 2) ** 2 + (cy2 - image2.shape[0] // 2) ** 2) return 1 if center_distance1 > center_distance2 else 0 +# Functions for the GIMP evaluator +def calculate_brightness(image): + """Calculate the average brightness of an image""" + grayscale = image.convert('L') + stat = ImageStat.Stat(grayscale) + return stat.mean[0] + + +def normalize_brightness(image, target_brightness): + """Normalize the brightness of an image to a target brightness in [0, 1]""" + current_brightness = calculate_brightness(image) + factor = target_brightness / current_brightness + + # Apply a point transform to each pixel + def point_transform(x): + return min(255, max(0, int(x * factor))) + + return image.point(point_transform) + + +def measure_saturation(hsv_image): + """Measure the average saturation of an image""" + # Split into H, S, V channels + _, s, _ = hsv_image.split() + # Convert the saturation channel to a numpy array + s_array = np.array(s) + # Calculate the average saturation + avg_saturation = np.mean(s_array) + return avg_saturation + + +def calculate_contrast(image): + """Calculate the contrast of an image as the standard deviation of the pixel + values.""" + pixels = np.asarray(image, dtype=np.float32) + return np.std(pixels) + + +def structure_check_by_mse(img1, img2, threshold=0.03): + """Check if two images are approximately the same by MSE""" + mse = np.mean( + (np.array(img1, dtype=np.float32) / 255 + - np.array(img2, dtype=np.float32) / 255) ** 2) + structure_same = True if mse < threshold else False + return structure_same + + +def structure_check_by_ssim(img1, img2, threshold=0.9): + """Check if two images are approximately the same by SSIM""" + similarity = ssim(np.array(img1), np.array(img2), multichannel=True) + return similarity >= threshold + + +def check_brightness_decrease_and_structure_sim(src_path, tgt_path): + """ + Compare the brightness and structure of two images + gimp:7a4deb26-d57d-4ea9-9a73-630f66a7b568 + """ + img_src = Image.open(src_path) + img_tgt = Image.open(tgt_path) + + # Brightness comparison + brightness_src = calculate_brightness(img_src) + brightness_tgt = calculate_brightness(img_tgt) + brightness_reduced = brightness_tgt < brightness_src + + # Normalize and compare images + target_brightness = 128 + img_src_normalized = normalize_brightness(img_src, target_brightness) + img_tgt_normalized = normalize_brightness(img_tgt, target_brightness) + + structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized) + return brightness_reduced and structure_same + + +def check_saturation_increase_and_structure_sim(src_path, tgt_path): + """ + Compare the saturation of two images + gimp:554785e9-4523-4e7a-b8e1-8016f565f56a + """ + img_src = Image.open(src_path) + hsv_img_src = img_src.convert('HSV') + img_tgt = Image.open(tgt_path) + hsv_img_tgt = img_tgt.convert('HSV') + + # Saturation comparison + src_saturation = measure_saturation(hsv_img_src) + tgt_saturation = measure_saturation(hsv_img_tgt) + + saturation_increased = tgt_saturation > src_saturation + + # Structure comparison + h1, s1, v1 = hsv_img_src.split() + h2, s2, v2 = hsv_img_tgt.split() + h_same = structure_check_by_ssim(h1, h2) + v_same = structure_check_by_ssim(v1, v2) + if h_same and v_same: + structure_same = True + else: + structure_same = False + + return saturation_increased and structure_same + + +def check_file_exists_and_structure_sim(src_path): + """ + Check if the image has been exported to the desktop + gimp:77b8ab4d-994f-43ac-8930-8ca087d7c4b4 + """ + desktop_directory = os.path.expanduser("~/Desktop") + target_image = os.path.join(desktop_directory, "export.jpg") + + # Check if the target image exists + file_exists = os.path.isfile(target_image) + + # Check whether the target image is the same as the source image + img_src = Image.open(src_path) + img_tgt = Image.open(target_image) + structure_same = structure_check_by_ssim(img_src, img_tgt) + + return file_exists and structure_same + + +def check_triangle_position(tgt_path): + """ + Check if the triangle is in the middle of the image. + gimp:f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce + """ + # Load the image + img = Image.open(tgt_path) + img_array = np.array(img) + + # We will determine if the triangle is in the middle of the picture by checking the centroid + # We assume the triangle is a different color from the background + # Find the unique colors + unique_colors = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0) + + # Assuming the background is the most common color and the triangle is a different color, + # we identify the triangle's color as the least common one + triangle_color = unique_colors[-1] + + # Create a mask where the triangle pixels are True + triangle_mask = np.all(img_array == triangle_color, axis=2) + + # Get the coordinates of the triangle pixels + triangle_coords = np.argwhere(triangle_mask) + + # Calculate the centroid of the triangle + centroid = triangle_coords.mean(axis=0) + + # Check if the centroid is approximately in the middle of the image + image_center = np.array(img_array.shape[:2]) / 2 + + # We will consider the triangle to be in the middle if the centroid is within 5% of the image's center + tolerance = 0.05 * np.array(img_array.shape[:2]) + middle = np.all(np.abs(centroid - image_center) < tolerance) + + return middle + + +def check_structure_sim(src_path, tgt_path): + """ + Check if the structure of the two images are similar + gimp:2a729ded-3296-423d-aec4-7dd55ed5fbb3 + """ + img_src = Image.open(src_path) + img_tgt = Image.open(tgt_path) + structure_same = structure_check_by_ssim(img_src, img_tgt) + return structure_same + + +def check_contrast_increase_and_structure_sim(src_path, tgt_path): + """ + Check if the contrast of the image has increased + gimp:f723c744-e62c-4ae6-98d1-750d3cd7d79d + """ + # Load images + source_image = Image.open(src_path) + target_image = Image.open(tgt_path) + + # Calculate contrast + source_contrast = calculate_contrast(source_image) + target_contrast = calculate_contrast(target_image) + higher_contrast = target_contrast > source_contrast + + # Check structure + structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.8) + + return higher_contrast and structure_same + + if __name__ == "__main__": image1_path = "../Downloads/1.png" image2_path = "../Downloads/edited_darker.png" diff --git a/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json b/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json new file mode 100644 index 0000000..e59346a --- /dev/null +++ b/evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json @@ -0,0 +1,49 @@ +{ + "id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568", + "snapshot": "gimp", + "instruction": "Could you make the background of this image transparent for me?", + "source": "https://www.youtube.com/watch?v=lOzSiOIipSM", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1TOtPi1CQsWblGUtQ6AqayfjsPZ_THBJo", + "path": "Desktop/dog_with_background.png" + }, + { + "url": "https://drive.google.com/uc?export=download&id=15YWmeOyUaA7vMX9Ts7-qyh82T8mHeepx", + "path": "Desktop/dog_cutout_gold.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "Desktop/dog_with_background.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "check_structure_sim", + "expected": { + "type": "vm_file", + "path": "Desktop/dog_cutout_gold.png", + "dest": "dog_cutout_gold.png" + }, + "result": { + "type": "vm_file", + "path": "Desktop/dog_without_background.png", + "dest": "dog_without_background.png" + } + } +} \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json index b315b92..c127aa0 100644 --- a/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json +++ b/evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json @@ -1,7 +1,7 @@ { "id": "554785e9-4523-4e7a-b8e1-8016f565f56a", "snapshot": "gimp", - "instruction": "Could you help me increase the saturation of my photo to make it more colorful?", + "instruction": "Could you assist me in enhancing the color vibrancy of my photo?", "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP", "config": [ { @@ -16,9 +16,12 @@ } }, { - "type": "open", + "type": "launch", "parameters": { - "path": "Desktop/woman_sitting_by_the_tree2.png" + "command": [ + "gimp", + "Desktop/woman_sitting_by_the_tree2.png" + ] } } ], @@ -27,7 +30,7 @@ "gimp" ], "evaluator": { - "func": "increase_saturation", + "func": "check_saturation_increase_and_structure_sim", "expected": { "type": "vm_file", "path": "Desktop/woman_sitting_by_the_tree2.png", diff --git a/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json index b5dfebb..e7e0888 100644 --- a/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json +++ b/evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json @@ -1,7 +1,7 @@ { "id": "77b8ab4d-994f-43ac-8930-8ca087d7c4b4", "snapshot": "gimp", - "instruction": "Could you help me keep my photo on the desktop and rename it export.jpg?", + "instruction": "Could you assist me in placing my photo on the desktop and renaming it to export.jpg?", "source": "https://superuser.com/questions/1636113/how-to-get-gimp-to-recognize-images-or-pictures-folder-as-the-default-folder-for", "config": [ { @@ -16,9 +16,12 @@ } }, { - "type": "open", + "type": "launch", "parameters": { - "path": "Desktop/The_Lost_River_Of_Dreams.jpg" + "command": [ + "gimp", + "Desktop/The_Lost_River_Of_Dreams.jpg" + ] } } ], @@ -28,7 +31,15 @@ ], "evaluator": { "func": "check_file_exists", - "file_name": "export.png", - "directory": "/home/user/Desktop/" + "expected": { + "type": "vm_file", + "path": "Desktop/The_Lost_River_Of_Dreams.jpg", + "dest": "The_Lost_River_Of_Dreams.jpg" + }, + "result": { + "type": "vm_file", + "path": "Desktop/export.jpg", + "dest": "export.png" + } } } \ No newline at end of file diff --git a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json index ec698ff..0896eda 100644 --- a/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json +++ b/evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json @@ -1,7 +1,7 @@ { "id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568", "snapshot": "gimp", - "instruction": "Make my picture less bright, please.", + "instruction": "Could you tone down the brightness of my photo?", "source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP", "config": [ { @@ -16,9 +16,12 @@ } }, { - "type": "open", + "type": "launch", "parameters": { - "path": "Desktop/woman_sitting_by_the_tree.png" + "command": [ + "gimp", + "Desktop/woman_sitting_by_the_tree.png" + ] } } ], @@ -27,7 +30,7 @@ "gimp" ], "evaluator": { - "func": "decrease_brightness", + "func": "check_brightness_decrease_and_structure_sim", "expected": { "type": "vm_file", "path": "Desktop/woman_sitting_by_the_tree.png", diff --git a/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json index 3f7d42f..566d273 100644 --- a/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json +++ b/evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json @@ -1,7 +1,7 @@ { "id": "f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce", "snapshot": "gimp", - "instruction": "Help me select the yellow triangle and move it to the center of my photo.", + "instruction": "Help me choose the yellow triangle and positioning it at the center of my picture.", "source": "https://superuser.com/questions/612338/how-do-i-select-and-move-an-object-in-gimp", "config": [ { @@ -16,9 +16,12 @@ } }, { - "type": "open", + "type": "launch", "parameters": { - "path": "Desktop/Triangle_On_The_Side.png" + "command": [ + "gimp", + "Desktop/Triangle_On_The_Side.png" + ] } } ], diff --git a/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json b/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json new file mode 100644 index 0000000..d846aab --- /dev/null +++ b/evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json @@ -0,0 +1,45 @@ +{ + "id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568", + "snapshot": "gimp", + "instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?", + "source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/", + "config": [ + { + "type": "download", + "parameters": { + "files": [ + { + "url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F", + "path": "Desktop/berries.png" + } + ] + } + }, + { + "type": "launch", + "parameters": { + "command": [ + "gimp", + "Desktop/berries.png" + ] + } + } + ], + "trajectory": "trajectories/", + "related_apps": [ + "gimp" + ], + "evaluator": { + "func": "check_contrast_increase_and_structure_sim", + "expected": { + "type": "vm_file", + "path": "Desktop/berries.png", + "dest": "berries.png" + }, + "result": { + "type": "vm_file", + "path": "Desktop/berries_contrast.png", + "dest": "berries_contrast.png" + } + } +} \ No newline at end of file