lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update gimp examples

Browse Source
This commit is contained in:
BlankCheng
2024-01-24 20:22:51 +08:00
parent a4a9fa0817
commit 908f25ad29
7 changed files with 337 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
desktop_env/evaluators/metrics/gimp.py
View File

@@ -1,5 +1,8 @@
import os
from PIL import Image, ImageChops, ImageStat
from PIL import Image, ImageStat
from skimage.metrics import structural_similarity as ssim
import numpy as np
def get_gimp_export_path():
# Path to GIMP's configuration file. This example assumes GIMP version 2.10.
@@ -20,11 +23,13 @@ def get_gimp_export_path():
# Handle the case where the configuration file is not found
print("GIMP configuration file not found")
return False
def check_file_exists(directory, filename):
file_path = os.path.join(directory, filename)
return 1 if os.path.isfile(file_path) else 0
def increase_saturation(image1_path: str, image2_path: str) -> float:
def calculate_saturation(image):
# convert the image to HSV mode
@@ -47,6 +52,7 @@ def increase_saturation(image1_path: str, image2_path: str) -> float:
return 1 if saturation1 < saturation2 else 0
def decrease_brightness(image1_path: str, image2_path: str) -> float:
def calculate_brightness(image):
# Convert the image to grayscale mode
@@ -65,9 +71,12 @@ def decrease_brightness(image1_path: str, image2_path: str) -> float:
brightness2 = calculate_brightness(image2)
return 1 if brightness1 > brightness2 else 0
import cv2
import numpy as np
def find_yellow_triangle(image):
# Convert the image to RGBA
rgba = cv2.cvtColor(image, cv2.COLOR_BGR2RGBA)
@@ -95,6 +104,7 @@ def find_yellow_triangle(image):
return cx, cy
def compare_triangle_positions(image1, image2):
image1 = cv2.imread(image1, cv2.IMREAD_COLOR)
image2 = cv2.imread(image2, cv2.IMREAD_COLOR)
@@ -103,12 +113,205 @@ def compare_triangle_positions(image1, image2):
cx2, cy2 = find_yellow_triangle(image2)
# calculate the distance between the center of the triangle and the center of the image
center_distance1 = np.sqrt((cx1 - image1.shape[1] // 2)**2 + (cy1 - image1.shape[0] // 2)**2)
center_distance2 = np.sqrt((cx2 - image2.shape[1] // 2)**2 + (cy2 - image2.shape[0] // 2)**2)
center_distance1 = np.sqrt(
(cx1 - image1.shape[1] // 2) ** 2 + (cy1 - image1.shape[0] // 2) ** 2)
center_distance2 = np.sqrt(
(cx2 - image2.shape[1] // 2) ** 2 + (cy2 - image2.shape[0] // 2) ** 2)
return 1 if center_distance1 > center_distance2 else 0
# Functions for the GIMP evaluator
def calculate_brightness(image):
"""Calculate the average brightness of an image"""
grayscale = image.convert('L')
stat = ImageStat.Stat(grayscale)
return stat.mean[0]
def normalize_brightness(image, target_brightness):
"""Normalize the brightness of an image to a target brightness in [0, 1]"""
current_brightness = calculate_brightness(image)
factor = target_brightness / current_brightness
# Apply a point transform to each pixel
def point_transform(x):
return min(255, max(0, int(x * factor)))
return image.point(point_transform)
def measure_saturation(hsv_image):
"""Measure the average saturation of an image"""
# Split into H, S, V channels
_, s, _ = hsv_image.split()
# Convert the saturation channel to a numpy array
s_array = np.array(s)
# Calculate the average saturation
avg_saturation = np.mean(s_array)
return avg_saturation
def calculate_contrast(image):
"""Calculate the contrast of an image as the standard deviation of the pixel
values."""
pixels = np.asarray(image, dtype=np.float32)
return np.std(pixels)
def structure_check_by_mse(img1, img2, threshold=0.03):
"""Check if two images are approximately the same by MSE"""
mse = np.mean(
(np.array(img1, dtype=np.float32) / 255
- np.array(img2, dtype=np.float32) / 255) ** 2)
structure_same = True if mse < threshold else False
return structure_same
def structure_check_by_ssim(img1, img2, threshold=0.9):
"""Check if two images are approximately the same by SSIM"""
similarity = ssim(np.array(img1), np.array(img2), multichannel=True)
return similarity >= threshold
def check_brightness_decrease_and_structure_sim(src_path, tgt_path):
"""
Compare the brightness and structure of two images
gimp:7a4deb26-d57d-4ea9-9a73-630f66a7b568
"""
img_src = Image.open(src_path)
img_tgt = Image.open(tgt_path)
# Brightness comparison
brightness_src = calculate_brightness(img_src)
brightness_tgt = calculate_brightness(img_tgt)
brightness_reduced = brightness_tgt < brightness_src
# Normalize and compare images
target_brightness = 128
img_src_normalized = normalize_brightness(img_src, target_brightness)
img_tgt_normalized = normalize_brightness(img_tgt, target_brightness)
structure_same = structure_check_by_mse(img_src_normalized, img_tgt_normalized)
return brightness_reduced and structure_same
def check_saturation_increase_and_structure_sim(src_path, tgt_path):
"""
Compare the saturation of two images
gimp:554785e9-4523-4e7a-b8e1-8016f565f56a
"""
img_src = Image.open(src_path)
hsv_img_src = img_src.convert('HSV')
img_tgt = Image.open(tgt_path)
hsv_img_tgt = img_tgt.convert('HSV')
# Saturation comparison
src_saturation = measure_saturation(hsv_img_src)
tgt_saturation = measure_saturation(hsv_img_tgt)
saturation_increased = tgt_saturation > src_saturation
# Structure comparison
h1, s1, v1 = hsv_img_src.split()
h2, s2, v2 = hsv_img_tgt.split()
h_same = structure_check_by_ssim(h1, h2)
v_same = structure_check_by_ssim(v1, v2)
if h_same and v_same:
structure_same = True
else:
structure_same = False
return saturation_increased and structure_same
def check_file_exists_and_structure_sim(src_path):
"""
Check if the image has been exported to the desktop
gimp:77b8ab4d-994f-43ac-8930-8ca087d7c4b4
"""
desktop_directory = os.path.expanduser("~/Desktop")
target_image = os.path.join(desktop_directory, "export.jpg")
# Check if the target image exists
file_exists = os.path.isfile(target_image)
# Check whether the target image is the same as the source image
img_src = Image.open(src_path)
img_tgt = Image.open(target_image)
structure_same = structure_check_by_ssim(img_src, img_tgt)
return file_exists and structure_same
def check_triangle_position(tgt_path):
"""
Check if the triangle is in the middle of the image.
gimp:f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce
"""
# Load the image
img = Image.open(tgt_path)
img_array = np.array(img)
# We will determine if the triangle is in the middle of the picture by checking the centroid
# We assume the triangle is a different color from the background
# Find the unique colors
unique_colors = np.unique(img_array.reshape(-1, img_array.shape[2]), axis=0)
# Assuming the background is the most common color and the triangle is a different color,
# we identify the triangle's color as the least common one
triangle_color = unique_colors[-1]
# Create a mask where the triangle pixels are True
triangle_mask = np.all(img_array == triangle_color, axis=2)
# Get the coordinates of the triangle pixels
triangle_coords = np.argwhere(triangle_mask)
# Calculate the centroid of the triangle
centroid = triangle_coords.mean(axis=0)
# Check if the centroid is approximately in the middle of the image
image_center = np.array(img_array.shape[:2]) / 2
# We will consider the triangle to be in the middle if the centroid is within 5% of the image's center
tolerance = 0.05 * np.array(img_array.shape[:2])
middle = np.all(np.abs(centroid - image_center) < tolerance)
return middle
def check_structure_sim(src_path, tgt_path):
"""
Check if the structure of the two images are similar
gimp:2a729ded-3296-423d-aec4-7dd55ed5fbb3
"""
img_src = Image.open(src_path)
img_tgt = Image.open(tgt_path)
structure_same = structure_check_by_ssim(img_src, img_tgt)
return structure_same
def check_contrast_increase_and_structure_sim(src_path, tgt_path):
"""
Check if the contrast of the image has increased
gimp:f723c744-e62c-4ae6-98d1-750d3cd7d79d
"""
# Load images
source_image = Image.open(src_path)
target_image = Image.open(tgt_path)
# Calculate contrast
source_contrast = calculate_contrast(source_image)
target_contrast = calculate_contrast(target_image)
higher_contrast = target_contrast > source_contrast
# Check structure
structure_same = structure_check_by_ssim(source_image, target_image, threshold=0.8)
return higher_contrast and structure_same
if __name__ == "__main__":
image1_path = "../Downloads/1.png"
image2_path = "../Downloads/edited_darker.png"

49
evaluation_examples/examples/gimp/2a729ded-3296-423d-aec4-7dd55ed5fbb3.json Normal file
View File

@@ -0,0 +1,49 @@
{
"id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568",
"snapshot": "gimp",
"instruction": "Could you make the background of this image transparent for me?",
"source": "https://www.youtube.com/watch?v=lOzSiOIipSM",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1TOtPi1CQsWblGUtQ6AqayfjsPZ_THBJo",
"path": "Desktop/dog_with_background.png"
},
{
"url": "https://drive.google.com/uc?export=download&id=15YWmeOyUaA7vMX9Ts7-qyh82T8mHeepx",
"path": "Desktop/dog_cutout_gold.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"Desktop/dog_with_background.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "check_structure_sim",
"expected": {
"type": "vm_file",
"path": "Desktop/dog_cutout_gold.png",
"dest": "dog_cutout_gold.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/dog_without_background.png",
"dest": "dog_without_background.png"
}
}
}

11
evaluation_examples/examples/gimp/554785e9-4523-4e7a-b8e1-8016f565f56a.json
View File

@@ -1,7 +1,7 @@
{
"id": "554785e9-4523-4e7a-b8e1-8016f565f56a",
"snapshot": "gimp",
"instruction": "Could you help me increase the saturation of my photo to make it more colorful?",
"instruction": "Could you assist me in enhancing the color vibrancy of my photo?",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{
@@ -16,9 +16,12 @@
}
},
{
"type": "open",
"type": "launch",
"parameters": {
"path": "Desktop/woman_sitting_by_the_tree2.png"
"command": [
"gimp",
"Desktop/woman_sitting_by_the_tree2.png"
]
}
}
],
@@ -27,7 +30,7 @@
"gimp"
],
"evaluator": {
"func": "increase_saturation",
"func": "check_saturation_increase_and_structure_sim",
"expected": {
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree2.png",

21
evaluation_examples/examples/gimp/77b8ab4d-994f-43ac-8930-8ca087d7c4b4.json
View File

@@ -1,7 +1,7 @@
{
"id": "77b8ab4d-994f-43ac-8930-8ca087d7c4b4",
"snapshot": "gimp",
"instruction": "Could you help me keep my photo on the desktop and rename it export.jpg?",
"instruction": "Could you assist me in placing my photo on the desktop and renaming it to export.jpg?",
"source": "https://superuser.com/questions/1636113/how-to-get-gimp-to-recognize-images-or-pictures-folder-as-the-default-folder-for",
"config": [
{
@@ -16,9 +16,12 @@
}
},
{
"type": "open",
"type": "launch",
"parameters": {
"path": "Desktop/The_Lost_River_Of_Dreams.jpg"
"command": [
"gimp",
"Desktop/The_Lost_River_Of_Dreams.jpg"
]
}
}
],
@@ -28,7 +31,15 @@
],
"evaluator": {
"func": "check_file_exists",
"file_name": "export.png",
"directory": "/home/user/Desktop/"
"expected": {
"type": "vm_file",
"path": "Desktop/The_Lost_River_Of_Dreams.jpg",
"dest": "The_Lost_River_Of_Dreams.jpg"
},
"result": {
"type": "vm_file",
"path": "Desktop/export.jpg",
"dest": "export.png"
}
}
}

11
evaluation_examples/examples/gimp/7a4deb26-d57d-4ea9-9a73-630f66a7b568.json
View File

@@ -1,7 +1,7 @@
{
"id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568",
"snapshot": "gimp",
"instruction": "Make my picture less bright, please.",
"instruction": "Could you tone down the brightness of my photo?",
"source": "https://www.quora.com/How-do-I-edit-a-photo-in-GIMP",
"config": [
{
@@ -16,9 +16,12 @@
}
},
{
"type": "open",
"type": "launch",
"parameters": {
"path": "Desktop/woman_sitting_by_the_tree.png"
"command": [
"gimp",
"Desktop/woman_sitting_by_the_tree.png"
]
}
}
],
@@ -27,7 +30,7 @@
"gimp"
],
"evaluator": {
"func": "decrease_brightness",
"func": "check_brightness_decrease_and_structure_sim",
"expected": {
"type": "vm_file",
"path": "Desktop/woman_sitting_by_the_tree.png",

9
evaluation_examples/examples/gimp/f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce.json
View File

@@ -1,7 +1,7 @@
{
"id": "f4aec372-4fb0-4df5-a52b-79e0e2a5d6ce",
"snapshot": "gimp",
"instruction": "Help me select the yellow triangle and move it to the center of my photo.",
"instruction": "Help me choose the yellow triangle and positioning it at the center of my picture.",
"source": "https://superuser.com/questions/612338/how-do-i-select-and-move-an-object-in-gimp",
"config": [
{
@@ -16,9 +16,12 @@
}
},
{
"type": "open",
"type": "launch",
"parameters": {
"path": "Desktop/Triangle_On_The_Side.png"
"command": [
"gimp",
"Desktop/Triangle_On_The_Side.png"
]
}
}
],

45
evaluation_examples/examples/gimp/f723c744-e62c-4ae6-98d1-750d3cd7d79d.json Normal file
View File

@@ -0,0 +1,45 @@
{
"id": "7a4deb26-d57d-4ea9-9a73-630f66a7b568",
"snapshot": "gimp",
"instruction": "I'd like to make the picture's contrast stronger to really bring out the main subject. Could you assist me in boosting the contrast?",
"source": "https://www.reddit.com/r/GIMP/comments/12e57w8/how_to_use_gimp_to_exaggerate_contrast/",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1X42_kOanL74vu_p6QdcZuiyzDQi3kA7F",
"path": "Desktop/berries.png"
}
]
}
},
{
"type": "launch",
"parameters": {
"command": [
"gimp",
"Desktop/berries.png"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"gimp"
],
"evaluator": {
"func": "check_contrast_increase_and_structure_sim",
"expected": {
"type": "vm_file",
"path": "Desktop/berries.png",
"dest": "berries.png"
},
"result": {
"type": "vm_file",
"path": "Desktop/berries_contrast.png",
"dest": "berries_contrast.png"
}
}
}