lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update libreoffice impress eval examples

Browse Source
This commit is contained in:
tsuky_chen
2024-01-28 12:18:25 +08:00
parent 8525825fb2
commit 394d6353fd
11 changed files with 484 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
desktop_env/evaluators/metrics/slides.py
View File

@@ -96,6 +96,38 @@ def check_slide_numbers_color(pptx_file_path):
print(font_color)
return 1 if font_color is not None and is_red_color(font_color) else 0
# import numpy as np
# from PIL import Image
# from skimage.metrics import structural_similarity as ssim
# def compare_images(image1_path, image2_path):
# # You would call this function with the paths to the two images you want to compare:
# # score = compare_images('path_to_image1', 'path_to_image2')
# # print("Similarity score:", score)
# if not image1_path or not image2_path:
# return 0
# # Open the images and convert to grayscale
# image1 = Image.open(image1_path).convert('L')
# image2 = Image.open(image2_path).convert('L')
# # Resize images to the smaller one's size for comparison
# image1_size = image1.size
# image2_size = image2.size
# new_size = min(image1_size, image2_size)
# image1 = image1.resize(new_size, Image.Resampling.LANCZOS)
# image2 = image2.resize(new_size, Image.Resampling.LANCZOS)
# # Convert images to numpy arrays
# image1_array = np.array(image1)
# image2_array = np.array(image2)
# # Calculate SSIM between two images
# similarity_index = ssim(image1_array, image2_array)
# return similarity_index
def compare_pptx_files(file1_path, file2_path, **options):
# todo: not strictly match since not all information is compared because we cannot get the info through pptx
@@ -114,49 +146,50 @@ def compare_pptx_files(file1_path, file2_path, **options):
# compare the number of slides
if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
return False
return 0
# compare the content of each slide
for slide1, slide2 in zip(prs1.slides, prs2.slides):
# check if the shapes are the same
for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
if (
shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
return False
return 0
if hasattr(shape1, "text") and hasattr(shape2, "text"):
if shape1.text != shape2.text and examine_text:
return False
return 0
# check if the paragraphs are the same
for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
# check if the runs are the same
for run1, run2 in zip(para1.runs, para2.runs):
if run1.text != run2.text and examine_text:
return False
return 0
# check if the font properties are the same
if run1.font.name != run2.font.name and examine_font_name:
return False
return 0
if run1.font.size != run2.font.size and examine_font_size:
return False
return 0
if run1.font.bold != run2.font.bold and examine_font_bold:
return False
return 0
if run1.font.italic != run2.font.italic and examine_font_italic:
return False
return 0
if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
return False
return 0
if run1.font.underline != run2.font.underline and examine_font_underline:
return False
return 0
# fixme: Actually there are more properties to be compared, but we cannot get them through pptx
return True
return 1
def check_strikethrough(pptx_path, rules):
@@ -220,15 +253,20 @@ def check_left_panel(accessibility_tree):
root = ET.fromstring(accessibility_tree)
for root_pane in root.iter('root-pane'):
for panel in root_pane.iter('panel'):
for split_pane in panel.iter('split-pane'):
# Get the left panel
if split_pane.attrib.get("{{{}}}parentcoord".format(namespaces['cp'])) == "(0, 0)":
# Get the visible attribute
visible = split_pane.attrib.get("{{{}}}visible".format(namespaces['st']))
if visible:
# decide if it is left panel
return 1.
for root_pane in root.iter('root-pane'):
for split_pane in root_pane.iter('split-pane'):
for panel in split_pane.iter('panel'):
for scroll_panel in panel.iter('scroll-pane'):
for document_frame in scroll_panel.iter('document-frame'):
# Get the left panel
panel_name = document_frame.get("name")
# visible = scroll_bar.attrib.get(f"{{{namespaces['st']}}}visible")
if panel_name == "Slides View":
# Left panel is open
return 1.0
return 0.
# Left panel is not open
return 0.0
# print(compare_pptx_files("D:\\NJU\\HKUNLP\\Desktop-Env\\cache\\bf4e9888-f10f-47af-8dba-76413038b73c\\4.3-Template_4.29.2016.pptx", "D:\\NJU\HKUNLP\\Desktop-Env\\cache\\bf4e9888-f10f-47af-8dba-76413038b73c\\4.3-Template_4.29.2016_Gold.pptx"))

22
evaluation_examples/examples/libreoffice_impress/0f84bef9-9790-432e-92b7-eece357603fb.json
View File

@@ -3,7 +3,25 @@
"snapshot": "libreoffice_impress",
"instruction": "On it Whenever I launch a LibreOffice Impress, it uses both screens, one for current slide and next slide and another for actual presentation. What I want is to use only one monitor which shows presentation. I dont want the screen with Current slide and Next slide so that it can be used for other purposes. How should I achieve this?",
"source": "https://stackoverflow.com/questions/29036788/how-to-disable-libreoffice-impress-to-use-multiple-display",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1qKOdf1Wx9nGtk_3l7hjZ9gXWFzWgsyoH&export=download&authuser=0&confirm=t&uuid=0bceb604-af00-4940-a137-8dd00512d060&at=APZUnTUlTutATfe49vsbBrobLPAG:1706370599333",
"path": "Desktop/multimedia_classroom_podium-2020.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/multimedia_classroom_podium-2020.pptx"
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_impress"
@@ -16,4 +34,4 @@
"dest": "registrymodifications.xcu"
}
}
}
}

58
evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json
View File

@@ -28,35 +28,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "lec17-gui-events.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "lec17-gui-events.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "evaluate_presentation_fill_to_rgb_distance",
"expected": {

31
evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -27,6 +27,37 @@
"libreoffice_impress"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "wssf-project-plan-on-a-page.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_images",
"expected": {
"type": "cloud_file",

87
evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -20,6 +20,22 @@
"parameters": {
"path": "Desktop/New_Club_Spring_2018_Training.pptx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(4); pyautogui.doubleClick(x=200, y=600); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
@@ -28,43 +44,50 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "New_Club_Spring_2018_Training.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "New_Club_Spring_2018_Training.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5)"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_strikethrough",
"expected": {
"type": "rule",
"rules": {
"slide_index_s": [4],
"shape_index_s": [1],
"paragraph_index_s": [1, 2]
"slide_index_s": [
4
],
"shape_index_s": [
1
],
"paragraph_index_s": [
1,
2
]
}
},
"result": {

58
evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json
View File

@@ -28,35 +28,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "CPD_Background_Investigation_Process.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_image_stretch_and_center",
"expected": {

60
evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json
View File

@@ -28,35 +28,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "MLA_Workshop_061X_Works_Cited.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "MLA_Workshop_061X_Works_Cited.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_pptx_files",
"expected": {
@@ -66,7 +66,7 @@
},
"result": {
"type": "vm_file",
"path": "/home/user/Desktop/MLA_Workshop_061X_Works_Cited.pptx",
"path": "Desktop/MLA_Workshop_061X_Works_Cited.pptx",
"dest": "MLA_Workshop_061X_Works_Cited.pptx"
}
}

58
evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json
View File

@@ -28,35 +28,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Forests.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "Forests.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_pptx_files",
"expected": {

142
evaluation_examples/examples/libreoffice_impress/bf4e9888-f10f-47af-8dba-76413038b73c.json Normal file
View File

@@ -0,0 +1,142 @@
{
"id": "bf4e9888-f10f-47af-8dba-76413038b73c",
"snapshot": "libreoffice_impress",
"instruction": "I have a series of .png images, saved as pic1.png, pic2.png,..., pic6.png on the Desktop. Now I want to insert these pictures into the current blank presentation to create a document suitable to run continuously in a kiosk or multimedia show. Could you help me?",
"source": "https://help.libreoffice.org/6.4/en-US/text/simpress/guide/photo_album.html?DbPAR=IMPRESS",
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1o8BOn9Q5NpQYYcXxeCHL_znkG11ni7ws&export=download&authuser=0&confirm=t&uuid=21a62a00-fd30-4405-ae87-00c3a165e381&at=APZUnTWlhaHv92g3wMYQJn80Vdpo:1706366778535",
"path": "Desktop/4.3-Template_4.29.2016.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/4.3-Template_4.29.2016.pptx"
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=15mklOcjIUJkzU3GhFhZM69h5gG0WOFVa&export=download&authuser=0&confirm=t&uuid=f7f61d48-5354-4eed-b743-ef620f308d79&at=APZUnTWzBs107NBYI1OvLHId8ff5:1706365135272",
"path": "Desktop/pic1.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=13-ATVSrmAIUNAzGlE65JkkHpa4d87tqj&export=download&authuser=0&confirm=t&uuid=5ebcaa42-805b-4176-9262-1a320e668666&at=APZUnTW0H_QRSJYAFPe4OppuwE5o:1706365136587",
"path": "Desktop/pic2.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1-K1XgMJj3yj7bz3BWm9Zddk4_lvYO4IP&export=download&authuser=0&confirm=t&uuid=6c268649-b627-49aa-a1aa-c97510b96e69&at=APZUnTV630QYwgh3nx-jtBp_uGAc:1706365137855",
"path": "Desktop/pic3.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1Bk8esN_UFxOU50AMxJoSY8C-mQgX3uY3&export=download&authuser=0&confirm=t&uuid=c5a6ae81-426e-40e2-ac1b-16ba161e347d&at=APZUnTWD9bHDiTsGWBE1tJfcR6Zp:1706365140032",
"path": "Desktop/pic4.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1DZapoWlmXPYsFdFGppCj2F-MZvpGZXca&export=download&authuser=0&confirm=t&uuid=7afc4de2-cdee-4c9e-9e4d-305509054627&at=APZUnTXX-2uDuQdsmA5kQRzDGRs7:1706365140910",
"path": "Desktop/pic5.png"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1jwBqiulZbICiCn3c9W6avo0-Mu7yeIAV&export=download&authuser=0&confirm=t&uuid=64f0e1d1-ec71-453a-887e-b81218e4e756&at=APZUnTWQeckP8zn8b8grHfMtQYr2:1706365141943",
"path": "Desktop/pic6.png"
}
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"libreoffice_impress"
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "4.3-Template_4.29.2016.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_pptx_files",
"expected": {
"type": "cloud_file",
"path": "https://drive.usercontent.google.com/download?id=1WeqZYQ0ZZvcFowo8rK15XHEIjoFCMVTN&export=download&authuser=0&confirm=t&uuid=e3bb39f8-0606-4bdf-808c-5b26f158dd88&at=APZUnTU_aAkyljmX3R_fkT5geNPv:1706366780602",
"dest": "4.3-Template_4.29.2016_Gold.pptx"
},
"result": {
"type": "vm_file",
"path": "Desktop/4.3-Template_4.29.2016.pptx",
"dest": "4.3-Template_4.29.2016.pptx"
},
"options": {
"examine_shape": false
}
}
}

58
evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json
View File

@@ -39,35 +39,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Mady_and_Mia_Baseball.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "Mady_and_Mia_Baseball.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "compare_audios",
"result": {

58
evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json
View File

@@ -28,35 +28,35 @@
],
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
{
"type": "activate_window",
"parameters": {
"window_name": "AM_Last_Page_Template.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_slide_orientation_Portrait",
"result": {