lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix some errors found in impress and thunderbird examples

Browse Source
This commit is contained in:
Timothyxxx
2024-01-29 20:14:47 +08:00
parent 461651e127
commit f9d9895541
14 changed files with 56 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -32,5 +32,5 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
check_qt_slider_colours, check_global_key_play_pause check_qt_slider_colours, check_global_key_play_pause
from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \ from .vscode import compare_text_file, compare_config, compare_answer, is_extension_installed, check_json_settings, \
check_json_keybindings check_json_keybindings
from .os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs from .basic_os import check_gnome_favorite_apps, is_utc_0, check_text_enlarged, check_moved_jpgs

0
desktop_env/evaluators/metrics/os.py → desktop_env/evaluators/metrics/basic_os.py
View File

42
desktop_env/evaluators/metrics/slides.py
View File

@@ -96,6 +96,7 @@ def check_slide_numbers_color(pptx_file_path):
print(font_color) print(font_color)
return 1 if font_color is not None and is_red_color(font_color) else 0 return 1 if font_color is not None and is_red_color(font_color) else 0
# import numpy as np # import numpy as np
# from PIL import Image # from PIL import Image
# from skimage.metrics import structural_similarity as ssim # from skimage.metrics import structural_similarity as ssim
@@ -143,6 +144,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
examine_font_italic = options.get("examine_font_italic", True) examine_font_italic = options.get("examine_font_italic", True)
examine_color_rgb = options.get("examine_color_rgb", True) examine_color_rgb = options.get("examine_color_rgb", True)
examine_font_underline = options.get("examine_font_underline", True) examine_font_underline = options.get("examine_font_underline", True)
examine_strike_through = options.get("examine_strike_through", True)
# compare the number of slides # compare the number of slides
if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides: if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
@@ -156,7 +158,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
if ( if (
shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape: shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
return 0 return 0
if hasattr(shape1, "text") and hasattr(shape2, "text"): if hasattr(shape1, "text") and hasattr(shape2, "text"):
if shape1.text != shape2.text and examine_text: if shape1.text != shape2.text and examine_text:
return 0 return 0
@@ -187,6 +189,9 @@ def compare_pptx_files(file1_path, file2_path, **options):
if run1.font.underline != run2.font.underline and examine_font_underline: if run1.font.underline != run2.font.underline and examine_font_underline:
return 0 return 0
if ('strike' in run1.font._element.attrib) != ('strike' in run2.font._element.attrib) and examine_strike_through:
return 0
# fixme: Actually there are more properties to be compared, but we cannot get them through pptx # fixme: Actually there are more properties to be compared, but we cannot get them through pptx
return 1 return 1
@@ -200,21 +205,27 @@ def check_strikethrough(pptx_path, rules):
shape_index_s = rules["shape_index_s"] shape_index_s = rules["shape_index_s"]
paragraph_index_s = rules["paragraph_index_s"] paragraph_index_s = rules["paragraph_index_s"]
for slide_index in slide_index_s: try:
# Get the slide for slide_index in slide_index_s:
slide = presentation.slides[slide_index] # Get the slide
slide = presentation.slides[slide_index]
for shape_index in shape_index_s: for shape_index in shape_index_s:
# Get the text box # Get the text box
paragraphs = slide.shapes[shape_index].text_frame.paragraphs paragraphs = slide.shapes[shape_index].text_frame.paragraphs
for paragraph_index in paragraph_index_s: for paragraph_index in paragraph_index_s:
paragraph = paragraphs[paragraph_index] paragraph = paragraphs[paragraph_index]
run = paragraph.runs[0] run = paragraph.runs[0]
if 'strike' not in run.font._element.attrib: if 'strike' not in run.font._element.attrib:
return False return 0
return True
except Exception as e:
logger.error(f"Error: {e}")
return 0
return 1
def check_slide_orientation_Portrait(pptx_path): def check_slide_orientation_Portrait(pptx_path):
@@ -253,7 +264,7 @@ def check_left_panel(accessibility_tree):
root = ET.fromstring(accessibility_tree) root = ET.fromstring(accessibility_tree)
for root_pane in root.iter('root-pane'): for root_pane in root.iter('root-pane'):
for split_pane in root_pane.iter('split-pane'): for split_pane in root_pane.iter('split-pane'):
for panel in split_pane.iter('panel'): for panel in split_pane.iter('panel'):
for scroll_panel in panel.iter('scroll-pane'): for scroll_panel in panel.iter('scroll-pane'):
@@ -267,6 +278,3 @@ def check_left_panel(accessibility_tree):
# Left panel is not open # Left panel is not open
return 0.0 return 0.0
# print(compare_pptx_files("D:\\NJU\\HKUNLP\\Desktop-Env\\cache\\bf4e9888-f10f-47af-8dba-76413038b73c\\4.3-Template_4.29.2016.pptx", "D:\\NJU\HKUNLP\\Desktop-Env\\cache\\bf4e9888-f10f-47af-8dba-76413038b73c\\4.3-Template_4.29.2016_Gold.pptx"))

2
evaluation_examples/examples/libreoffice_impress/3b27600c-3668-4abd-8f84-7bcdebbccbdb.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/455d3c66-7dc6-4537-a39a-36d3e9119df7.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

27
evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -9,7 +9,7 @@
"parameters": { "parameters": {
"files": [ "files": [
{ {
"url": "https://drive.usercontent.google.com/download?id=1fw0baVZ15s0r1WGEBftgED2H0ljZgYtu&export=download&authuser=0&confirm=t&uuid=df03788a-81ef-4e55-b33a-2fba7ab28cb8&at=APZUnTXPb-sm88KNwmNeugbhPrzx:17052529805399", "url": "https://drive.usercontent.google.com/download?id=126TZ3vOBN2XAfdEmvJVBaa4qs_MZlgN7&export=download&authuser=0&confirm=t&uuid=acea6223-64ff-44cf-9d83-fe1a1640d374&at=APZUnTVBY9Kkm1Xo3ZU-Fe8hoOqC:1706521734182",
"path": "Desktop/New_Club_Spring_2018_Training.pptx" "path": "Desktop/New_Club_Spring_2018_Training.pptx"
} }
] ]
@@ -63,7 +63,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5)" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },
@@ -74,26 +74,19 @@
} }
} }
], ],
"func": "check_strikethrough", "func": "compare_pptx_files",
"expected": { "expected": {
"type": "rule", "type": "cloud_file",
"rules": { "path": "https://drive.usercontent.google.com/download?id=1FNoJ8nGSNbsf9rhj9He1Lw17RCe2LJHD&export=download&authuser=0&confirm=t&uuid=8b876496-91c1-4209-be53-34437901b613&at=APZUnTUpOTXFXdPMATBQXSgJHsk7:1706521731233",
"slide_index_s": [ "dest": "New_Club_Spring_2018_Training_Gold.pptx"
4
],
"shape_index_s": [
1
],
"paragraph_index_s": [
1,
2
]
}
}, },
"result": { "result": {
"type": "vm_file", "type": "vm_file",
"path": "Desktop/New_Club_Spring_2018_Training.pptx", "path": "Desktop/New_Club_Spring_2018_Training.pptx",
"dest": "New_Club_Spring_2018_Training.pptx" "dest": "New_Club_Spring_2018_Training.pptx"
} },
"options": {
"examine_shape": false
}
} }
} }

2
evaluation_examples/examples/libreoffice_impress/5d901039-a89c-4bfb-967b-bf66f4df075e.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/9ec204e4-f0a3-42f8-8458-b772a6797cab.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/af23762e-2bfd-4a1d-aada-20fa8de9ce07.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/bf4e9888-f10f-47af-8dba-76413038b73c.json
View File

@@ -113,7 +113,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/c59742c0-4323-4b9d-8a02-723c251deaa0.json
View File

@@ -58,7 +58,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

2
evaluation_examples/examples/libreoffice_impress/ce88f674-ab7a-43da-9201-468d38539e4a.json
View File

@@ -47,7 +47,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter'); time.sleep(0.5);" "import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
] ]
} }
}, },

12
evaluation_examples/examples/libreoffice_impress/ef9d12bd-bcee-4ba0-a40e-918400f43ddf.json
View File

@@ -12,7 +12,17 @@
"--impress" "--impress"
] ]
} }
} },
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.click(960, 540); time.sleep(0.5); pyautogui.press('esc'); time.sleep(0.3); pyautogui.press('f10'); time.sleep(0.3); pyautogui.press('right', presses=2, interval=0.1); time.sleep(0.3); pyautogui.press('down', presses=11, interval=0.1); pyautogui.press('enter')"
]
}
}
], ],
"trajectory": "trajectories/", "trajectory": "trajectories/",
"related_apps": [ "related_apps": [

2
evaluation_examples/examples/libreoffice_impress/f0a334af-f91b-4c03-b578-aac9bec2b543.json
View File

@@ -58,7 +58,7 @@
"command": [ "command": [
"python", "python",
"-c", "-c",
"import pyautogui; pyautogui.press([\"ctrl\", \"s\"]);" "import pyautogui; pyautogui.hotkey('ctrl', 's');"
] ]
} }
}, },