lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add impress examples; remove the auto-saving pyautogui commands change to libreoffice pre-setting

Browse Source
This commit is contained in:
Timothyxxx
2024-01-29 21:34:58 +08:00
parent 17514907d9
commit 343813a29b
29 changed files with 163 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -18,7 +18,7 @@ from .libreoffice import check_libre_locale
from .pdf import check_pdf_pages
from .slides import check_presenter_console_disable, check_image_stretch_and_center, check_slide_numbers_color, \
compare_pptx_files, check_strikethrough, \
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel
check_slide_orientation_Portrait, evaluate_presentation_fill_to_rgb_distance, check_left_panel, check_transition
# from .table import check_sheet_list, check_xlsx_freeze, check_xlsx_zoom, check_data_validations
from .table import compare_table, compare_csv
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter

6
desktop_env/evaluators/metrics/docs.py
View File

@@ -253,15 +253,15 @@ def evaluate_colored_words_in_tables(file_path1, file_path2):
def check_highlighted_words(file_path1, file_path2):
if not compare_docx_files(file_path1, file_path2):
return 0
# if not compare_docx_files(file_path1, file_path2):
# return 0
# Extract content.xml from the .odt file
extract_dir = file_path1 + "_extracted"
with zipfile.ZipFile(file_path1, 'r') as zip_ref:
zip_ref.extractall(extract_dir)
content_xml_path = os.path.join(extract_dir, 'content.xml')
with open(content_xml_path, 'r') as file:
with open(content_xml_path, 'r', encoding="utf-8") as file:
content_xml = file.read()
# Check for yellow highlights in the content.xml

42
desktop_env/evaluators/metrics/slides.py
View File

@@ -1,5 +1,6 @@
import logging
import xml.etree.ElementTree as ET
import zipfile
from math import sqrt
from pptx import Presentation
@@ -189,7 +190,8 @@ def compare_pptx_files(file1_path, file2_path, **options):
if run1.font.underline != run2.font.underline and examine_font_underline:
return 0
if ('strike' in run1.font._element.attrib) != ('strike' in run2.font._element.attrib) and examine_strike_through:
if ('strike' in run1.font._element.attrib) != (
'strike' in run2.font._element.attrib) and examine_strike_through:
return 0
# fixme: Actually there are more properties to be compared, but we cannot get them through pptx
@@ -278,3 +280,41 @@ def check_left_panel(accessibility_tree):
# Left panel is not open
return 0.0
def check_transition(pptx_file, rules):
slide_idx = rules['slide_idx']
transition_type = rules['transition_type']
# Use the zipfile module to open the .pptx file
with zipfile.ZipFile(pptx_file, 'r') as zip_ref:
# Get the slide XML file
slide_name = 'ppt/slides/slide{}.xml'.format(slide_idx + 1)
try:
zip_ref.getinfo(slide_name)
except KeyError:
# Slide does not exist
return 0.
with zip_ref.open(slide_name) as slide_file:
# 解析XML
tree = ET.parse(slide_file)
root = tree.getroot()
# XML namespace
namespaces = {
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
}
# Search for the transition element
transition = root.find('.//p:transition', namespaces)
if transition is not None:
# Check if the transition is an expected transition
dissolve = transition.find('.//p:{}'.format(transition_type), namespaces)
if dissolve is not None:
return 1.
else:
return 0.
else:
return 0.

85
evaluation_examples/examples/libreoffice_impress/21760ecb-8f62-40d2-8d85-0cee5725cb72.json
View File

@@ -3,10 +3,89 @@
"snapshot": "libreoffice_impress",
"instruction": "Could you help me add silde transition \"dissolve\" to my first page?",
"source": "https://www.libreofficehelp.com/add-animations-transitions-libreoffice-impress-slides/",
"config": [],
"config": [
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.usercontent.google.com/download?id=1A9nNj3hWBB0Bnr2Cf7zHs3WFiTKdEeNt&export=download&authuser=0&confirm=t&uuid=7c0f29fd-1f1a-4d48-b717-5c677373e615&at=APZUnTX9YQH_-UbVyFRLblwZ3rAn:1706530787330",
"path": "Desktop/Ch4 Video Effect.pptx"
}
]
}
},
{
"type": "open",
"parameters": {
"path": "Desktop/Ch4 Video Effect.pptx"
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(4); pyautogui.doubleClick(x=960, y=540); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
""
"libreoffice_impress"
],
"evaluator": "evaluation_dir"
"evaluator": {
"postconfig": [
{
"type": "activate_window",
"parameters": {
"window_name": "Ch4 Video Effect.pptx - LibreOffice Impress",
"strict": true
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
]
}
},
{
"type": "sleep",
"parameters": {
"seconds": 0.5
}
}
],
"func": "check_transition",
"expected": {
"type": "rule",
"rules": {
"slide_idx": 0,
"transition_type": "dissolve"
}
},
"result": {
"type": "vm_file",
"path": "Desktop/Ch4 Video Effect.pptx",
"dest": "Ch4 Video Effect.pptx"
}
}
}

2
evaluation_examples/examples/libreoffice_impress/550ce7e7-747b-495f-b122-acdc4d0b8e54.json
View File

@@ -33,7 +33,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; time.sleep(4); pyautogui.doubleClick(x=200, y=600); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
"import pyautogui; import time; time.sleep(4); pyautogui.doubleClick(x=200, y=650); time.sleep(0.5);pyautogui.mouseDown(); pyautogui.mouseUp(); time.sleep(0.5);"
]
}
}

2
evaluation_examples/examples/libreoffice_writer/0810415c-bde4-4443-9047-d5f70165a697.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/0a0faba3-5580-44df-965d-f562a99b291c.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/0b17a146-2934-46c7-8727-73ff6b6483e8.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/0e47de2a-32e0-456c-a366-8c607ef7a9d2.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/0e763496-b6bb-4508-a427-fad0b6c3e195.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/3ef2b351-8a84-4ff2-8724-d86eae9b842e.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/41c621f7-3544-49e1-af8d-dafd0f834f75.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/45d61a06-6545-4422-97b7-bc76cfa964c1.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/663876c7-3471-43db-ba51-f410b13d9d7d.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/66399b0d-8fda-4618-95c4-bfc6191617e9.json
View File

@@ -57,7 +57,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5);"
]
}
}

2
evaluation_examples/examples/libreoffice_writer/6ada715d-3aae-4a32-a6a7-429b2e43fb93.json
View File

@@ -68,7 +68,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/6f81754e-285d-4ce0-b59e-af7edb02d108.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/8472fece-c7dd-4241-8d65-9b3cd1a0b568.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/88fe4b2d-3040-4c70-9a70-546a47764b48.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/936321ce-5236-426a-9a20-e0e3c5dc536f.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/adf5e2c3-64c7-4644-b7b6-d2f0167927e7.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/b21acd93-60fd-4127-8a43-2f5178f4a830.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/d53ff5ee-3b1a-431e-b2be-30ed2673079b.json
View File

@@ -48,7 +48,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/e246f6d8-78d7-44ac-b668-fcf47946cb50.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/e528b65e-1107-4b8c-8988-490e4fece599.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

2
evaluation_examples/examples/libreoffice_writer/ecc2413d-8a48-416e-a3a2-d30106ca36cb.json
View File

@@ -47,7 +47,7 @@
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); pyautogui.press('down'); time.sleep(0.5); pyautogui.press('enter')"
"import pyautogui; import time; pyautogui.hotkey('ctrl', 's'); time.sleep(0.5); "
]
}
}

24
main.py
View File

@@ -46,12 +46,12 @@ def human_agent():
Runs the Gym environment with human input.
"""
with open("evaluation_examples/examples/libreoffice_writer/72b810ef-4156-4d09-8f08-a0cf57e7cefe.json", "r") as f:
with open("evaluation_examples/examples/libreoffice_writer/6a33f9b9-0a56-4844-9c3f-96ec3ffb3ba2.json", "r") as f:
example = json.load(f)
example["snapshot"] = "base18"
example["snapshot"] = "exp_v1"
env = DesktopEnv(
path_to_vm=r"D:\Ubuntu\Ubuntu\Ubuntu.vmx",
path_to_vm=r"C:\Users\tianbaox\Documents\Virtual Machines\Ubuntu\Ubuntu.vmx",
action_space="computer_13",
task_config=example
)
@@ -60,14 +60,14 @@ def human_agent():
done = False
trajectory = [
{
"action_type": "MOVE_TO",
"parameters": {
"x": 754,
"y": 1057
}
},
{"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
# {
# "action_type": "MOVE_TO",
# "parameters": {
# "x": 754,
# "y": 1057
# }
# },
# {"action_type": "CLICK", "parameters": {"button": "right", "num_clicks": 1}}
]
for i in range(len(trajectory)):
@@ -91,7 +91,7 @@ def human_agent():
logger.info("The episode is done.")
break
#input("PAUSING")
input("PAUSING")
result = env.evaluate()
logger.info("Result: %.2f", result)