lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'xiaochuanli/addChromeExtensions' of github.com:xlang-ai/DesktopEnv into xiaochuanli/addChromeExtensions

Browse Source
This commit is contained in:
Jason Lee
2024-02-18 22:16:48 +08:00
parent 17cd897780 84051ab340
commit e31e1dacde
69 changed files with 2852 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
desktop_env/evaluators/getters/general.py
View File

@@ -13,6 +13,8 @@ def get_vm_command_line(env, config: Dict[str, str]):
response = requests.post(f"http://{vm_ip}:{port}/execute", json={"command": command, "shell": shell})
print(response.json())
if response.status_code == 200:
return response.json()["output"]
else:

7
desktop_env/evaluators/metrics/__init__.py
View File

@@ -32,9 +32,7 @@ from .docs import (
evaluate_alignment,
get_unique_train_ids,
check_no_duplicates,
compare_init_lines
)
from .docs import (
compare_init_lines,
find_default_font,
contains_page_break,
compare_docx_files,
@@ -89,7 +87,8 @@ from .slides import (
evaluate_presentation_fill_to_rgb_distance,
check_left_panel,
check_transition,
check_page_number_colors
check_page_number_colors,
check_auto_saving_time
)
from .table import (
compare_table,

1
desktop_env/evaluators/metrics/basic_os.py
View File

@@ -46,7 +46,6 @@ def check_text_enlarged(scaling_factor_str):
def check_moved_jpgs(directory_list, rule):
expected_jpgs = rule["expected"]
moved_jpgs = [node['name'] for node in directory_list['children']]

164
desktop_env/evaluators/metrics/slides.py
View File

@@ -65,7 +65,6 @@ def check_image_stretch_and_center(modified_ppt, original_ppt):
def is_red_color(color):
# judge if the color is red
print(color.rgb)
return color and color.rgb == (255, 0, 0)
@@ -94,7 +93,6 @@ def check_slide_numbers_color(pptx_file_path):
# "SlidePlaceholder" is the name of the placeholder in the master slide
page_number_text = shape.text
font_color = get_master_placeholder_color(presentation)
print(font_color)
return 1 if font_color is not None and is_red_color(font_color) else 0
@@ -139,6 +137,7 @@ def compare_pptx_files(file1_path, file2_path, **options):
examine_number_of_slides = options.get("examine_number_of_slides", True)
examine_shape = options.get("examine_shape", True)
examine_text = options.get("examine_text", True)
examine_indent = options.get("examine_indent", True)
examine_font_name = options.get("examine_font_name", True)
examine_font_size = options.get("examine_font_size", True)
examine_font_bold = options.get("examine_font_bold", True)
@@ -146,6 +145,13 @@ def compare_pptx_files(file1_path, file2_path, **options):
examine_color_rgb = options.get("examine_color_rgb", True)
examine_font_underline = options.get("examine_font_underline", True)
examine_strike_through = options.get("examine_strike_through", True)
examine_alignment = options.get("examine_alignment", True)
examine_bottom_position = options.get("examine_bottom_position", False)
examine_right_position = options.get("examine_right_position", False)
examine_image_size = options.get("examine_image_size", True)
examine_bullets = options.get("examine_bullets", True)
examine_background_color = options.get("examine_background_color", True)
examine_note = options.get("examine_note", True)
# compare the number of slides
if len(prs1.slides) != len(prs2.slides) and examine_number_of_slides:
@@ -153,26 +159,67 @@ def compare_pptx_files(file1_path, file2_path, **options):
# compare the content of each slide
for slide1, slide2 in zip(prs1.slides, prs2.slides):
def get_slide_background_color(slide):
background = slide.background
if background.fill.background():
return background.fill.fore_color.rgb
else:
return None
if get_slide_background_color(slide1) != get_slide_background_color(slide2) and examine_background_color:
return 0
def get_slide_notes(slide):
notes_slide = slide.notes_slide
if notes_slide:
return notes_slide.notes_text_frame.text
else:
return None
if get_slide_notes(slide1) != get_slide_notes(slide2) and examine_note:
return 0
# check if the shapes are the same
for shape1, shape2 in zip(slide1.shapes, slide2.shapes):
if (
shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
if examine_bottom_position and shape1.top != shape2.top:
if hasattr(shape1, "text") and hasattr(shape2, "text") and shape1.text == shape2.text and shape1.text == "Product Comparison":
if shape1.top >= shape2.top:
return 0
if examine_right_position and shape1.left != shape2.left:
if not hasattr(shape1, "text") and not hasattr(shape2, "text"):
if shape1.left >= shape2.left:
return 0
if (shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height) and examine_shape:
return 0
if examine_image_size:
if shape1.shape_type == 13 and shape2.shape_type == 13:
if shape1.width != shape2.width or shape1.height != shape2.height:
return 0
elif shape1.left != shape2.left or shape1.top != shape2.top or shape1.width != shape2.width or shape1.height != shape2.height:
return 0
if hasattr(shape1, "text") and hasattr(shape2, "text"):
if shape1.text != shape2.text and examine_text:
return 0
return 0
# check if the paragraphs are the same
for para1, para2 in zip(shape1.text_frame.paragraphs, shape2.text_frame.paragraphs):
if para1.alignment != para2.alignment and examine_alignment:
return 0
# check if the runs are the same
for run1, run2 in zip(para1.runs, para2.runs):
if run1.text != run2.text and examine_text:
return 0
if para1.text != para2.text and examine_text:
return 0
# check if the font properties are the same
if run1.font.name != run2.font.name and examine_font_name:
if para1.level != para2.level and examine_indent:
return 0
for run1, run2 in zip(para1.runs, para2.runs):
# check if the font properties are the same
if run1.font.name != run2.font.name and examine_font_name:
return 0
if run1.font.size != run2.font.size and examine_font_size:
@@ -184,16 +231,52 @@ def compare_pptx_files(file1_path, file2_path, **options):
if run1.font.italic != run2.font.italic and examine_font_italic:
return 0
if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
return 0
if hasattr(run1.font.color, "rgb") and hasattr(run2.font.color, "rgb"):
if run1.font.color.rgb != run2.font.color.rgb and examine_color_rgb:
return 0
if run1.font.underline != run2.font.underline and examine_font_underline:
return 0
if run1.font._element.attrib.get('strike', 'noStrike') != run2.font._element.attrib.get('strike', 'noStrike') and examine_strike_through:
if run1.font._element.attrib.get('strike', 'noStrike') != run2.font._element.attrib.get(
'strike', 'noStrike') and examine_strike_through:
return 0
# fixme: Actually there are more properties to be compared, but we cannot get them through pptx
def _extract_bullets(xml_data):
root = ET.fromstring(xml_data)
namespaces = {
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
}
bullets = []
for paragraph in root.findall('.//a:p', namespaces):
pPr = paragraph.find('a:pPr', namespaces)
if pPr is not None:
lvl = pPr.get('lvl')
buChar = pPr.find('a:buChar', namespaces)
char = buChar.get('char') if buChar is not None else "No Bullet"
buClr = pPr.find('a:buClr/a:srgbClr', namespaces)
color = buClr.get('val') if buClr is not None else "No Color"
else:
lvl = "No Level"
char = "No Bullet"
color = "No Color"
text = "".join(t.text for t in paragraph.findall('.//a:t', namespaces))
bullets.append((lvl, char, text, color))
return bullets
if examine_bullets and _extract_bullets(run1.part.blob.decode('utf-8')) != _extract_bullets(run2.part.blob.decode('utf-8')):
return 0
# fixme: Actually there are more properties to be compared, we can add them later via parsing the xml data
return 1
@@ -371,6 +454,51 @@ def check_page_number_colors(pptx_file, rules):
return 1
def check_auto_saving_time(pptx_file, rules):
minutes = rules["minutes"]
# open and parse xml file
try:
tree = ET.parse(pptx_file)
root = tree.getroot()
# Traverse the XML tree to find the autosave time setting
autosave_time = None
for item in root.findall(".//item"):
# Check the path attribute
path = item.get('{http://openoffice.org/2001/registry}path')
if path == "/org.openoffice.Office.Common/Save/Document":
# Once the correct item is found, look for the prop element with the name "AutoSaveTimeIntervall"
for prop in item.findall(".//prop"):
name = prop.get('{http://openoffice.org/2001/registry}name')
if name == "AutoSaveTimeIntervall":
# Extract the value of the autosave time interval
autosave_time = prop.find(".//value").text
break
if autosave_time is None:
return 0
else:
autosave_time = int(autosave_time)
if autosave_time == minutes:
return 1
else:
return 0
except ET.ParseError as e:
logger.error(f"Error parsing XML: {e}")
except FileNotFoundError:
logger.error(f"File not found: {pptx_file}")
if __name__ == '__main__':
print(compare_pptx_files(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx", r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
# print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
# print(compare_pptx_files(
# r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx",
# r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\550ce7e7-747b-495f-b122-acdc4d0b8e54\New_Club_Spring_2018_Training_Gold.pptx"))
# print(evaluate_presentation_fill_to_rgb_distance(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\3b27600c-3668-4abd-8f84-7bcdebbccbdb\lec17-gui-events.pptx", {"rgb": (0, 0, 255)}))
# print(check_auto_saving_time(r"C:\Users\tianbaox\Desktop\DesktopEnv\cache\2cd43775-7085-45d8-89fa-9e35c0a915cf\registrymodifications.xcu", {"minutes": 3}))
print(compare_pptx_files(
r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6_Gold.pptx",
r"D:\NJU\HKUNLP\Desktop-Env\DesktopEnv\cache\08aced46-45a2-48d7-993b-ed3fb5b32302\22_6.pptx",
examine_shape=False))