lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Loading libreoffice writer examples and find few problems, will do another round tomorrow for the rest

Browse Source
This commit is contained in:
Timothyxxx
2024-01-02 17:50:05 +08:00
parent 8ac88e9617
commit 03e99a68fb
22 changed files with 191 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
desktop_env/envs/desktop_env.py
View File

@@ -78,7 +78,8 @@ class DesktopEnv(gym.Env):
self.evaluator = task_config["evaluator"]
self.metric: Metric = getattr(metrics, self.evaluator["func"])
self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(
self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
self.metric_options: Dict[str, Any] = self.evaluator.get("options", {})
# Initialize emulator and controller
@@ -95,7 +96,7 @@ class DesktopEnv(gym.Env):
# episodic stuffs, like tmp dir and counters, will be updated or reset
# when calling self.reset()
self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
self.tmp_dir: str = self.tmp_dir_base # just an init value, updated during reset
self._traj_no: int = -1
self._step_no: int = 0
self.action_history: List[Dict[str, any]] = []
@@ -165,7 +166,8 @@ class DesktopEnv(gym.Env):
self.evaluator = task_config["evaluator"]
self.metric: Metric = getattr(metrics, self.evaluator["func"])
self.result_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["result"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(self.evaluator["expected"]["type"]))
self.expected_getter: Getter = getattr(getters, "get_{:}".format(
self.evaluator["expected"]["type"])) if "expected" in self.evaluator else None
self.metric_options = self.evaluator.get("options", {})
self.setup_controller.reset_cache_dir(self.cache_dir)
@@ -226,20 +228,12 @@ class DesktopEnv(gym.Env):
"""
Evaluate whether the task is successfully completed.
"""
result_state = self.result_getter(self, self.evaluator["result"])
expected_state = self.expected_getter(self, self.evaluator["expected"]) if "expected" in self.evaluator \
else None
# todo: make this more flexible by refactoring
# eval_func = eval_funcs[self.evaluator["func"]]
# eval_func_vars = {}
#
# for var_name, file_info in self.evaluator["paths"].items():
# path = copy_file_to_local(file_info)
# eval_func_vars[var_name] = path
#
# return eval_func(**eval_func_vars)
result = self.result_getter(self, self.evaluator["result"])
expected = self.expected_getter(self, self.evaluator["expected"])
metric: float = self.metric(result, expected, **self.metric_options)
metric: float = self.metric(result_state, expected_state, **self.metric_options) if expected_state is not None \
else self.metric(result_state, **self.metric_options)
return metric

1
desktop_env/evaluators/getters/file.py
View File

@@ -41,3 +41,4 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
f.write(file)
return _path

2
desktop_env/evaluators/getters/misc.py
View File

@@ -5,4 +5,4 @@ def get_rule(env, config: R) -> R:
"""
Returns the rule as-is.
"""
return config
return config["rules"]

6
desktop_env/evaluators/metrics/README.md
View File

@@ -1,5 +1,11 @@
# Setup Instructions
## LibreOffice Writer
### Setting Up the python-docx Library
```shell
pip install python-docx
```
## Chrome

54
desktop_env/evaluators/metrics/docs.py
View File

@@ -1,15 +1,20 @@
import xml.etree.ElementTree as ET
import os
from typing import List, Dict, Any
from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
def find_default_font(expected, config_file_path):
def find_default_font(config_file_path, rules):
"""Find the default font in LibreOffice Writer."""
default_font = None
expected_font = rules["font_name"]
try:
tree = ET.parse(config_file_path)
root = tree.getroot()
# Define the XML namespace used in the file
# Define the XML namespace used in the file
namespace = {'oor': 'http://openoffice.org/2001/registry'}
# Search for the node containing the default font setting for LibreOffice Writer
@@ -19,24 +24,26 @@ def find_default_font(expected, config_file_path):
default_font = value.text
except Exception as e:
print(f"Error: {e}")
return 1 if default_font == expected else 0
return 1 if default_font == expected_font else 0
def contains_page_break(docx_file):
doc = Document(docx_file)
namespaces = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'}
for paragraph in doc.paragraphs:
for run in paragraph.runs:
br_elems = run.element.findall('.//w:br', namespaces)
for br in br_elems:
if br is not None and '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}type' in br.attrib and br.attrib['{http://schemas.openxmlformats.org/wordprocessingml/2006/main}type'] == 'page':
if br is not None and '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}type' in br.attrib and \
br.attrib['{http://schemas.openxmlformats.org/wordprocessingml/2006/main}type'] == 'page':
return 1
return 0
def compare_docx_files(file1, file2):
def compare_docx_files(file1, file2):
doc1 = Document(file1)
doc2 = Document(file2)
@@ -53,6 +60,7 @@ def compare_docx_files(file1, file2):
return 1
def compare_docx_tables(docx_file1, docx_file2):
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -78,8 +86,8 @@ def compare_docx_tables(docx_file1, docx_file2):
return 1
def compare_line_spacing(docx_file1, docx_file2):
def compare_line_spacing(docx_file1, docx_file2):
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -96,12 +104,12 @@ def compare_line_spacing(docx_file1, docx_file2):
return 0
return 1
def compare_insert_equation(docx_file1, docx_file2):
if not compare_docx_files(docx_file1, docx_file2):
return 0
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -112,8 +120,10 @@ def compare_insert_equation(docx_file1, docx_file2):
return 1
return 0
def compare_font_names(expected_font, docx_file):
def compare_font_names(docx_file, rules: List[Dict[str, Any]]):
doc = Document(docx_file)
expected_font = rules["font_name"]
for paragraph in doc.paragraphs:
for run in paragraph.runs:
@@ -122,6 +132,7 @@ def compare_font_names(expected_font, docx_file):
return 0
return 1
def compare_subscript_contains(docx_file1, docx_file2):
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
@@ -133,6 +144,7 @@ def compare_subscript_contains(docx_file1, docx_file2):
return 1
return 0
def has_page_numbers_in_footers(docx_file):
doc = Document(docx_file)
@@ -146,7 +158,6 @@ def has_page_numbers_in_footers(docx_file):
return 0
return 1
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
def is_first_line_centered(docx_file):
doc = Document(docx_file)
@@ -155,19 +166,20 @@ def is_first_line_centered(docx_file):
# check if the first line is center justified
return 1 if first_paragraph.paragraph_format.alignment == WD_PARAGRAPH_ALIGNMENT.CENTER else 0
import os
def check_file_exists(directory, filename):
def check_file_exists(directory, filename):
file_path = os.path.join(directory, filename)
return 1 if os.path.isfile(file_path) else 0
def compare_contains_image(docx_file1, docx_file2):
doc1 = Document(docx_file1)
doc2 = Document(docx_file2)
for para1, para2 in zip(doc1.paragraphs, doc2.paragraphs):
for run1, run2 in zip(para1.runs, para2.runs):
if ('graphicData' in run1._element.xml and 'graphicData' not in run2._element.xml) or ('graphicData' not in run1._element.xml and 'graphicData' in run2._element.xml):
if ('graphicData' in run1._element.xml and 'graphicData' not in run2._element.xml) or (
'graphicData' not in run1._element.xml and 'graphicData' in run2._element.xml):
return 0
return 1
@@ -178,6 +190,6 @@ def compare_contains_image(docx_file1, docx_file2):
# Replace 'your_document.docx' with the path to your document
# result = contains_page_break('your_document.docx')
# print(result)
#config_path = "/home/[username]/.config/libreoffice/4/user/registrymodifications.xcu"
#print(find_default_font("Ani", config_path))
# config_path = "/home/[username]/.config/libreoffice/4/user/registrymodifications.xcu"
# print(find_default_font("Ani", config_path))

44
desktop_env/evaluators/metrics/general.py
View File

@@ -1,5 +1,7 @@
import os
import platform
import subprocess
import ctypes
import os
# todo: move to getter module
@@ -13,3 +15,43 @@ def get_desktop_path():
return os.path.join("/home", username, "Desktop")
else:
raise Exception("Unsupported operating system")
def get_wallpaper():
def get_wallpaper_windows():
SPI_GETDESKWALLPAPER = 0x73
MAX_PATH = 260
buffer = ctypes.create_unicode_buffer(MAX_PATH)
ctypes.windll.user32.SystemParametersInfoW(SPI_GETDESKWALLPAPER, MAX_PATH, buffer, 0)
return buffer.value
def get_wallpaper_macos():
script = """
tell application "System Events" to tell every desktop to get picture
"""
process = subprocess.Popen(['osascript', '-e', script], stdout=subprocess.PIPE)
output, error = process.communicate()
if error:
print("Error:", error)
else:
return output.strip().decode('utf-8')
def get_wallpaper_linux():
try:
output = subprocess.check_output(["gsettings", "get", "org.gnome.desktop.background", "picture-uri"])
return output.decode('utf-8').strip().replace('file://', '').replace("'", "")
except Exception as e:
print("Error:", e)
return None
os_name = platform.system()
if os_name == 'Windows':
return get_wallpaper_windows()
elif os_name == 'Darwin':
return get_wallpaper_macos()
elif os_name == 'Linux':
return get_wallpaper_linux()
else:
return "Unsupported OS"