lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix minor bugs caused from merging in setupcontroller; Initialize vscode example loading

Browse Source
This commit is contained in:
Timothyxxx
2024-01-14 00:51:26 +08:00
parent 347160a35f
commit 2228f346a9
14 changed files with 148 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
desktop_env/controllers/setup.py
View File

@@ -20,6 +20,7 @@ class SetupController:
def __init__(self, vm_ip: str, cache_dir: str): def __init__(self, vm_ip: str, cache_dir: str):
self.vm_ip: str = vm_ip self.vm_ip: str = vm_ip
self.http_server: str = f"http://{vm_ip}:5000" self.http_server: str = f"http://{vm_ip}:5000"
self.http_server_setup_root: str = f"http://{vm_ip}:5000/setup"
self.cache_dir: str = cache_dir self.cache_dir: str = cache_dir
def reset_cache_dir(self, cache_dir: str): def reset_cache_dir(self, cache_dir: str):

11
desktop_env/evaluators/getters/general.py
View File

@@ -1,7 +1,5 @@
from typing import Dict
import os import os
import requests from typing import Dict
def get_string(env, config: Dict[str, str]) -> str: def get_string(env, config: Dict[str, str]) -> str:
@@ -12,12 +10,13 @@ def get_string(env, config: Dict[str, str]) -> str:
return config["string"] return config["string"]
def get_command_line(env, config: Dict[str, str]) -> str: def get_command_line(env, config: Dict[str, str]) -> str:
""" """
Config: Config:
string (str) string (str)
""" """
f = os.popen(config["command"]) f = os.popen(config["command"])
return f.read() return f.read()

2
desktop_env/evaluators/getters/misc.py
View File

@@ -1,6 +1,5 @@
import logging import logging
from typing import TypeVar from typing import TypeVar
#from typing import Dict, List
logger = logging.getLogger("desktopenv.getters.misc") logger = logging.getLogger("desktopenv.getters.misc")
@@ -13,6 +12,7 @@ def get_rule(env, config: R) -> R:
""" """
return config["rules"] return config["rules"]
def get_accessibility_tree(env, *args) -> str: def get_accessibility_tree(env, *args) -> str:
accessibility_tree: str = env.controller.get_accessibility_tree() accessibility_tree: str = env.controller.get_accessibility_tree()
logger.debug("AT@eval: %s", accessibility_tree) logger.debug("AT@eval: %s", accessibility_tree)

4
desktop_env/evaluators/getters/replay.py
View File

@@ -1,7 +1,7 @@
from typing import List, Dict, Any from typing import List, Dict, Any
def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
def parse(action): def parse(action):
if action["type"] == "hotkey": if action["type"] == "hotkey":
keys = "', '".join(action["param"]) keys = "', '".join(action["param"])
@@ -16,4 +16,4 @@ def get_replay(env, trajectory: List[Dict[str, Any]]) -> None:
return f"pyautogui.press('{key}')" return f"pyautogui.press('{key}')"
for action in trajectory: for action in trajectory:
env.controller.execute_python_command(parse(action)) env.controller.execute_python_command(parse(action))

19
desktop_env/evaluators/getters/vscode.py
View File

@@ -1,17 +1,18 @@
from typing import Dict
from typing import Any from typing import Any
from replay import get_replay from typing import Dict
from file import get_vm_file
from .file import get_vm_file
from .replay import get_replay
def get_vscode_config(env, config: Dict[str, Any]) -> str: def get_vscode_config(env, config: Dict[str, Any]) -> str:
trajectory = [{"type": "hotkey", "param": ["command", "shift", "p"]}, trajectory = [{"type": "hotkey", "param": ["command", "shift", "p"]},
{"type": "typewrite", "param": "OpenProject"}, {"type": "typewrite", "param": "OpenProject"},
{"type": "press", "param": "enter"}] {"type": "press", "param": "enter"}]
get_replay(env, trajectory) get_replay(env, trajectory)
return get_vm_file(env, { return get_vm_file(env, {
"path": config["path"], "path": config["path"],
"dest": config["dest"] "dest": config["dest"]
}) })

2
desktop_env/evaluators/metrics/__init__.py
View File

@@ -13,4 +13,4 @@ from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, co
from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions from .gimp import increase_saturation, decrease_brightness, check_file_exists, compare_triangle_positions
from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3 from .general import check_csv, check_accessibility_tree, check_list, run_sqlite3
from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter from .thunderbird import check_thunderbird_prefs, check_thunderbird_filter
from .vscode import compare_text_file, compare_config, compare_answer

6
desktop_env/evaluators/metrics/docs.py
View File

@@ -1,12 +1,14 @@
import xml.etree.ElementTree as ET import logging
import os import os
import xml.etree.ElementTree as ET
from typing import List, Dict, Any from typing import List, Dict, Any
from docx import Document from docx import Document
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
import logging
logger = logging.getLogger("desktopenv.metric.docs") logger = logging.getLogger("desktopenv.metric.docs")
def find_default_font(config_file_path, rules): def find_default_font(config_file_path, rules):
"""Find the default font in LibreOffice Writer.""" """Find the default font in LibreOffice Writer."""
default_font = None default_font = None

43
desktop_env/evaluators/metrics/libreoffice.py
View File

@@ -1,37 +1,38 @@
import lxml.cssselect
from lxml.etree import _Element as Element
import lxml.etree
import fnmatch import fnmatch
from typing import Dict, List from typing import Dict, List
import lxml.cssselect
import lxml.etree
from lxml.etree import _Element as Element
_libconf_namespaces = [("oor", "http://openoffice.org/2001/registry")] _libconf_namespaces = [("oor", "http://openoffice.org/2001/registry")]
_libconf_ns_mapping = dict(_libconf_namespaces) _libconf_ns_mapping = dict(_libconf_namespaces)
_setup_locale_selector = lxml.cssselect.CSSSelector( 'item[oor|path$=L10N]>prop[oor|name=ooSetupSystemLocale]>value' _setup_locale_selector = lxml.cssselect.CSSSelector('item[oor|path$=L10N]>prop[oor|name=ooSetupSystemLocale]>value',
, namespaces=_libconf_ns_mapping namespaces=_libconf_ns_mapping)
) _locale_selector = lxml.cssselect.CSSSelector('item[oor|path$=L10N]>prop[oor|name=ooLocale]>value',
_locale_selector = lxml.cssselect.CSSSelector( 'item[oor|path$=L10N]>prop[oor|name=ooLocale]>value' namespaces=_libconf_ns_mapping)
, namespaces=_libconf_ns_mapping
)
def check_libre_locale(config_file: str, rules: Dict[str, List[str]]) -> float: def check_libre_locale(config_file: str, rules: Dict[str, List[str]]) -> float:
config: Element = lxml.etree.parse(config_file).getroot() config: Element = lxml.etree.parse(config_file).getroot()
setup_locale_setting: List[Element] = _setup_locale_selector(config) setup_locale_setting: List[Element] = _setup_locale_selector(config)
locale_setting: List[Element] = _locale_selector(config) locale_setting: List[Element] = _locale_selector(config)
setup_locale_setting: str = setup_locale_setting[0].text\ setup_locale_setting: str = setup_locale_setting[0].text \
if len(setup_locale_setting)>0\ if len(setup_locale_setting) > 0 \
else locale_setting[0].text else locale_setting[0].text
return float( any( fnmatch.fnmatchcase(setup_locale_setting, ptn)\ return float(any(fnmatch.fnmatchcase(setup_locale_setting, ptn) \
for ptn in rules["locale_set"] for ptn in rules["locale_set"]
) )
) )
if __name__ == "__main__": if __name__ == "__main__":
path1 = "../../任务数据/LibreOffice Calc/registrymodifications.ru.xcu" path1 = "../../任务数据/LibreOffice Calc/registrymodifications.ru.xcu"
print( check_libre_locale( path1, { "locale_set": [ "ru-*", "de-*", "fr-*" print(check_libre_locale(path1, {"locale_set": ["ru-*", "de-*", "fr-*"
, "pt-*", "es-*", "it-*" , "pt-*", "es-*", "it-*"
] ]
} }
) )
) )

12
desktop_env/evaluators/metrics/pdf.py
View File

@@ -1,13 +1,11 @@
from pypdf import PdfReader
import operator import operator
from typing import Dict
from typing import Any from typing import Any
from typing import Dict
from pypdf import PdfReader
def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float: def check_pdf_pages(pdf_file: str, rules: Dict[str, Any]) -> float:
reader = PdfReader(pdf_file) reader = PdfReader(pdf_file)
nb_pages: int = len(reader.pages) nb_pages: int = len(reader.pages)
return float( getattr(operator, rules["relation"])( nb_pages return float(getattr(operator, rules["relation"])(nb_pages, rules["ref_value"]))
, rules["ref_value"]
)
)

137
desktop_env/evaluators/metrics/table.py
View File

@@ -1,18 +1,19 @@
import pandas as pd import logging
import operator
from numbers import Number
from typing import Any, Union
from typing import Dict, List
import openpyxl import openpyxl
import pandas as pd
from openpyxl import Workbook from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet from openpyxl.worksheet.worksheet import Worksheet
from .utils import load_charts, load_sparklines from .utils import load_charts, load_sparklines
import operator
from typing import Dict, List
from typing import Any, Union
from numbers import Number
import logging
logger = logging.getLogger("desktopenv.metric.table") logger = logging.getLogger("desktopenv.metric.table")
def compare_table(actual: str, expected: str, **options) -> float: def compare_table(actual: str, expected: str, **options) -> float:
""" """
Args: Args:
@@ -44,28 +45,28 @@ def compare_table(actual: str, expected: str, **options) -> float:
workbook1: Workbook = openpyxl.load_workbook(actual) workbook1: Workbook = openpyxl.load_workbook(actual)
workbook2: Workbook = openpyxl.load_workbook(expected) workbook2: Workbook = openpyxl.load_workbook(expected)
if ftr=="sparkline": if ftr == "sparkline":
sp1 = load_sparklines(actual) sp1 = load_sparklines(actual)
sp2 = load_sparklines(expected) sp2 = load_sparklines(expected)
new_metric: bool = sp1 == sp2 new_metric: bool = sp1 == sp2
logger.debug("Sparkline Metric: {:}".format(new_metric)) logger.debug("Sparkline Metric: {:}".format(new_metric))
elif ftr=="chart": elif ftr == "chart":
charts1 = load_charts(workbook1, **options) charts1 = load_charts(workbook1, **options)
charts2 = load_charts(workbook2, **options) charts2 = load_charts(workbook2, **options)
new_metric: bool = charts1 == charts2 new_metric: bool = charts1 == charts2
logger.debug("Chart Metric: {:}".format(new_metric)) logger.debug("Chart Metric: {:}".format(new_metric))
elif ftr=="number_format": elif ftr == "number_format":
number_formats1: List[str] = [ c.number_format.lower()\ number_formats1: List[str] = [c.number_format.lower() \
for col in workbook1.active.iter_cols()\ for col in workbook1.active.iter_cols() \
for c in col\ for c in col \
if c.data_type=="n" if c.data_type == "n"
] ]
number_formats2: List[str] = [ c.number_format.lower()\ number_formats2: List[str] = [c.number_format.lower() \
for col in workbook2.active.iter_cols()\ for col in workbook2.active.iter_cols() \
for c in col\ for c in col \
if c.data_type=="n" if c.data_type == "n"
] ]
new_metric: bool = number_formats1==number_formats2 new_metric: bool = number_formats1 == number_formats2
logger.debug("Number Format Metric: {:}".format(new_metric)) logger.debug("Number Format Metric: {:}".format(new_metric))
else: else:
raise NotImplementedError("Unsupported xlsx feature: {:}".format(ftr)) raise NotImplementedError("Unsupported xlsx feature: {:}".format(ftr))
@@ -73,6 +74,7 @@ def compare_table(actual: str, expected: str, **options) -> float:
return float(metric) return float(metric)
def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float: def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
if result is None: if result is None:
return 0. return 0.
@@ -114,6 +116,7 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
return float(passes) return float(passes)
def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float: def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
if result is None: if result is None:
return 0. return 0.
@@ -121,16 +124,18 @@ def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
worksheet: Worksheet = openpyxl.load_workbook(filename=result).active worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
return float(worksheet.freeze_panes == rules["position"]) return float(worksheet.freeze_panes == rules["position"])
def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float: def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
if result is None: if result is None:
return 0. return 0.
worksheet = openpyxl.load_workbook(filename=result).active worksheet = openpyxl.load_workbook(filename=result).active
zoom_scale: Number = worksheet.sheet_view.zoomScale or 100. zoom_scale: Number = worksheet.sheet_view.zoomScale or 100.
return float( getattr(operator, rules["relation"])( zoom_scale return float(getattr(operator, rules["relation"])(zoom_scale
, rules["ref_value"] , rules["ref_value"]
) )
) )
if __name__ == '__main__': if __name__ == '__main__':
# path1 = "" # path1 = ""
@@ -168,51 +173,51 @@ if __name__ == '__main__':
# ] # ]
# print(check_sheet_list(path1, rule)) # print(check_sheet_list(path1, rule))
#path1 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx" # path1 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx" # path2 = "../../任务数据/LibreOffice Calc/Create_column_charts_using_statistics_gold2.xlsx"
#print(compare_table(path1, path2, features=["chart"], chart_props=["type", "direction"])) # print(compare_table(path1, path2, features=["chart"], chart_props=["type", "direction"]))
#path1 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold.xlsx" # path1 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold3.xlsx" # path2 = "../../任务数据/LibreOffice Calc/Represent_in_millions_billions_gold3.xlsx"
#path1 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot.xlsx" # path1 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot_gold.xlsx" # path2 = "../../任务数据/LibreOffice Calc/Set_Decimal_Separator_Dot_gold.xlsx"
#workbook1: Workbook = openpyxl.load_workbook(filename=path1) # workbook1: Workbook = openpyxl.load_workbook(filename=path1)
#worksheet1: Worksheet = workbook1.active # worksheet1: Worksheet = workbook1.active
#import itertools # import itertools
#for col, r in itertools.product( ['A', 'B'] # for col, r in itertools.product( ['A', 'B']
#, range(1, 20) # , range(1, 20)
#): # ):
#position: str = "{:}{:d}".format(col, r) # position: str = "{:}{:d}".format(col, r)
#print(worksheet1[position]) # print(worksheet1[position])
#print(worksheet1[position].value) # print(worksheet1[position].value)
#print(worksheet1[position].number_format) # print(worksheet1[position].number_format)
#workbook2: Workbook = openpyxl.load_workbook(filename=path2) # workbook2: Workbook = openpyxl.load_workbook(filename=path2)
#worksheet2: Worksheet = workbook2.active # worksheet2: Worksheet = workbook2.active
#for col, r in itertools.product( ['A', 'B'] # for col, r in itertools.product( ['A', 'B']
#, range(1, 20) # , range(1, 20)
#): # ):
#position: str = "{:}{:d}".format(col, r) # position: str = "{:}{:d}".format(col, r)
#print(worksheet2[position]) # print(worksheet2[position])
#print(worksheet2[position].value) # print(worksheet2[position].value)
#print(worksheet2[position].number_format) # print(worksheet2[position].number_format)
#print(compare_table(path1, path2, features=["number_format"])) # print(compare_table(path1, path2, features=["number_format"]))
#path1 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells_gold.xlsx" # path1 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells_gold.xlsx"
#path2 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells.xlsx" # path2 = "../../任务数据/LibreOffice Calc/Zoom_Out_Oversized_Cells.xlsx"
#workbook1: Workbook = openpyxl.load_workbook(filename=path1) # workbook1: Workbook = openpyxl.load_workbook(filename=path1)
#worksheet1: Worksheet = workbook1.active # worksheet1: Worksheet = workbook1.active
#print(worksheet1.sheet_view.zoomScale) # print(worksheet1.sheet_view.zoomScale)
#print(type(worksheet1.sheet_view.zoomScale)) # print(type(worksheet1.sheet_view.zoomScale))
# #
#import os # import os
#import os.path # import os.path
#for wb in filter( lambda f: f.endswith(".xlsx") # for wb in filter( lambda f: f.endswith(".xlsx")
#, os.listdir("../../任务数据/LibreOffice Calc/") # , os.listdir("../../任务数据/LibreOffice Calc/")
#): # ):
#path = os.path.join("../../任务数据/LibreOffice Calc/", wb) # path = os.path.join("../../任务数据/LibreOffice Calc/", wb)
#print(wb, openpyxl.load_workbook(filename=path).active.sheet_view.zoomScale) # print(wb, openpyxl.load_workbook(filename=path).active.sheet_view.zoomScale)
#print(check_zoom(path1, {"relation": "lt", "ref_value": 100})) # print(check_zoom(path1, {"relation": "lt", "ref_value": 100}))
#print(check_zoom(path2, {"relation": "lt", "ref_value": 100})) # print(check_zoom(path2, {"relation": "lt", "ref_value": 100}))
path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx" path1 = "../../任务数据/LibreOffice Calc/Padding_Decimals_In_Formular_gold.xlsx"
data_frame: pd.DataFrame = pd.read_excel(path1) data_frame: pd.DataFrame = pd.read_excel(path1)

27
desktop_env/evaluators/metrics/vscode.py
View File

@@ -1,3 +1,6 @@
from typing import Dict
def compare_text_file(actual: str, expected: str, **options) -> float: def compare_text_file(actual: str, expected: str, **options) -> float:
""" """
Args: Args:
@@ -7,7 +10,7 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
Return: Return:
float: the score float: the score
""" """
with open(actual) as f1: with open(actual) as f1:
actual_text = f1.read() actual_text = f1.read()
with open(expected) as f2: with open(expected) as f2:
@@ -17,24 +20,17 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
return 1.0 return 1.0
return 0.0 return 0.0
def compare_config(actual: str, expected: str, **options) -> float:
"""
Args:
actual (str): path to result text file
expected (str): gold string
Return: def compare_config(actual: str, rules: Dict, **options) -> float:
float: the score
"""
with open(actual) as f1: with open(actual) as f1:
actual_text = f1.read() actual_text = f1.read()
if actual_text == expected: if actual_text == rules['expect']:
return 1.0 return 1.0
return 0.0 return 0.0
def compare_answer(actual: str, expected: str, **options) -> float:
def compare_answer(actual: str, rules: Dict, **options) -> float:
""" """
Args: Args:
actual (str): result string actual (str): result string
@@ -44,11 +40,8 @@ def compare_answer(actual: str, expected: str, **options) -> float:
float: the score float: the score
""" """
if actual == expected: if actual == rules['expect']:
return 1.0 return 1.0
# TODO: can use text embedding to get non-zero return # TODO: can use text embedding to get non-zero return
return 0.0 return 0.0
if __name__ == '__main__':
print(compare_text_file("README.md", "README.md"))

4
evaluation_examples/examples/vs_code/0ed39f63-6049-43d4-ba4d-5fa2fe04a951.json
View File

@@ -16,9 +16,9 @@
} }
}, },
{ {
"type": "open", "type": "launch",
"parameters": { "parameters": {
"path": "Desktop/vscode_replace_text.txt" "command": ["code", "Desktop/vscode_replace_text.txt"]
} }
} }
], ],

12
evaluation_examples/examples/vs_code/53ad5833-3455-407b-bbc6-45b4c79ab8fb.json
View File

@@ -4,6 +4,12 @@
"instruction": "Could you help me open the project at /home/user/project?", "instruction": "Could you help me open the project at /home/user/project?",
"source": "https://www.youtube.com/watch?v=VqCgcpAypFQ", "source": "https://www.youtube.com/watch?v=VqCgcpAypFQ",
"config": [ "config": [
{
"type": "launch",
"parameters": {
"command": ["code"]
}
},
{ {
"type": "command", "type": "command",
"parameters": { "parameters": {
@@ -18,8 +24,10 @@
"evaluator": { "evaluator": {
"func": "compare_config", "func": "compare_config",
"expected": { "expected": {
"type": "string", "type": "rule",
"string": "project" "rules": {
"expect": "project"
}
}, },
"result": { "result": {
"type": "vscode_config", "type": "vscode_config",

6
evaluation_examples/examples/vs_code/59ed65c7-e9a6-43db-833f-76d6730c0004.json
View File

@@ -29,8 +29,10 @@
"evaluator": { "evaluator": {
"func": "compare_config", "func": "compare_config",
"expected": { "expected": {
"type": "string", "type": "rule",
"string": "100" "rules": {
"expect": "100"
}
}, },
"result": { "result": {
"type": "vscode_config", "type": "vscode_config",