lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main'

Browse Source 
# Conflicts:
#	desktop_env/evaluators/getters/__init__.py
#	desktop_env/evaluators/metrics/__init__.py
#	requirements.txt
This commit is contained in:
Timothyxxx
2024-01-10 23:20:49 +08:00
parent 49ece15ac3 031b0d07ab
commit 287876affc
32 changed files with 626 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
desktop_env/controllers/python.py
View File

@@ -1,7 +1,7 @@
import json
import logging
import random
from typing import Any, Dict
from typing import Any, Dict, Optional
import requests
@@ -26,6 +26,15 @@ class PythonController:
logger.error("Failed to get screenshot. Status code: %d", response.status_code)
return None
def get_accessibility_tree(self) -> Optional[str]:
response: requests.Response = requests.get(self.http_server + "/accessibility")
if response.status_code == 200:
return response.json()["AT"]
else:
logger.error("Failed to get accessibility tree. Status code: %d", response.status_code)
return None
def get_file(self, file_path: str):
"""
Gets a file from the server.
@@ -73,7 +82,7 @@ class PythonController:
if action_type == "MOVE_TO":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.moveTo()")
self.execute_python_command("pyautogui.moveTo()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -83,7 +92,7 @@ class PythonController:
elif action_type == "CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.click()")
self.execute_python_command("pyautogui.click()")
elif "button" in parameters and "x" in parameters and "y" in parameters:
button = parameters["button"]
x = parameters["x"]
@@ -114,7 +123,7 @@ class PythonController:
elif action_type == "MOUSE_DOWN":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.mouseDown()")
self.execute_python_command("pyautogui.mouseDown()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseDown(button='{button}')")
@@ -123,7 +132,7 @@ class PythonController:
elif action_type == "MOUSE_UP":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.mouseUp()")
self.execute_python_command("pyautogui.mouseUp()")
elif "button" in parameters:
button = parameters["button"]
self.execute_python_command(f"pyautogui.mouseUp(button='{button}')")
@@ -132,7 +141,7 @@ class PythonController:
elif action_type == "RIGHT_CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.rightClick()")
self.execute_python_command("pyautogui.rightClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -142,7 +151,7 @@ class PythonController:
elif action_type == "DOUBLE_CLICK":
if parameters == {} or None:
self.execute_python_command(f"pyautogui.doubleClick()")
self.execute_python_command("pyautogui.doubleClick()")
elif "x" in parameters and "y" in parameters:
x = parameters["x"]
y = parameters["y"]
@@ -208,7 +217,7 @@ class PythonController:
raise Exception(f"Unknown parameters: {parameters}")
keys = parameters["keys"]
if not isinstance(keys, list):
raise Exception(f"Keys must be a list of keys")
raise Exception("Keys must be a list of keys")
for key in keys:
if key.lower() not in KEYBOARD_KEYS:
raise Exception(f"Key must be one of {KEYBOARD_KEYS}")

3
desktop_env/controllers/setup.py
View File

@@ -246,6 +246,9 @@ class SetupController:
logger.error("An error occurred while trying to send the request: %s", e)
traceback.print_exc()
def _command_setup(self, command: List[str], stdout: str = "", stderr: str = ""):
self._execute_setup(command, stdout, stderr)
def _act_setup(self, action_seq: List[Union[Dict[str, Any], str]]):
# TODO
raise NotImplementedError()

2
desktop_env/envs/desktop_env.py
View File

@@ -223,7 +223,7 @@ class DesktopEnv(gym.Env):
Evaluate whether the task is successfully completed.
"""
self.setup_controller.setup(self.evaluator["postconfig"]) if "postconfig" in self.evaluator else None
self.setup_controller.setup(self.evaluator.get("postconfig", []))
try:
result_state = self.result_getter(self, self.evaluator["result"])

1
desktop_env/evaluators/getters/__init__.py
View File

@@ -1,4 +1,5 @@
from .file import get_cloud_file, get_vm_file, get_cache_file
from .misc import get_rule
from .info import get_vm_screen_size, get_vm_window_size, get_vm_wallpaper
from .misc import get_rule, get_accessibility_tree
from .vlc import get_vlc_playing_info, get_vlc_config

8
desktop_env/evaluators/getters/file.py
View File

@@ -1,5 +1,6 @@
import os
from typing import Dict
from typing import Optional
import requests
@@ -27,7 +28,7 @@ def get_cloud_file(env, config: Dict[str, str]) -> str:
return _path
def get_vm_file(env, config: Dict[str, str]) -> str:
def get_vm_file(env, config: Dict[str, str]) -> Optional[str]:
"""
Config:
path (str): absolute path on the VM to fetch
@@ -37,10 +38,9 @@ def get_vm_file(env, config: Dict[str, str]) -> str:
_path = os.path.join(env.cache_dir, config["dest"])
file = env.controller.get_file(config["path"])
if file is None:
raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
return None
#raise FileNotFoundError("File not found on VM: {:}".format(config["path"]))
with open(_path, "wb") as f:
f.write(file)

23
desktop_env/evaluators/getters/general.py Normal file
View File

@@ -0,0 +1,23 @@
from typing import Dict
import os
import requests
def get_string(env, config: Dict[str, str]) -> str:
"""
Config:
string (str)
"""
return config["string"]
def get_command_line(env, config: Dict[str, str]) -> str:
"""
Config:
string (str)
"""
f = os.popen(config["command"])
return f.read()

6
desktop_env/evaluators/getters/misc.py
View File

@@ -1,5 +1,6 @@
import logging
from typing import TypeVar
#from typing import Dict, List
logger = logging.getLogger("desktopenv.getters.misc")
@@ -11,3 +12,8 @@ def get_rule(env, config: R) -> R:
Returns the rule as-is.
"""
return config["rules"]
def get_accessibility_tree(env, *args) -> str:
accessibility_tree: str = env.controller.get_accessibility_tree()
logger.debug("AT@eval: %s", accessibility_tree)
return accessibility_tree

3
desktop_env/evaluators/metrics/__init__.py
View File

@@ -6,4 +6,5 @@ from .docs import is_first_line_centered, check_file_exists, compare_contains_im
from .pdf import check_pdf_pages
from .libreoffice import check_libre_locale
from .vlc import is_vlc_playing, is_vlc_recordings_folder, is_vlc_fullscreen, compare_images, compare_audios, compare_videos
from .general import check_csv
from .general import check_csv, check_accessibility_tree, check_list

101
desktop_env/evaluators/metrics/general.py
View File

@@ -1,5 +1,17 @@
import csv
from typing import Dict, List
import lxml.etree
from lxml.etree import _Element
from lxml.cssselect import CSSSelector
from typing import Dict, List, Pattern
from typing import Callable, Any
from numbers import Number
import operator
from rapidfuzz import fuzz
import functools
import re
def _match_record(pattern: Dict[str, str], item: Dict[str, str]) -> float:
return all(k in item and item[k]==val for k, val in pattern.items())
@@ -22,9 +34,92 @@ def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
unexpect_metric = True
with open(result) as f:
reader = csv.DictReader(f)
for rcd in reader:
for i, r in enumerate(rules.get("expect", [])):
expect_metrics[i] = expect_metrics[i] or _match_record(r, rcd)
unexpect_metric = unexpect_metric and all(_match_record(r, rcd) for r in rules.get("unexpect", []))
unexpect_metric = unexpect_metric and not any(_match_record(r, rcd) for r in rules.get("unexpect", []))
return float(all(expect_metrics) and unexpect_metric)
def check_list(result: str, rules: Dict[str, List[str]]) -> float:
"""
Args:
result (str): path to list file
rules (Dict[str, List[str]]): dict like
{
"expect": list of str as regexes
"unexpect": list of str as regexes
}
Returns:
float
"""
expect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("expect", [])]
unexpect_patterns: List[Pattern[str]] = [re.compile(ptt) for ptt in rules.get("unexpect", [])]
expect_metrics = [False] * len(expect_patterns)
unexpect_metric = True
with open(result) as f:
for l in f:
for i, r in enumerate(expect_patterns):
expect_metrics[i] = expect_metrics[i] or (r.search(l) is not None)
unexpect_metric = unexpect_metric and all(r.search(l) is None for r in unexpect_patterns)
return float(all(expect_metrics) and unexpect_metric)
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
def check_accessibility_tree(result: str, rules: Dict[str, Any]) -> float:
"""
Args:
result (str): XML of GNOME Accessibility Tree
rules (Dict[str, Any]): dict like
{
"selectors": list of str as CSS selectors, will be connected by ", "
to form a composite selector. Only one from `selectors` and
`xpath` is needed. If both are present, `xpath` takes the
priority.
"xpath": str as xpath. Only one from `selectors` and `xpath` is
needed. If both are present, `xpath` takes the priority.
"text": str as the expected text content of the selected element.
"exact": bool specifying whether exact match or fuzzy match should
be performed. defaults to True
}
Returns:
float
"""
at: _Element = lxml.etree.fromstring(result)
if "xpath" in rules:
elements: List[_Element] = at.xpath(rules["xpath"], namespaces=_accessibility_ns_map)
elif "selectors" in rules:
selector = CSSSelector(", ".join(rules["selectors"]), namespaces=_accessibility_ns_map)
elements: List[_Element] = selector(at)
else:
raise ValueError("At least one of xpath and selectors is required")
if len(elements)==0:
return 0.
if "text" in rules:
match_func: Callable[[str], Number] = functools.partial( operator.eq if rules["exact"] else fuzz.ratio
, rules["text"]
)
match_score: Number = 0
for elm in elements:
match_score = max(match_score, match_func(elm.text or None))
else:
match_score = 1.
return float(match_score)
#def check_existence(result: str, *args) -> float:
#return 1. - (result is None)

12
desktop_env/evaluators/metrics/table.py
View File

@@ -31,6 +31,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
float: the score
"""
if actual is None:
return 0.
df1 = pd.read_excel(expected)
df2 = pd.read_excel(actual)
metric: bool = df1.equals(df2)
@@ -71,6 +74,9 @@ def compare_table(actual: str, expected: str, **options) -> float:
return float(metric)
def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
if result is None:
return 0.
# workbook: Workbook = openpyxl.load_workbook(filename=result)
workbook = pd.ExcelFile(result)
worksheet_names: List[str] = workbook.sheet_names
@@ -109,10 +115,16 @@ def check_sheet_list(result: str, rules: List[Dict[str, Any]]) -> float:
return float(passes)
def check_xlsx_freeze(result: str, rules: Dict[str, str]) -> float:
if result is None:
return 0.
worksheet: Worksheet = openpyxl.load_workbook(filename=result).active
return float(worksheet.freeze_panes == rules["position"])
def check_xlsx_zoom(result: str, rules: Dict[str, Union[str, Number]]) -> float:
if result is None:
return 0.
worksheet = openpyxl.load_workbook(filename=result).active
zoom_scale: Number = worksheet.sheet_view.zoomScale or 100.
return float( getattr(operator, rules["relation"])( zoom_scale

53
desktop_env/evaluators/metrics/thunderbird.py Normal file
View File

@@ -0,0 +1,53 @@
#from playwright.sync_api import sync_playwright, Browser
#from marionette_driver.marionette import Marionette
#import marionette
#import pyatspi
import lxml.etree
from lxml.cssselect import CSSSelector
from lxml.etree import _Element
from typing import List
if __name__ == "__main__":
#with sync_playwright() as plwr:
#while True:
##try:
#thunderbird: Browser = plwr.firefox.connect("http://127.0.0.1:6000", timeout=60)
#break
##except:
##pass
#for ctx in thunderbird.contexts:
#for p in ctx.pages:
#print(p.url)
#thunderbird = Marionette()
#print(thunderbird.start_session())
#print(thunderbird.chrome_window_handles)
#print(thunderbird.window_handles)
#print(thunderbird.current_chrome_window_handle)
#thunderbird.set_context(Marionette.CONTEXT_CONTENT)
#print(thunderbird.current_window_handle)
#thunderbird.switch_to_window(thunderbird.chrome_window_handles[0])
#thunderbird.switch_to_default_content()
#thunderbird.switch_to_frame()
#print(thunderbird.get_url())
#print(thunderbird.get_window_type())
#thunderbird.fullscreen()
#print(thunderbird.close())
#registry = pyatspi.Registry.get_default()
#registry
#xml = "../../任务数据/Thunderbird/vertical-card-view.xml"
xml = "../../任务数据/Thunderbird/vertical-table-view.xml"
at: _Element = lxml.etree.parse(xml)
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] page-tab-list')(at) # page tab tags
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section:nth-child(3)')(at) # email tag page
#elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>section[attr|class~="tree-table-header"]>table-row>column-header[name=Subject]>push-button', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
elements: List[_Element] = CSSSelector('application[name=Thunderbird] panel>scroll-pane>internal-frame>panel[name$="anonym-x2024@outlook.com"]>section[attr|id=threadPane]>section[attr|id="threadTree"]>table[attr|class="tree-table"]>tree>tree-item>section[name="Subject"]>section>section', namespaces={"attr": "uri:deskat:attributes.at-spi.gnome.org"})(at) # table view, column header
print(len(elements))
for elm in elements:
print(lxml.etree.tostring(elm, encoding="unicode", pretty_print=True))

8
desktop_env/evaluators/metrics/vscode.py
View File

@@ -20,5 +20,13 @@ def compare_text_file(actual: str, expected: str, **options) -> float:
return 1.0
return 0.0
def compare_answer(actual: str, expected: str, **options) -> float:
if actual == expected:
return 1.0
# TODO: can use text embedding to get non-zero return
return 0.0
if __name__ == '__main__':
print(compare_text_file("README.md", "README.md"))

73
desktop_env/server/README.md Normal file
View File

@@ -0,0 +1,73 @@
<!-- vimc: call SyntaxRange#Include('```xml', '```', 'xml', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```css', '```', 'css', 'NonText'): -->
<!-- vimc: call SyntaxRange#Include('```sh', '```', 'sh', 'NonText'): -->
### About the Converted Accessibility Tree
For several applications like Firefox or Thunderbird, you should first enable
```sh
gsettings set org.gnome.desktop.interface toolkit-accessibility true
```
to see their accessibility tree.
#### Example of AT
An example of a node:
```xml
<section xmlns:attr="uri:deskat:attributes.at-spi.gnome.org" attr:class="subject" st:enabled="true" cp:screencoord="(1525, 169)", cp:windowcoord="(342, 162)", cp:size="(327, 21)">
歡迎使用新的 Outlook.com 帳戶
</section>
```
An example of a tree:
```xml
<desktop-frame ...>
<application name="Thunderbird" ...>
... <!-- nodes of windows -->
</application>
...
</desktop-frame>
```
#### Useful attributes
1. `name` - shows the name of application, title of window, or name of some
component
2. `attr:class` - somewhat the same role as `class` in HTML
3. `attr:id` - somewhat the same role as `id` in HTML
4. `cp:screencoord` - absolute coordinator on the screen
5. `cp:windowcoord` - relative coordinator in the window
6. `cp:size` - the size
Also several states like `st:enabled` and `st:visible` can be indicated. A full
state list is available at
<https://gitlab.gnome.org/GNOME/pyatspi2/-/blob/master/pyatspi/state.py?ref_type=heads>.
#### How to use it in evaluation
See example `thunderbird/12086550-11c0-466b-b367-1d9e75b3910e.json` and
function `check_accessibility_tree` in `metrics/general.py`. You can use CSS
selector or XPath to reference a target nodes. You can also check its text
contents.
An example of a CSS selector:
```css
application[name=Thunderbird] page-tab-list[attr|id=\"tabmail-tabs\"]>page-tab[name=\"About Profiles\"]
```
This selector will select the page tab of profile manager in Thunderbird (if open).
For usage of CSS selector: <https://www.w3.org/TR/selectors-3/>. For usage of XPath: <https://www.w3.org/TR/xpath-31/>.
#### Manual check
You can use accerciser to check the accessibility tree on GNOME VM.
```sh
sudo apt install accerciser
```

154
desktop_env/server/main.py
View File

@@ -3,18 +3,29 @@ import os
import platform
import subprocess
from pathlib import Path
from typing import List
import lxml.etree
from lxml.etree import _Element
import pyatspi
from pyatspi import Accessible, StateType
from pyatspi import Component, Document
from pyatspi import Text as ATText
from pyatspi import Value as ATValue
from pyatspi import Action as ATAction
from typing import List, Dict
from typing import Any
import Xlib
import pyautogui
import requests
from PIL import Image
from Xlib import display, X
from pyxcursor import Xcursor
import requests
from flask import Flask, request, jsonify, send_file, abort
from werkzeug.utils import secure_filename
from pyxcursor import Xcursor
app = Flask(__name__)
pyautogui.PAUSE = 0
@@ -102,6 +113,141 @@ def capture_screen_with_cursor():
return send_file(file_path, mimetype='image/png')
_accessibility_ns_map = { "st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
def _create_node(node: Accessible) -> _Element:
attribute_dict: Dict[str, Any] = {"name": node.name}
# States {{{ #
states: List[StateType] = node.getState().get_states()
for st in states:
state_name: str = StateType._enum_lookup[st]
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["st"]
, state_name.split("_", maxsplit=1)[1].lower()
)
] = "true"
# }}} States #
# Attributes {{{ #
attributes: List[str] = node.getAttributes()
for attrbt in attributes:
attribute_name: str
attribute_value: str
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["attr"]
, attribute_name
)
] = attribute_value
# }}} Attributes #
# Component {{{ #
try:
component: Component = node.queryComponent()
except NotImplementedError:
pass
else:
attribute_dict["{{{:}}}screencoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_SCREEN))
attribute_dict["{{{:}}}windowcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_WINDOW))
attribute_dict["{{{:}}}parentcoord".format(_accessibility_ns_map["cp"])] = str(component.getPosition(pyatspi.XY_PARENT))
attribute_dict["{{{:}}}size".format(_accessibility_ns_map["cp"])] = str(component.getSize())
# }}} Component #
# Document {{{ #
try:
document: Document = node.queryDocument()
except NotImplementedError:
pass
else:
attribute_dict["{{{:}}}locale".format(_accessibility_ns_map["doc"])] = document.getLocale()
attribute_dict["{{{:}}}pagecount".format(_accessibility_ns_map["doc"])] = str(document.getPageCount())
attribute_dict["{{{:}}}currentpage".format(_accessibility_ns_map["doc"])] = str(document.getCurrentPageNumber())
for attrbt in document.getAttributes():
attribute_name: str
attribute_value: str
attribute_name, attribute_value = attrbt.split(":", maxsplit=1)
attribute_dict[ "{{{:}}}{:}"\
.format( _accessibility_ns_map["docattr"]
, attribute_name
)
] = attribute_value
# }}} Document #
# Text {{{ #
try:
text_obj: ATText = node.queryText()
except NotImplementedError:
pass
else:
# only text shown on current screen is available
#attribute_dict["txt:text"] = text_obj.getText(0, text_obj.characterCount)
text: str = text_obj.getText(0, text_obj.characterCount)
# }}} Text #
# Selection {{{ #
try:
node.querySelection()
except NotImplementedError:
pass
else:
attribute_dict["selection"] = "true"
# }}} Selection #
# Value {{{ #
try:
value: ATValue = node.queryValue()
except NotImplementedError:
pass
else:
attribute_dict["{{{:}}}value".format(_accessibility_ns_map["val"])] = str(value.currentValue)
attribute_dict["{{{:}}}min".format(_accessibility_ns_map["val"])] = str(value.minimumValue)
attribute_dict["{{{:}}}max".format(_accessibility_ns_map["val"])] = str(value.maximumValue)
attribute_dict["{{{:}}}step".format(_accessibility_ns_map["val"])] = str(value.minimumIncrement)
# }}} Value #
# Action {{{ #
try:
action: ATAction = node.queryAction()
except NotImplementedError:
pass
else:
for i in range(action.nActions):
action_name: str = action.getName(i).replace(" ", "-")
attribute_dict[ "{{{:}}}{:}_desc"\
.format( _accessibility_ns_map["act"]
, action_name
)
] = action.getDescription(i)
attribute_dict[ "{{{:}}}{:}_kb"\
.format( _accessibility_ns_map["act"]
, action_name
)
] = action.getKeyBinding(i)
# }}} Action #
xml_node = lxml.etree.Element( node.getRoleName().replace(" ", "-")
, attrib=attribute_dict
, nsmap=_accessibility_ns_map
)
if "text" in locals() and len(text)>0:
xml_node.text = text
for ch in node:
xml_node.append(_create_node(ch))
return xml_node
@app.route("/accessibility", methods=["GET"])
def get_accessibility_tree():
desktop: Accessible = pyatspi.Registry.getDesktop(0)
desktop_xml: _Element = _create_node(desktop)
return jsonify({"AT": lxml.etree.tostring(desktop_xml, encoding="unicode")})
@app.route('/screen_size', methods=['POST'])
def get_screen_size():