lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:xlang-ai/OSWorld

Browse Source
This commit is contained in:
yuanmengqi
2025-07-06 07:52:42 +00:00
parent 9be6fcd688 30a346eb68
commit a1891f7d88
4 changed files with 44 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
desktop_env/evaluators/metrics/general.py
View File

@@ -178,18 +178,43 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float:
return float(all(expect_metrics) and unexpect_metric)
_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org"
, "attr": "uri:deskat:attributes.at-spi.gnome.org"
, "cp": "uri:deskat:component.at-spi.gnome.org"
, "doc": "uri:deskat:document.at-spi.gnome.org"
, "docattr": "uri:deskat:attributes.document.at-spi.gnome.org"
, "txt": "uri:deskat:text.at-spi.gnome.org"
, "val": "uri:deskat:value.at-spi.gnome.org"
, "act": "uri:deskat:action.at-spi.gnome.org"
}
_accessibility_ns_map = {
"ubuntu": {
"st": "https://accessibility.ubuntu.example.org/ns/state",
"attr": "https://accessibility.ubuntu.example.org/ns/attributes",
"cp": "https://accessibility.ubuntu.example.org/ns/component",
"doc": "https://accessibility.ubuntu.example.org/ns/document",
"docattr": "https://accessibility.ubuntu.example.org/ns/document/attributes",
"txt": "https://accessibility.ubuntu.example.org/ns/text",
"val": "https://accessibility.ubuntu.example.org/ns/value",
"act": "https://accessibility.ubuntu.example.org/ns/action",
},
"windows": {
"st": "https://accessibility.windows.example.org/ns/state",
"attr": "https://accessibility.windows.example.org/ns/attributes",
"cp": "https://accessibility.windows.example.org/ns/component",
"doc": "https://accessibility.windows.example.org/ns/document",
"docattr": "https://accessibility.windows.example.org/ns/document/attributes",
"txt": "https://accessibility.windows.example.org/ns/text",
"val": "https://accessibility.windows.example.org/ns/value",
"act": "https://accessibility.windows.example.org/ns/action",
"class": "https://accessibility.windows.example.org/ns/class"
},
"macos": {
"st": "https://accessibility.macos.example.org/ns/state",
"attr": "https://accessibility.macos.example.org/ns/attributes",
"cp": "https://accessibility.macos.example.org/ns/component",
"doc": "https://accessibility.macos.example.org/ns/document",
"txt": "https://accessibility.macos.example.org/ns/text",
"val": "https://accessibility.macos.example.org/ns/value",
"act": "https://accessibility.macos.example.org/ns/action",
"role": "https://accessibility.macos.example.org/ns/role",
}
}
def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
def check_accessibility_tree(result: str, rules: List[Dict[str, Any]], osname: str = "ubuntu") -> float:
"""
Args:
result (str): XML of GNOME Accessibility Tree
@@ -205,18 +230,21 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float:
"exact": bool specifying whether exact match or fuzzy match should
be performed. defaults to True.
}
osname (str): "ubuntu" | "windows" | "macos". "ubuntu" by default.
Returns:
float
"""
a11y_ns_map = _accessibility_ns_map[osname]
at: _Element = lxml.etree.fromstring(result)
total_match_score = 1.
for r in rules:
if "xpath" in r:
elements: List[_Element] = at.xpath(r["xpath"], namespaces=_accessibility_ns_map)
elements: List[_Element] = at.xpath(r["xpath"], namespaces=a11y_ns_map)
elif "selectors" in r:
selector = CSSSelector(", ".join(r["selectors"]), namespaces=_accessibility_ns_map)
selector = CSSSelector(", ".join(r["selectors"]), namespaces=a11y_ns_map)
elements: List[_Element] = selector(at)
else:
raise ValueError("At least one of xpath and selectors is required")
@@ -403,7 +431,7 @@ def compare_time_in_speedtest_results(speedtest_result_path, time_diff):
return 0
# open the speedtest results file(csv)
date_col = None
#date_col = None
try:
with open(speedtest_result_path, 'r') as f:
for i, line in enumerate(f):

1
evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json
View File

@@ -9,6 +9,7 @@
"parameters": {
"command": [
"google-chrome",
"--proxy-server=http://127.0.0.1:18888",
"--remote-debugging-port=1337"
]
}

2
evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json
View File

@@ -1,7 +1,7 @@
{
"id": "8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f",
"snapshot": "base_setup",
"instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.",
"instruction": "Can you enable fullscreen mode in VLC so that the video fill up the whole screen?",
"source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s",
"config": [
{

2
evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json
View File

@@ -1,7 +1,7 @@
{
"id": "8f080098-ddb1-424c-b438-4e96e5e4786e",
"snapshot": "base_setup",
"instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"instruction": "Could you convert the song from this music video as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!",
"source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb",
"config": [
{