diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 581f1fc..2380b35 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -178,18 +178,43 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float: return float(all(expect_metrics) and unexpect_metric) -_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } +_accessibility_ns_map = { + "ubuntu": { + "st": "https://accessibility.ubuntu.example.org/ns/state", + "attr": "https://accessibility.ubuntu.example.org/ns/attributes", + "cp": "https://accessibility.ubuntu.example.org/ns/component", + "doc": "https://accessibility.ubuntu.example.org/ns/document", + "docattr": "https://accessibility.ubuntu.example.org/ns/document/attributes", + "txt": "https://accessibility.ubuntu.example.org/ns/text", + "val": "https://accessibility.ubuntu.example.org/ns/value", + "act": "https://accessibility.ubuntu.example.org/ns/action", + }, + "windows": { + "st": "https://accessibility.windows.example.org/ns/state", + "attr": "https://accessibility.windows.example.org/ns/attributes", + "cp": "https://accessibility.windows.example.org/ns/component", + "doc": "https://accessibility.windows.example.org/ns/document", + "docattr": "https://accessibility.windows.example.org/ns/document/attributes", + "txt": "https://accessibility.windows.example.org/ns/text", + "val": "https://accessibility.windows.example.org/ns/value", + "act": "https://accessibility.windows.example.org/ns/action", + "class": "https://accessibility.windows.example.org/ns/class" + }, + "macos": { + "st": "https://accessibility.macos.example.org/ns/state", + "attr": "https://accessibility.macos.example.org/ns/attributes", + "cp": "https://accessibility.macos.example.org/ns/component", + "doc": "https://accessibility.macos.example.org/ns/document", + "txt": "https://accessibility.macos.example.org/ns/text", + "val": "https://accessibility.macos.example.org/ns/value", + "act": "https://accessibility.macos.example.org/ns/action", + "role": "https://accessibility.macos.example.org/ns/role", + } + +} -def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: +def check_accessibility_tree(result: str, rules: List[Dict[str, Any]], osname: str = "ubuntu") -> float: """ Args: result (str): XML of GNOME Accessibility Tree @@ -205,18 +230,21 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: "exact": bool specifying whether exact match or fuzzy match should be performed. defaults to True. } + osname (str): "ubuntu" | "windows" | "macos". "ubuntu" by default. Returns: float """ + a11y_ns_map = _accessibility_ns_map[osname] + at: _Element = lxml.etree.fromstring(result) total_match_score = 1. for r in rules: if "xpath" in r: - elements: List[_Element] = at.xpath(r["xpath"], namespaces=_accessibility_ns_map) + elements: List[_Element] = at.xpath(r["xpath"], namespaces=a11y_ns_map) elif "selectors" in r: - selector = CSSSelector(", ".join(r["selectors"]), namespaces=_accessibility_ns_map) + selector = CSSSelector(", ".join(r["selectors"]), namespaces=a11y_ns_map) elements: List[_Element] = selector(at) else: raise ValueError("At least one of xpath and selectors is required") @@ -403,7 +431,7 @@ def compare_time_in_speedtest_results(speedtest_result_path, time_diff): return 0 # open the speedtest results file(csv) - date_col = None + #date_col = None try: with open(speedtest_result_path, 'r') as f: for i, line in enumerate(f): diff --git a/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json index dfad72e..3e8ed28 100644 --- a/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json +++ b/evaluation_examples/examples/multi_apps/5990457f-2adb-467b-a4af-5c857c92d762.json @@ -9,6 +9,7 @@ "parameters": { "command": [ "google-chrome", + "--proxy-server=http://127.0.0.1:18888", "--remote-debugging-port=1337" ] } diff --git a/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json b/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json index 7cb2ffe..c398268 100644 --- a/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json +++ b/evaluation_examples/examples/vlc/8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f.json @@ -1,7 +1,7 @@ { "id": "8d9fd4e2-6fdb-46b0-b9b9-02f06495c62f", "snapshot": "base_setup", - "instruction": "Can you make the video fill up the whole screen? It's a bit too small to see right now, and I'd like to see it better.", + "instruction": "Can you enable fullscreen mode in VLC so that the video fill up the whole screen?", "source": "https://www.youtube.com/watch?v=XHprwDJ0-fU&t=436s", "config": [ { diff --git a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json index 81170ee..96641a3 100644 --- a/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json +++ b/evaluation_examples/examples/vlc/8f080098-ddb1-424c-b438-4e96e5e4786e.json @@ -1,7 +1,7 @@ { "id": "8f080098-ddb1-424c-b438-4e96e5e4786e", "snapshot": "base_setup", - "instruction": "Could you download the song from this music video and save it as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!", + "instruction": "Could you convert the song from this music video as an MP3 file? I'd like to have it on my device to play whenever I want. Please save the file just on the desktop and title the file \"Baby Justin Bieber.mp3.\" I really appreciate your help!", "source": "https://medium.com/@jetscribe_ai/how-to-extract-mp3-audio-from-videos-using-vlc-media-player-beeef644ebfb", "config": [ {