From c4b47886d93c68fa11417e55dfe34513baee28af Mon Sep 17 00:00:00 2001 From: zdy023 Date: Wed, 2 Jul 2025 20:46:04 +0800 Subject: [PATCH 1/3] ver Jul2nd updated task requiring set up new email account --- .../thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evaluation_examples/examples/thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json b/evaluation_examples/examples/thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json index e0848eb..59f8ce0 100644 --- a/evaluation_examples/examples/thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json +++ b/evaluation_examples/examples/thunderbird/15c3b339-88f7-4a86-ab16-e71c58dcb01e.json @@ -1,7 +1,7 @@ { "id": "15c3b339-88f7-4a86-ab16-e71c58dcb01e", "snapshot": "thunderbird", - "instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'Wlv(z._6y|a,rrjfQuQhIi\\$;' (without ')", + "instruction": "Help me access my outlook account with address \"anonym-x2024@outlook.com\" and password 'password' (without ') in Thunderbird. It doesn't mather if Thunderbird remind of login or connection failure. Just finish the account setup and I will check the things like password mannually later.", "source": "https://www.wikihow.com/Access-Gmail-With-Mozilla-Thunderbird", "config": [ { @@ -85,16 +85,16 @@ { "url": "imap://outlook.office365.com", "user": "anonym-x2024@outlook.com", - "password": "Wlv(z._6y|a,rrjfQuQhIi\\$;" + "password": "password" }, { "url": "smtp://smtp.office365.com", "user": "anonym-x2024@outlook.com", - "password": "Wlv(z._6y|a,rrjfQuQhIi\\$;" + "password": "password" } ] } } }, "proxy": false -} \ No newline at end of file +} From 3cf80eaab83c795175ff8180536941dcf21b4476 Mon Sep 17 00:00:00 2001 From: zdy023 Date: Thu, 3 Jul 2025 20:55:30 +0800 Subject: [PATCH 2/3] ver Jul3rd fixed several tasks --- .../multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json | 4 ++-- .../thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json | 6 +++--- .../thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json index 8bcd85e..04f43a6 100644 --- a/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json +++ b/evaluation_examples/examples/multi_apps/415ef462-bed3-493a-ac36-ca8c6d23bf1b.json @@ -1,7 +1,7 @@ { "id": "415ef462-bed3-493a-ac36-ca8c6d23bf1b", "snapshot": "thunderbird", - "instruction": "There's an e-mail containing the AWS invoice for December saved in local \"Bills\" folder. Extract the invoice PDF to the my receipts folder. Follow the file name pattern of the old files and update a record in my tally book.", + "instruction": "There's an e-mail containing the AWS invoice for December saved in local \"Bills\" folder. Extract the invoice PDF to the my receipts folder. Follow the file name pattern of the old files and append a record at the end of my tally book.", "source": "authors", "config": [ { @@ -202,4 +202,4 @@ ] }, "proxy": false -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json b/evaluation_examples/examples/thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json index 2409586..3863c2a 100644 --- a/evaluation_examples/examples/thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json +++ b/evaluation_examples/examples/thunderbird/3f28fe4f-5d9d-4994-a456-efd78cfae1a3.json @@ -1,7 +1,7 @@ { "id": "3f28fe4f-5d9d-4994-a456-efd78cfae1a3", "snapshot": "thunderbird", - "instruction": "Set up a signature using my name and affiliation. My name is Anonym and my affiliation is XYZ Lab.", + "instruction": "Set up a plain text signature for my email account in Thunderbird. The first line is my name \"Anonym\" and the second line is my affiliation \"XYZ Lab\".", "source": "https://www.adsigner.com/user-manual/signatures/setup-email-client-thunderbird/#:~:text=is%20probably%20hidden.-,Right%20click%20on%20the%20empty%20space%20at%20the%20top%20of,signature%20from%20a%20file%20instead.", "config": [ { @@ -70,7 +70,7 @@ "expect": { "mail.identity.id1.htmlSigText": { "method": "re.S", - "ref": "Anonym.+XYZ Lab" + "ref": "Anonym\\nXYZ Lab" } } } @@ -78,4 +78,4 @@ "func": "check_thunderbird_prefs" }, "proxy": false -} \ No newline at end of file +} diff --git a/evaluation_examples/examples/thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json b/evaluation_examples/examples/thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json index 939a016..def2b5d 100644 --- a/evaluation_examples/examples/thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json +++ b/evaluation_examples/examples/thunderbird/7b1e1ff9-bb85-49be-b01d-d6424be18cd0.json @@ -1,7 +1,7 @@ { "id": "7b1e1ff9-bb85-49be-b01d-d6424be18cd0", "snapshot": "thunderbird", - "instruction": "Could you help me open up the Thunderbird profile manager utility?", + "instruction": "Could you help me open up the profile management tabpage in Thunderbird? I want the profile management tabpage inside Thunderbird app, but not the profile chooser dialog during app launch.", "source": "https://www.quora.com/How-do-I-open-a-Thunderbird-profile-manager-utility", "config": [ { @@ -58,4 +58,4 @@ "func": "check_accessibility_tree" }, "proxy": false -} \ No newline at end of file +} From 690f6ed6e79bae4cfaa35fd2d25df656f9b6915d Mon Sep 17 00:00:00 2001 From: zdy023 Date: Fri, 4 Jul 2025 23:20:51 +0800 Subject: [PATCH 3/3] ver Jul4th fixed check_accessibility_tree function, updated the namespace definitons according the values defined in server/main.py --- desktop_env/evaluators/metrics/general.py | 54 +++++++++++++++++------ 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/desktop_env/evaluators/metrics/general.py b/desktop_env/evaluators/metrics/general.py index 361c099..362b969 100644 --- a/desktop_env/evaluators/metrics/general.py +++ b/desktop_env/evaluators/metrics/general.py @@ -178,18 +178,43 @@ def check_list(result: str, rules: Dict[str, List[str]]) -> float: return float(all(expect_metrics) and unexpect_metric) -_accessibility_ns_map = {"st": "uri:deskat:state.at-spi.gnome.org" - , "attr": "uri:deskat:attributes.at-spi.gnome.org" - , "cp": "uri:deskat:component.at-spi.gnome.org" - , "doc": "uri:deskat:document.at-spi.gnome.org" - , "docattr": "uri:deskat:attributes.document.at-spi.gnome.org" - , "txt": "uri:deskat:text.at-spi.gnome.org" - , "val": "uri:deskat:value.at-spi.gnome.org" - , "act": "uri:deskat:action.at-spi.gnome.org" - } +_accessibility_ns_map = { + "ubuntu": { + "st": "https://accessibility.ubuntu.example.org/ns/state", + "attr": "https://accessibility.ubuntu.example.org/ns/attributes", + "cp": "https://accessibility.ubuntu.example.org/ns/component", + "doc": "https://accessibility.ubuntu.example.org/ns/document", + "docattr": "https://accessibility.ubuntu.example.org/ns/document/attributes", + "txt": "https://accessibility.ubuntu.example.org/ns/text", + "val": "https://accessibility.ubuntu.example.org/ns/value", + "act": "https://accessibility.ubuntu.example.org/ns/action", + }, + "windows": { + "st": "https://accessibility.windows.example.org/ns/state", + "attr": "https://accessibility.windows.example.org/ns/attributes", + "cp": "https://accessibility.windows.example.org/ns/component", + "doc": "https://accessibility.windows.example.org/ns/document", + "docattr": "https://accessibility.windows.example.org/ns/document/attributes", + "txt": "https://accessibility.windows.example.org/ns/text", + "val": "https://accessibility.windows.example.org/ns/value", + "act": "https://accessibility.windows.example.org/ns/action", + "class": "https://accessibility.windows.example.org/ns/class" + }, + "macos": { + "st": "https://accessibility.macos.example.org/ns/state", + "attr": "https://accessibility.macos.example.org/ns/attributes", + "cp": "https://accessibility.macos.example.org/ns/component", + "doc": "https://accessibility.macos.example.org/ns/document", + "txt": "https://accessibility.macos.example.org/ns/text", + "val": "https://accessibility.macos.example.org/ns/value", + "act": "https://accessibility.macos.example.org/ns/action", + "role": "https://accessibility.macos.example.org/ns/role", + } + +} -def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: +def check_accessibility_tree(result: str, rules: List[Dict[str, Any]], osname: str = "ubuntu") -> float: """ Args: result (str): XML of GNOME Accessibility Tree @@ -205,18 +230,21 @@ def check_accessibility_tree(result: str, rules: List[Dict[str, Any]]) -> float: "exact": bool specifying whether exact match or fuzzy match should be performed. defaults to True. } + osname (str): "ubuntu" | "windows" | "macos". "ubuntu" by default. Returns: float """ + a11y_ns_map = _accessibility_ns_map[osname] + at: _Element = lxml.etree.fromstring(result) total_match_score = 1. for r in rules: if "xpath" in r: - elements: List[_Element] = at.xpath(r["xpath"], namespaces=_accessibility_ns_map) + elements: List[_Element] = at.xpath(r["xpath"], namespaces=a11y_ns_map) elif "selectors" in r: - selector = CSSSelector(", ".join(r["selectors"]), namespaces=_accessibility_ns_map) + selector = CSSSelector(", ".join(r["selectors"]), namespaces=a11y_ns_map) elements: List[_Element] = selector(at) else: raise ValueError("At least one of xpath and selectors is required") @@ -361,7 +389,7 @@ def compare_time_in_speedtest_results(speedtest_result_path, time_diff): return 0 # open the speedtest results file(csv) - date_col = None + #date_col = None try: with open(speedtest_result_path, 'r') as f: for i, line in enumerate(f):