lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix selector bug and determine the path according to arch

Browse Source
This commit is contained in:
Jason Lee
2024-02-25 23:15:47 +08:00
parent 3244098664
commit ca24d2a649
21 changed files with 18 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
desktop_env/evaluators/getters/chrome.py
View File

@@ -1,6 +1,7 @@
import json
import logging
import os
import platform
import time
import sqlite3
from typing import Dict, Any, List
@@ -363,10 +364,11 @@ def get_active_url_from_accessTree(env, config):
This function is used to get the active tab url from the accessibility tree.
config:
Dict[str, str]{
'xpath':
the same as in metrics.general.accessibility_tree.
'selectors':
the same as in metrics.general.accessibility_tree.
# we no longer need to specify the xpath or selectors, since we will use defalut value
# 'xpath':
# the same as in metrics.general.accessibility_tree.
# 'selectors':
# the same as in metrics.general.accessibility_tree.
'goto_prefix':
the prefix you want to add to the beginning of the url to be opened, default is "https://",
(the url we get from accTree does not have prefix)
@@ -380,14 +382,17 @@ def get_active_url_from_accessTree(env, config):
logger.debug("AT@eval: %s", accessibility_tree)
# first, use accessibility API to get the active tab URL
at: _Element = lxml.etree.fromstring(accessibility_tree)
if "xpath" in config:
elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
elif "selectors" in config:
selector = CSSSelector(", ".join(config["selectors"]), namespaces=_accessibility_ns_map)
elements: List[_Element] = selector(at)
arch = platform.machine()
if "arm" in arch:
selector = CSSSelector(", application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map)
else:
raise ValueError("At least one of xpath and selectors is required")
selector = CSSSelector(", application[name=Google\\ Chrome] entry[name=Address\\ and\\ search\\ bar]", namespaces=_accessibility_ns_map)
elements: List[_Element] = selector(at)
# if "xpath" in config:
# elements: List[_Element] = at.xpath(config["xpath"], namespaces=_accessibility_ns_map)
# elif "selectors" in config:
# selector = CSSSelector(", ".join(config["selectors"]), namespaces=_accessibility_ns_map)
# elements: List[_Element] = selector(at)
if len(elements) == 0:
print("no elements found")
return 0.

6
evaluation_examples/examples/chrome/1704f00f-79e6-43a7-961b-cedd3724d5fd.json
View File

@@ -46,17 +46,11 @@
"func":["check_direct_json_object", "check_direct_json_object"],
"result": [{
"type": "active_tab_url_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"parse_keys": ["locationName", "dropLocationName", "filterCriteria_carCategory", "filterCriteria_sortBy"]
},
{
"type": "active_tab_html_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{

3
evaluation_examples/examples/chrome/2888b4e6-5b47-4b57-8bf5-c73827890774.json
View File

@@ -47,9 +47,6 @@
"func":"exact_match",
"result": {
"type": "url_dashPart",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": true,

6
evaluation_examples/examples/chrome/368d9ba4-203c-40c1-9fa3-da2f1430ce63.json
View File

@@ -46,9 +46,6 @@
"func":["check_direct_json_object", "is_expected_url_pattern_match"],
"result": [{
"type": "url_dashPart",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"partIndex": -2,
"needDeleteId": false,
@@ -57,9 +54,6 @@
},
{
"type": "active_tab_info",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
}],
"expected":[

9
evaluation_examples/examples/chrome/47543840-672a-467d-80df-8f7c3b9788c9.json
View File

@@ -48,16 +48,10 @@
"result": [
{
"type": "active_url_from_accessTree",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
{
"type": "active_tab_html_parse",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{},
@@ -74,9 +68,6 @@
},
{
"type": "active_tab_html_parse",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{

3
evaluation_examples/examples/chrome/6c4c23a1-42a4-43cc-9db1-2f86ff3738cc.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{

3
evaluation_examples/examples/chrome/7f52cab9-535c-4835-ac8c-391ee64dc930.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"parse_keys": ["q", "tbs"]
},

3
evaluation_examples/examples/chrome/82279c77-8fc6-46f6-9622-3ba96f61b477.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"parse_keys": ["list_price_max", "maximum_distance", "zip","fuel_slugs[]"]
},

3
evaluation_examples/examples/chrome/82bc8d6a-36eb-4d2d-8801-ef714fb1e55a.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"parse_keys": ["fromStation", "toStation", "departing"],
"replace":{

8
evaluation_examples/examples/chrome/9f3f70fc-5afc-4958-a7b7-3bb4fcb01805.json
View File

@@ -47,16 +47,10 @@
"conj": "and",
"result": [
{
"type": "active_tab_info",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
]
"type": "active_tab_info"
},
{
"type": "active_tab_html_parse",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"category": "xpath",
"xpathObject":{
"/html/body/div[2]/div/div[6]/div[2]/div[2]/div/div[1]/div[4]/ul/li[2]": "money"

3
evaluation_examples/examples/chrome/9f935cce-0a9f-435f-8007-817732bfc0a5.json
View File

@@ -46,9 +46,6 @@
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
"expected":{

3
evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json
View File

@@ -56,9 +56,6 @@
"func":"is_expected_active_tab",
"result": {
"type": "active_tab_info",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
"expected":{

3
evaluation_examples/examples/chrome/b070486d-e161-459b-aa2b-ef442d973b92.json
View File

@@ -46,9 +46,6 @@
"func":"exact_match",
"result": {
"type": "url_dashPart",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"partIndex": -1,
"needDeleteId": false,

3
evaluation_examples/examples/chrome/b4f95342-463e-4179-8c3f-193cd7241fb2.json
View File

@@ -47,9 +47,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category":"class",
"class_singleObject":{},

3
evaluation_examples/examples/chrome/b7895e80-f4d1-4648-bee0-4eb45a6f1fa8.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "xpath",
"xpathObject":{

3
evaluation_examples/examples/chrome/c1fa57f3-c3db-4596-8f09-020701085416.json
View File

@@ -56,9 +56,6 @@
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
"expected":{

3
evaluation_examples/examples/chrome/cabb3bae-cccb-41bd-9f5d-0f3a9fecd825.json
View File

@@ -46,9 +46,6 @@
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_url_from_accessTree",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
"expected":{

11
evaluation_examples/examples/chrome/da46d875-6b82-4681-9284-653b0c7ae241.json
View File

@@ -47,16 +47,10 @@
"conj": "and",
"result": [
{
"type": "active_tab_info",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
]
"type": "active_tab_info"
},
{
"type": "active_tab_html_parse",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"category": "class",
"class_singleObject":{},
"class_multiObject":{
@@ -68,9 +62,6 @@
},
{
"type": "active_tab_html_parse",
"selectors": [
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"category": "input",
"inputObject":{
"/html/body/div/div/form/div[7]/div/div/div[1]/input[1]": "name",

3
evaluation_examples/examples/chrome/f3b19d1e-2d48-44e9-b4e1-defcae1a0197.json
View File

@@ -32,9 +32,6 @@
"func":"is_expected_url_pattern_match",
"result": {
"type": "active_tab_info",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www."
},
"expected":{

3
evaluation_examples/examples/chrome/f79439ad-3ee8-4f99-a518-0eb60e5652b0.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_url_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"parse_keys":["originIata", "destinationIata", "tpAdults", "tpTeens", "tpChildren", "tpStartDate", "isReturn"],
"replace":{

3
evaluation_examples/examples/chrome/fc6d8143-9452-4171-9459-7f515143419a.json
View File

@@ -46,9 +46,6 @@
"func":"check_direct_json_object",
"result": {
"type": "active_tab_html_parse",
"selectors":[
"application[name=Chromium] entry[name=Address\\ and\\ search\\ bar]"
],
"goto_prefix": "https://www.",
"category": "class",
"class_singleObject":{