lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve on eval script on web browsing tasks; Add one setup example

Browse Source
This commit is contained in:
Timothyxxx
2024-02-23 11:57:50 +08:00
parent 543666ff4d
commit 81863b26dd
4 changed files with 72 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
desktop_env/envs/desktop_env.py
View File

@@ -316,6 +316,9 @@ class DesktopEnv(gym.Env):
return 1 return 1
else: else:
return 0 return 0
else:
if len(self.action_history) > 0 and self.action_history[-1] == "FAIL":
return 0
if type(self.metric) == list: if type(self.metric) == list:
results = [] results = []

2
desktop_env/evaluators/metrics/chrome.py
View File

@@ -17,7 +17,7 @@ def is_expected_active_tab(active_tab_info: Dict[str, str], rule: Dict[str, Any]
if match_type == "url": if match_type == "url":
expected_url = rule['url'] expected_url = rule['url']
actual_url = active_tab_info['url'] actual_url = active_tab_info.get('url', None)
print("expected_url: {}".format(expected_url)) print("expected_url: {}".format(expected_url))
print("actual_url: {}".format(actual_url)) print("actual_url: {}".format(actual_url))
return 1 if compare_urls(expected_url, actual_url) else 0 return 1 if compare_urls(expected_url, actual_url) else 0

3
desktop_env/evaluators/metrics/utils.py
View File

@@ -625,6 +625,9 @@ def are_lists_equal(list1, list2, comparison_func):
def compare_urls(url1, url2): def compare_urls(url1, url2):
if url1 is None or url2 is None:
return url1 == url2
def normalize_url(url): def normalize_url(url):
# Parse the URL # Parse the URL
parsed_url = urlparse(url) parsed_url = urlparse(url)

105
evaluation_examples/examples/chrome/a96b564e-dbe9-42c3-9ccf-b4498073438a.json
View File

@@ -1,45 +1,70 @@
{ {
"id": "a96b564e-dbe9-42c3-9ccf-b4498073438a", "id": "a96b564e-dbe9-42c3-9ccf-b4498073438a",
"snapshot": "chrome", "snapshot": "chrome",
"instruction": "Find discussions of community and open one with most replies.", "instruction": "Find discussions of community and open one with most replies.",
"source": "test_task_0", "change_possibility": "low",
"config": [ "source": "test_task_0",
{ "config": [
"type": "launch", {
"parameters": { "type": "launch",
"command": [ "parameters": {
"google-chrome", "command": [
"--remote-debugging-port=1337" "google-chrome",
] "--remote-debugging-port=1337"
} ]
}, }
{ },
"type": "launch", {
"parameters": { "type": "launch",
"command": [ "parameters": {
"socat", "command": [
"tcp-listen:9222,fork", "socat",
"tcp:localhost:1337" "tcp-listen:9222,fork",
] "tcp:localhost:1337"
} ]
}
},
{
"type": "chrome_open_tabs",
"parameters": {
"urls_to_open": [
"https://www.flightaware.com/"
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "Google Chrome"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey('alt', 'f10'); time.sleep(0.5);"
]
}
}
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func": "is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected": {
"type": "rule",
"rules": {
"type": "url",
"url": "https://discussions.flightaware.com/t/the-banter-thread/4412"
} }
],
"trajectory": "trajectories/",
"related_apps": [
"chrome"
],
"evaluator": {
"func":"is_expected_active_tab",
"result": {
"type": "active_tab_info"
},
"expected":{
"type": "rule",
"rules":{
"type": "url",
"url": "https://discussions.flightaware.com/t/graphs-for-dump1090-my-version-with-install-script/46263"
}
}
} }
} }
}