lzy/sci-gui-agent-benchmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update google drive file link in json

Browse Source
This commit is contained in:
Jason Lee
2024-03-09 18:06:48 +08:00
parent d68e7369b7
commit 2291af394f
16 changed files with 117 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
desktop_env/evaluators/getters/chrome.py
View File

@@ -1075,8 +1075,8 @@ def get_data_delete_automacally(env, config: Dict[str, str]):
try:
content = env.controller.get_file(preference_file_path)
data = json.loads(content)
data_delete_state = data["profile"]["exit_type"]
return data_delete_state
data_delete_state = data["profile"].get("default_content_setting_values", None)
return "true" if data_delete_state is not None else "false"
except Exception as e:
logger.error(f"Error: {e}")
return "Google"

11
desktop_env/evaluators/metrics/calc.py
View File

@@ -17,8 +17,11 @@ def compare_conference_city_in_order( actual_city_list_path, expected_city):
print("actual_city_list_path:")
print(actual_city_list)
wrong_list = []
for i in range(len(actual_city_list)):
if expected_city_list[i] not in actual_city_list[i]:
wrong_list.append(i)
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
try:
for i in range(len(actual_city_list)):
if expected_city_list[i] not in actual_city_list[i]:
wrong_list.append(i)
print(f"Expected city {expected_city_list[i]}; Actual city {actual_city_list[i]}")
except:
return False
return True if len(wrong_list) == 0 else False

13
desktop_env/evaluators/metrics/general.py
View File

@@ -98,13 +98,16 @@ def fuzzy_place_math(result_file_path, rules) -> float:
for para in doc.paragraphs:
words_list.extend(para.text.split())
# Print out the list of extracted words
print("Your Answers: ")
print(words_list)
fuzzy_score_list = []
for word in words_list:
if not any(ans in word for ans in expect):
print("Wrong place:", word)
return 0
return 1
max_score = 0
for ans in expect:
score = fuzz.ratio(word, ans)
max_score = max(max_score, score)
fuzzy_score_list.append(max_score)
return sum(fuzzy_score_list) / len(fuzzy_score_list)
def check_csv(result: str, rules: Dict[str, List[Dict[str, str]]]) -> float:
"""

11
evaluation_examples/examples/chrome/99146c54-4f37-4ab8-9327-5f3291665e1e.json
View File

@@ -1,7 +1,7 @@
{
"id": "99146c54-4f37-4ab8-9327-5f3291665e1e",
"snapshot": "chrome",
"instruction": "Please help me set Chrome to delete my browsing history automatically every time I close the browser.",
"instruction": "Please help me set Chrome to delete my browsing data automatically every time I close the browser.",
"source": "https://www.youtube.com/watch?v=v0kxqB7Xa6I",
"config": [
{
@@ -29,6 +29,13 @@
"chrome"
],
"evaluator": {
"postconfig":[{
"type": "execute",
"parameters": {
"command": "pkill chrome",
"shell": "true"
}
}],
"func": "exact_match",
"result": {
"type": "data_delete_automacally"
@@ -36,7 +43,7 @@
"expected": {
"type": "rule",
"rules": {
"expected": "Crashed"
"expected": "true"
}
}
}

2
evaluation_examples/examples/multi_apps/2373b66a-092d-44cb-bfd7-82e86e7a3b4d.json
View File

@@ -7,7 +7,7 @@
{
"type": "command",
"parameters":{
"command": "echo lixiaochuan20 | sudo -S apt-get update && echo lixiaochuan20 | sudo -S apt-get install sysstat",
"command": "echo password | sudo -S apt-get update && echo password | sudo -S apt-get install sysstat",
"shell": "true"
}
}

2
evaluation_examples/examples/multi_apps/3eb2a122-a5e3-4f89-9820-f7fa1a582969.json
View File

@@ -26,7 +26,7 @@
{
"type": "command",
"parameters":{
"command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc",
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}

4
evaluation_examples/examples/multi_apps/5bc63fb9-276a-4439-a7c1-9dc76401737f.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1XIJHJ6jtYITRG6vA1-rQlnKegIk0u6j6",
"url": "https://drive.google.com/uc?export=download&id=1OtN8poSktw5s9w7y28wr5JmXszMOpn2q",
"path": "/home/user/Desktop/llm_answers.json"
}
]
@@ -55,7 +55,7 @@
"func": "compare_docx_files_and_ignore_new_lines",
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1N5LlYhrRWtcC24LKiDQAYJYqCDYMBh9b",
"path": "https://drive.google.com/uc?export=download&id=1gmg5mC7qfmPvspC-ICOIJJ3aRjuSvP8u",
"dest": "gemini_results_Gold.docx"
},
"result": {

42
evaluation_examples/examples/multi_apps/6f4073b8-d8ea-4ade-8a18-c5d1d5d5aa9a.json
View File

@@ -9,23 +9,12 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1MhoRWCk-_ZZYg1agfQWHaOEYxXOrXLKC",
"url": "https://drive.google.com/uc?export=download&id=19wUxTQeoKr6ihJWJ_9cu2tzKQH0cnxWH",
"path": "/home/user/Desktop/ConferenceCity.xlsx"
}
]
}
},
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1p10oM6S3iZ6x2UVCgZE1mQ9D1yeZGCbO",
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
}
]
}
},
{
"type": "open",
"parameters": {
@@ -38,6 +27,35 @@
"calc", "chrome", "os"
],
"evaluator": {
"postconfig":[
{
"type": "download",
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1ZcITkIOs2Z86S5L6MShSohFs3_xVfeCP",
"path": "/home/user/Desktop/ConferenceCity_Gold.xlsx"
}
]
}
},
{
"type": "activate_window",
"parameters": {
"window_name": "ConferenceCity.xlsx - LibreOffice Calc"
}
},
{
"type": "execute",
"parameters": {
"command": [
"python",
"-c",
"import pyautogui; import time; pyautogui.hotkey(\"ctrl\", \"s\"); time.sleep(0.5); pyautogui.press(\"enter\");"
]
}
}
],
"func": "compare_conference_city_in_order",
"expected": {
"type": "rule",

4
evaluation_examples/examples/multi_apps/716a6079-22da-47f1-ba73-c9d58f986a38.json
View File

@@ -16,7 +16,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1kjfHFNmAy4ZVqGgFsxPDxWwMuZkCHxAv",
"url": "https://drive.google.com/uc?export=download&id=1iaqtXHZs2sA11nmoLHPK9VXGO0_6xq2l",
"path": "/home/user/Data3/List3/secret.docx"
}
]
@@ -25,7 +25,7 @@
{
"type": "command",
"parameters":{
"command": "echo lixiaochuan20 | sudo -S apt install xsel && xsel -bc",
"command": "echo password | sudo -S apt install xsel && xsel -bc",
"shell": "true"
}
}

37
evaluation_examples/examples/multi_apps/7ff48d5b-2df2-49da-b500-a5150ffc7f18.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1wrxlOrxZr3I5vaF1KRzc9nA9UcWW2ReU",
"url": "https://drive.usercontent.google.com/download?id=1H-GVXzyn30zxBKw02w5p5ZQzcwEZSlE4&export=download&authuser=0&confirm=t&uuid=da987f7a-7aa5-43d9-8426-741a97393726&at=APZUnTWyce0qNBfoOjOZ5pNRwIvX:1709971794981",
"path": "/home/user/Desktop/AllLocations.docx"
}
]
@@ -42,7 +42,40 @@
"expected": {
"type": "rule",
"rules":{
"expected": ["正义", "海天综合大厦", "盛世家园", "香梅路", "金沙嘴大厦", "卓悦汇", "昌泰公寓", "天安数码城", "振兴路", "梅岗南街", "滨河大道", "上沙", "香蜜原著", "八卦岭", "竹园", "竹子林五路", "体育中心", "赤尾", "书城中心广场", "侨城东路", "沙头所", "水围村", "黄木岗", "华强广场", "国大药房","科学馆","江苏宾馆", "梅林"]
"expected": [
"深圳市福田区益田路5055号信息枢纽大厦西门一楼",
"深圳市福田区福华三路111号北三门会展中心警务室",
"深圳市福田区正义街1号",
"福田区莲科路18号莲花一村警务室",
"深圳市福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"深圳市福田区香梅路2002-4号",
"福田区水围村龙景楼一楼",
"深圳市福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"深圳市福田区福强路3028号金沙嘴大厦",
"深圳市福田区天安数码城昌泰公寓一楼",
"福田区泰然五路5号天安数码城9栋",
"深圳市福田区振兴路108号",
"深圳市福田区滨河大道2033号",
"深圳市福田区上沙四十八栋一巷11",
"深圳市福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"深圳市福田区八卦路38号八卦岭派出所",
"深圳市福田区宝能城市公馆B栋一楼竹园警务室",
"深圳市福田区竹子林五路12号",
"福田区福强路3028号金沙嘴大厦",
"福田区彩云路2-8长城盛世家园一期C座一楼一期管理处旁边",
"福田区益田路5055号信息枢纽大厦西门一楼",
"福田区正义街1号",
"福田区香梅路2002-4号",
"福田区梅林路与梅康路交汇处卓悦汇4号、5号门对面",
"福田区天安数码城昌泰公寓一楼",
"福田区振兴路108号",
"福田区滨河大道2033号",
"福田区上沙四十八栋一巷11",
"福田区北环大道与香蜜湖路交汇处香蜜原著警务室",
"福田区八卦路38号八卦岭派出所",
"福田区宝能城市公馆B栋一楼竹园警务室",
"福田区竹子林五路12号"
]
}
}
}

2
evaluation_examples/examples/multi_apps/873cafdd-a581-47f6-8b33-b9696ddb7b05.json
View File

@@ -28,7 +28,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1el4WdiJVpUP49FABN_DpC7yy7obgJSC5",
"url": "https://drive.google.com/uc?export=download&id=1EMpoFNW3reLgZE0lj6scRFmL-OPjWuzf",
"path": "/home/user/Desktop/Recommended_plugin_list.docx"
}
]

21
evaluation_examples/examples/multi_apps/9f3bb592-209d-43bc-bb47-d77d9df56504.json
View File

@@ -9,19 +9,12 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1YrPG1sXDACdXlrBlW8-RUzovWUJ0qoC5",
"path": "/home/user/video.txt"
"url": "https://drive.usercontent.google.com/download?id=17vIS65RVA7e9n5u8lLNqwjBl56MQpkk8&export=download&authuser=0&confirm=t&uuid=1c8882db-dd55-450d-a998-52c03bfaf801&at=APZUnTWPObUX-3Pqtusro4T5Ga5B:1709972109572",
"path": "/home/user/video.mp4"
}
]
}
},
{
"type": "execute",
"parameters":{
"command":"echo lixiaochuan20 | sudo -S mv /home/user/video.txt /home/user/video.mp4",
"shell": "true"
}
},
{
"type": "open",
"parameters": {
@@ -46,7 +39,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1Q6GV39h74bc_8_kFKjNW3RMxUSgXaaWR",
"url": "https://drive.usercontent.google.com/download?id=1oMCDx2hqj0xzkQP4le0Mn09y-vvCYXlt&export=download&authuser=0&confirm=t&uuid=2a2cac5c-f031-41f7-8504-4a013c02ee15&at=APZUnTUa188AtQ93qZG8HInS_o90:1709972317568",
"path": "/home/user/subtitles_Gold.srt"
}
]
@@ -57,8 +50,8 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=14eBaLVfUekkpj3dPkdDwHehHRmzx2njy",
"path": "/home/user/Desktop/subtitles_script.txt"
"url": "https://drive.usercontent.google.com/download?id=19ooP4gPY2HYQVI9n31CtMlJ3q-1dr6cU&export=download&authuser=0&confirm=t&uuid=ba968219-3a9b-44d1-bee6-64e289ce4ffe&at=APZUnTXlL0jQBoia7KuzELsBCVxE:1709972303196",
"path": "/home/user/Desktop/subtitles_script.py"
}
]
}
@@ -66,7 +59,7 @@
{
"type": "command",
"parameters": {
"command": "echo lixiaochuan20 | sudo -S pip install pysrt",
"command": "echo password | sudo -S pip install pysrt",
"shell" :"true"
}
}
@@ -74,7 +67,7 @@
"func": "exact_match",
"result": {
"type": "vm_command_line",
"command": "echo lixiaochuan20 | mv /home/user/Desktop/subtitles_script.txt /home/user/Desktop/subtitles_script.py && python /home/user/Desktop/subtitles_script.py",
"command": "python /home/user/Desktop/subtitles_script.py",
"shell" :"true"
},
"expected": {

2
evaluation_examples/examples/multi_apps/a74b607e-6bb5-4ea8-8a7c-5d97c7bbcd2a.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files":[
{
"url":"https://drive.google.com/uc?id=1t-6Qrpgox0UvqQ0EAhkCnSV8Rqsp4cMF&export=download",
"url":"https://drive.google.com/uc?export=download&id=1fe-MuG57hf12cEi4qI4dEI3OYJOHYoY2",
"path":"/home/user/Desktop/helloExtension.zip"
}
]

12
evaluation_examples/examples/multi_apps/ce2b64a2-ddc1-4f91-8c7d-a88be7121aac.json
View File

@@ -9,7 +9,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1LSFS3AI4t5aOIEUby73ZHvURqnx5cw7C",
"url": "https://drive.usercontent.google.com/download?id=1sKdxRx4Uxs-wxohYkmxPfxOUpw72BINV&export=download&authuser=0&confirm=t&uuid=2cd18b66-93a6-4771-a92b-a1b1f0adf12d&at=APZUnTUvT9iZ0IdRnnNf1ys4mZaE:1709972639977",
"path": "/home/user/Pictures/picture1.jpg"
}
]
@@ -20,7 +20,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1pdRoU_FnqoDgenvp1WzFJZrsGnsexRoo",
"url": "https://drive.usercontent.google.com/download?id=1iquZNI4ktx2915srQ1MMlhcwCuXnmq5T&export=download&authuser=0&confirm=t&uuid=e0bfd86f-a0f1-4a9e-8344-0f645eb379dd&at=APZUnTWQnCd7bNtf8ZN4KfkQUaJ9:1709972645590",
"path": "/home/user/Pictures/picture2.jpg"
}
@@ -32,7 +32,7 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1ajGd9b_YvE1Ay59Pqh8RnePgbnepGc_g",
"url": "https://drive.usercontent.google.com/download?id=1ie3rhHzsW293xhnmgeZm6wQj_uaFmPS7&export=download&authuser=0&confirm=t&uuid=633c8b0c-0d3f-4f07-a772-945148956244&at=APZUnTX6OIN5YfudwPVT3Mxntd9E:1709972650733",
"path": "/home/user/Pictures/picture3.jpg"
}
@@ -51,8 +51,8 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=1cbBn9BqtkmiZH-nNJAfQ_RYjaAUGHHTG",
"path": "/home/user/Desktop/image_script.txt"
"url": "https://drive.usercontent.google.com/download?id=1eWGQ3z-Ps5p784NZjj9KyhHfX7NbHgWd&export=download&authuser=0&confirm=t&uuid=8ba167f4-8669-477b-89ee-a8f6996f43d7&at=APZUnTWShYYueBb8hRnrjjabU1ZD:1709972787510",
"path": "/home/user/Desktop/image_script.py"
}
]
}
@@ -61,7 +61,7 @@
"func": "check_direct_json_object",
"result":{
"type": "vm_command_line",
"command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/image_script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py",
"command":"python /home/user/Desktop/image_script.py",
"shell": "true"
},
"expected": {

6
evaluation_examples/examples/multi_apps/da922383-bfa4-4cd3-bbad-6bebab3d7742.json
View File

@@ -44,8 +44,8 @@
"parameters": {
"files": [
{
"url": "https://drive.google.com/uc?export=download&id=19SJ4ydB36lwIK4K4B57sl3bsY9T-3WNv",
"path": "/home/user/Desktop/script.txt"
"url": "https://drive.usercontent.google.com/download?id=1fm4wqp91_3yzVSlenc9k2Tlh3-7QuRvu&export=download&authuser=0&confirm=t&uuid=5bb46f9e-ef04-4b52-bceb-a88f0058f696&at=APZUnTU6aWNzsiqLwAlyr_8pxKUd:1709971656456",
"path": "/home/user/Desktop/script.py"
}
]
}
@@ -61,7 +61,7 @@
"func": "exact_match",
"result":{
"type": "vm_command_line",
"command":"echo lixiaochuan20 | sudo -S mv /home/user/Desktop/script.txt /home/user/Desktop/script.py && python /home/user/Desktop/script.py",
"command":"python /home/user/Desktop/script.py",
"shell": "true"
},
"expected": {

4
evaluation_examples/examples/multi_apps/dd60633f-2c72-42ba-8547-6f2c8cb0fdb0.json
View File

@@ -45,8 +45,8 @@
},
"expected": {
"type": "cloud_file",
"path": "https://drive.google.com/uc?export=download&id=1Z1Vl7SOXf0yjoKttkZD1oYbLM7TgXr5s",
"dest": "colab_Gold.txt"
"path": "https://drive.usercontent.google.com/download?id=1r0VyQZf5Hi8n8R9w9iGKMsqQaxAjYN7O&export=download&authuser=0&confirm=t&uuid=56efd2ff-4494-4026-9e96-82783b5ed600&at=APZUnTUL7BOMtV3FIoc0kXe18LWz:1709972550788",
"dest": "colab_Gold.py"
}
}
}