From 8d69eec68fe616ab3b9d76a8da8f7d3b42ece289 Mon Sep 17 00:00:00 2001 From: Timothyxxx <384084775@qq.com> Date: Wed, 14 Feb 2024 16:51:07 +0800 Subject: [PATCH] Update infeasible examples from Chrome and Calc --- .../3720f614-37fd-4d04-8a6b-76f54f8c222d.json | 19 ++++++ .../93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json | 19 ++++++ .../ae78f875-5b98-4907-bbb5-9c737fc68c03.json | 26 +------- .../2bd59342-0664-4ccb-ba87-79379096cc08.json | 66 +------------------ .../7b802dad-6e0f-4204-9815-d4e3f57627d8.json | 19 ++++++ 5 files changed, 60 insertions(+), 89 deletions(-) create mode 100644 evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json create mode 100644 evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json create mode 100644 evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json diff --git a/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json b/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json new file mode 100644 index 0000000..ee0b3c5 --- /dev/null +++ b/evaluation_examples/examples/chrome/3720f614-37fd-4d04-8a6b-76f54f8c222d.json @@ -0,0 +1,19 @@ +{ + "id": "3720f614-37fd-4d04-8a6b-76f54f8c222d", + "snapshot": "chrome", + "instruction": "I am more familiar with Korean as I am from Korea. I want to use chrome with my mother tongue. Could you help me change the Chrome interface language to Korean? ", + "source": "https://superuser.com/questions/984668/change-interface-language-of-chrome-to-english", + "config": [ + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "infeasible", + "result": { + }, + "expected": { + } + } +} diff --git a/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json b/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json new file mode 100644 index 0000000..331a46e --- /dev/null +++ b/evaluation_examples/examples/chrome/93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9.json @@ -0,0 +1,19 @@ +{ + "id": "93eabf48-6a27-4cb6-b963-7d5fe1e0d3a9", + "snapshot": "chrome", + "instruction": "Could you assist me in turning off the dark mode feature in Google Chrome? I've noticed that while dark mode is great for reducing glare, it actually makes it more challenging for me to read text clearly, especially with my astigmatism.", + "source": "https://superuser.com/questions/1417973/how-to-disable-google-chrome-dark-mode", + "config": [ + ], + "trajectory": "trajectories/", + "related_apps": [ + "chrome" + ], + "evaluator": { + "func": "infeasible", + "result": { + }, + "expected": { + } + } +} diff --git a/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json b/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json index 41c0dea..e19fdc5 100644 --- a/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json +++ b/evaluation_examples/examples/chrome/ae78f875-5b98-4907-bbb5-9c737fc68c03.json @@ -4,40 +4,16 @@ "instruction": "Could you please change the number of search results displayed on one page to 50? I find that having more results visible at once significantly enhances my research efficiency, as it reduces the need to constantly click through multiple pages. ", "source": "https://support.google.com/chrome/thread/219988391/increase-search-results-per-page?hl=en", "config": [ - { - "type": "launch", - "parameters": { - "command": [ - "google-chrome", - "--remote-debugging-port=1337" - ] - } - }, - { - "type": "launch", - "parameters": { - "command": [ - "socat", - "tcp-listen:9222,fork", - "tcp:localhost:1337" - ] - } - } ], "trajectory": "trajectories/", "related_apps": [ "chrome" ], "evaluator": { - "func": "exact_match", + "func": "infeasible", "result": { - "type": "number_of_search_results" }, "expected": { - "type": "rule", - "rules": { - "expected": 50 - } } } } diff --git a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json index 6b92cc1..aba58cd 100644 --- a/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json +++ b/evaluation_examples/examples/libreoffice_calc/2bd59342-0664-4ccb-ba87-79379096cc08.json @@ -4,78 +4,16 @@ "instruction": "Make sparkline chart line by line", "source": "https://www.youtube.com/shorts/L3Z-F1QTQFY", "config": [ - { - "type": "download", - "parameters": { - "files": [ - { - "url": "https://drive.usercontent.google.com/download?id=1uywX5XWMvesnb4-8LPKEzr2HFU7HmoIu&export=download&authuser=0&confirm=t&uuid=267bfe49-a861-4272-ae7c-39c95df35e84&at=APZUnTUbs-FF06hSMv3yWfdXc02l:1703508870351", - "path": "/home/user/OrderId_Month_Chart.xlsx" - } - ] - } - }, - { - "type": "open", - "parameters": { - "path": "/home/user/OrderId_Month_Chart.xlsx" - } - } ], "trajectory": "trajectories/2bd59342-0664-4ccb-ba87-79379096cc08", "related_apps": [ - "libreoffice calc" + "libreoffice_calc" ], "evaluator": { - "postconfig": [ - { - "type": "activate_window", - "parameters": { - "window_name": "OrderId_Month_Chart.xlsx - LibreOffice Calc", - "strict": true - } - }, - { - "type": "sleep", - "parameters": { - "seconds": 0.5 - } - }, - { - "type": "execute", - "parameters": { - "command": [ - "python", - "-c", - "import pyautogui; pyautogui.hotkey(\"ctrl\", \"s\");" - ] - } - } - ], - "func": "compare_table", + "func": "infeasible", "expected": { - "type": "cloud_file", - "path": "https://drive.usercontent.google.com/download?id=1KQJJLVPGtTL_7ArEWvwwbFbJSiA3cgSE&export=download&authuser=0&confirm=t&uuid=6b11c721-caad-439a-b369-4c13c7a485df&at=APZUnTV5-1isKrDKSHV9NeJ6TDeS:1703509054094", - "dest": "OrderId_Month_Chart_gold.xlsx" }, "result": { - "type": "vm_file", - "path": "/home/user/OrderId_Month_Chart.xlsx", - "dest": "OrderId_Month_Chart.xlsx" - }, - "options": { - "rules": [ - { - "type": "sparkline", - "sheet_idx0": 0, - "sheet_idx1": "EI0" - }, - { - "type": "sheet_data", - "sheet_idx0": 0, - "sheet_idx1": "EI0" - } - ] } } } \ No newline at end of file diff --git a/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json b/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json new file mode 100644 index 0000000..0ebfeaf --- /dev/null +++ b/evaluation_examples/examples/libreoffice_calc/7b802dad-6e0f-4204-9815-d4e3f57627d8.json @@ -0,0 +1,19 @@ +{ + "id": "7b802dad-6e0f-4204-9815-d4e3f57627d8", + "snapshot": "libreoffice_calc", + "instruction": "I would like to sort this table based on cell color, placing all the rows marked with pink at the beginning, while keeping their order among themselves unchanged.", + "source": "https://www.youtube.com/shorts/Of-lzeP1usE", + "config": [ + ], + "trajectory": "trajectories/7b802dad-6e0f-4204-9815-d4e3f57627d8", + "related_apps": [ + "libreoffice_calc" + ], + "evaluator": { + "func": "infeasible", + "expected": { + }, + "result": { + } + } +} \ No newline at end of file